From 5ea54bf8fa47ece322c690f762d1149746c7b758 Mon Sep 17 00:00:00 2001 From: hollyhan Date: Fri, 15 Apr 2022 14:57:20 -0600 Subject: [PATCH 01/58] Add 'Ismip6Forcing' testcase to the landice core class --- compass/landice/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/compass/landice/__init__.py b/compass/landice/__init__.py index a006dad5a8..2c010dc486 100644 --- a/compass/landice/__init__.py +++ b/compass/landice/__init__.py @@ -8,6 +8,7 @@ from compass.landice.tests.greenland import Greenland from compass.landice.tests.humboldt import Humboldt from compass.landice.tests.hydro_radial import HydroRadial +from compass.landice.tests.ismip6_forcing import Ismip6Forcing from compass.landice.tests.kangerlussuaq import Kangerlussuaq from compass.landice.tests.koge_bugt_s import KogeBugtS from compass.landice.tests.mismipplus import MISMIPplus @@ -34,6 +35,7 @@ def __init__(self): self.add_test_group(Greenland(mpas_core=self)) self.add_test_group(Humboldt(mpas_core=self)) self.add_test_group(HydroRadial(mpas_core=self)) + self.add_test_group(Ismip6Forcing(mpas_core=self)) self.add_test_group(Kangerlussuaq(mpas_core=self)) self.add_test_group(KogeBugtS(mpas_core=self)) self.add_test_group(MISMIPplus(mpas_core=self)) From e9fb4ac9305c4a943990b50cf8e076c0e0ccff9d Mon Sep 17 00:00:00 2001 From: hollyhan Date: Fri, 15 Apr 2022 14:59:36 -0600 Subject: [PATCH 02/58] Create a testgroup class "Ismip6Forcing" --- .../landice/tests/ismip6_forcing/__init__.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 compass/landice/tests/ismip6_forcing/__init__.py diff --git a/compass/landice/tests/ismip6_forcing/__init__.py b/compass/landice/tests/ismip6_forcing/__init__.py new file mode 100644 index 0000000000..e033e8fac8 --- /dev/null +++ b/compass/landice/tests/ismip6_forcing/__init__.py @@ -0,0 +1,19 @@ +from compass.testgroup import TestGroup +from compass.landice.tests.ismip6_forcing.atmosphere import Atmosphere +from compass.landice.tests.ismip6_forcing.ocean import Ocean + + +class Ismip6Forcing(TestGroup): + """ + A test group for processing the ISMIP6 + ocean and atmosphere forcing data + """ + def __init__(self, mpas_core): + """ + mpas_core : compass.landice.Landice + the MPAS core that this test group belongs to + """ + super().__init__(mpas_core=mpas_core, name='ismip6_forcing') + + self.add_test_case(Atmosphere(test_group=self)) + self.add_test_case(Ocean(test_group=self)) From 68977846602912b43370f41be477d8f9fb6fae28 Mon Sep 17 00:00:00 2001 From: hollyhan Date: Fri, 15 Apr 2022 15:08:16 -0600 Subject: [PATCH 03/58] Add a testcase for processing ismip6 AIS atmosphere forcing data --- .../ismip6_forcing/atmosphere/__init__.py | 45 +++++ .../atmosphere/create_mapfile_smb.py | 150 ++++++++++++++++ .../ismip6_forcing/atmosphere/process_smb.py | 161 ++++++++++++++++++ 3 files changed, 356 insertions(+) create mode 100644 compass/landice/tests/ismip6_forcing/atmosphere/__init__.py create mode 100644 compass/landice/tests/ismip6_forcing/atmosphere/create_mapfile_smb.py create mode 100644 compass/landice/tests/ismip6_forcing/atmosphere/process_smb.py diff --git a/compass/landice/tests/ismip6_forcing/atmosphere/__init__.py b/compass/landice/tests/ismip6_forcing/atmosphere/__init__.py new file mode 100644 index 0000000000..432982f68f --- /dev/null +++ b/compass/landice/tests/ismip6_forcing/atmosphere/__init__.py @@ -0,0 +1,45 @@ +from compass.validate import compare_variables +from compass.testcase import TestCase +from compass.landice.tests.ismip6_forcing.atmosphere.process_smb \ + import ProcessSMB + +class Atmosphere(TestCase): + """ + A test case for processing the ISMIP6 atmosphere forcing data. + The test case builds a mapping file for interpolation between the + ISMIP6 8km polarstereo grid and MALI mesh, regrids the forcing data + and rename the ISMIP6 variables to corresponding MALI variables. + + Attributes + ---------- + mesh_type : str >>>>>>> what kind of attributes would we have? + The resolution or type of mesh of the test case + """ + + def __init__(self, test_group): + """ + Create the test case + + Parameters + ---------- + test_group : compass.landice.tests.ismip6_forcing.Ismip6Forcing + The test group that this test case belongs to + """ + name = 'atmosphere' + subdir = name + super().__init__(test_group=test_group, name=name, subdir=subdir) + + step = ProcessSMB(test_case=self) + self.add_step(step) + + def configure(self): + """ + Configures test case + """ + input_path = self.config.get(section="ismip6_ais_atmosphere", + option="input_path") + if input_path == "NotAvailable": + raise ValueError("You need to supply a user config file, which " + "should contain the ismip6_ais_atmosphere section" + "with the input_path option") + diff --git a/compass/landice/tests/ismip6_forcing/atmosphere/create_mapfile_smb.py b/compass/landice/tests/ismip6_forcing/atmosphere/create_mapfile_smb.py new file mode 100644 index 0000000000..354a0184d7 --- /dev/null +++ b/compass/landice/tests/ismip6_forcing/atmosphere/create_mapfile_smb.py @@ -0,0 +1,150 @@ +import os +import subprocess +import netCDF4 +import xarray as xr +from mpas_tools.scrip.from_mpas import scrip_from_mpas + + +# function that creates a mapping file from ismip6 grid to mali mesh +def build_mapping_file(ismip6_grid_file, mapping_file, mali_mesh_file=None, + method_remap=None): + """ + Build a mapping file if it does not exist. + + Parameters + ---------- + ismip6_grid_file : str + ismip6 grid file + mapping_file : str + weights for interpolation from ismip6_grid_file to mali_mesh_file + mali_mesh_file : str, optional + The MALI mesh file if mapping file does not exist + method_remap : str, optional + Remapping method used in building a mapping file + """ + + if os.path.exists(mapping_file): + print("Mapping file exists. Not building a new one.") + return + + if mali_mesh_file is None: + raise ValueError("Mapping file does not exist. To build one, Mali " + "mesh file with '-f' should be provided. " + "Type --help for info") + + ismip6_scripfile = "temp_ismip6_8km_scrip.nc" + mali_scripfile = "temp_mali_scrip.nc" + ismip6_projection = "ais-bedmap2" + + # create the ismip6 scripfile if mapping file does not exist + # this is the projection of ismip6 data for Antarctica + print("Mapping file does not exist. Building one based on the " + "input/ouptut meshes") + print("Creating temporary scripfiles for ismip6 grid and mali mesh...") + + # create a scripfile for the atmosphere forcing data + create_atm_scrip(ismip6_grid_file, ismip6_scripfile) + + # create a MALI mesh scripfile if mapping file does not exist + # make sure the mali mesh file uses the longitude convention of [0 2pi] + args = ["set_lat_lon_fields_in_planar_grid.py", + "--file", mali_mesh_file, + "--proj", ismip6_projection] + + subprocess.check_call(args) + + # create a MALI mesh scripfile if mapping file does not exist + scrip_from_mpas(mali_mesh_file, mali_scripfile) + + # create a mapping file using ESMF weight gen + print("Creating a mapping file...") + + if method_remap is None: + raise ValueError("Desired remapping option should be provided with " + "--method. Available options are 'bilinear'," + "'neareststod', 'conserve'.") + + args = ["ESMF_RegridWeightGen", + "-s", ismip6_scripfile, + "-d", mali_scripfile, + "-w", mapping_file, + "-m", method_remap, + "-i", "-64bit_offset", + "--dst_regional", "--src_regional"] + + # include flag and input and output file names + subprocess.check_call(args) + + # remove the temporary scripfiles once the mapping file is generated + print("Removing the temporary scripfiles...") + os.remove(ismip6_scripfile) + os.remove(mali_scripfile) + + +def create_atm_scrip(ismip6_grid_file, ismip6_scripfile): + """ + create a scripfile for the ismip6 atmospheric forcing data. + Note: the atmospheric forcing data do not have 'x' and 'y' coordinates and + only have dimensions of them. This function uses 'lat' and 'lon' + coordinates to generate a scripfile. + + Parameters + ---------- + ismip6_grid_file : str + input ismip6 grid file + + ismip6_scripfile : str + outputput ismip6 scrip file + """ + + ds = xr.open_dataset(ismip6_grid_file) + out_file = netCDF4.Dataset(ismip6_scripfile, 'w') + + nx = ds.sizes["x"] + ny = ds.sizes["y"] + units = 'degrees' + + grid_size = nx * ny + + out_file.createDimension("grid_size", grid_size) + out_file.createDimension("grid_corners", 4) + out_file.createDimension("grid_rank", 2) + + # Variables + grid_center_lat = out_file.createVariable('grid_center_lat', 'f8', + ('grid_size',)) + grid_center_lat.units = units + grid_center_lon = out_file.createVariable('grid_center_lon', 'f8', + ('grid_size',)) + grid_center_lon.units = units + grid_corner_lat = out_file.createVariable('grid_corner_lat', 'f8', + ('grid_size', 'grid_corners')) + grid_corner_lat.units = units + grid_corner_lon = out_file.createVariable('grid_corner_lon', 'f8', + ('grid_size', 'grid_corners')) + grid_corner_lon.units = units + grid_imask = out_file.createVariable('grid_imask', 'i4', ('grid_size',)) + grid_imask.units = 'unitless' + out_file.createVariable('grid_dims', 'i4', ('grid_rank',)) + + out_file.variables['grid_center_lat'][:] = ds.lat.values.flat + out_file.variables['grid_center_lon'][:] = ds.lon.values.flat + out_file.variables['grid_dims'][:] = [nx, ny] # => check the scrip format docs + out_file.variables['grid_imask'][:] = 1 + + lat_corner = ds.lat_bnds + if "time" in lat_corner.dims: + lat_corner = lat_corner.isel(time=0) + + grid_corner_lat = lat_corner.values.reshape((grid_size, 4)) + + lon_corner = ds.lon_bnds + if "time" in lon_corner.dims: + lon_corner = lon_corner.isel(time=0) + + grid_corner_lon = lon_corner.values.reshape((grid_size, 4)) + + out_file.variables['grid_corner_lat'][:] = grid_corner_lat + out_file.variables['grid_corner_lon'][:] = grid_corner_lon + + out_file.close() diff --git a/compass/landice/tests/ismip6_forcing/atmosphere/process_smb.py b/compass/landice/tests/ismip6_forcing/atmosphere/process_smb.py new file mode 100644 index 0000000000..086f959eb7 --- /dev/null +++ b/compass/landice/tests/ismip6_forcing/atmosphere/process_smb.py @@ -0,0 +1,161 @@ +import os +import pandas as pd +import shutil +import subprocess +import xarray as xr +import numpy as np +from compass.landice.tests.ismip6_forcing.atmosphere.create_mapfile_smb import build_mapping_file +from mpas_tools.scrip.from_mpas import scrip_from_mpas +from mpas_tools.io import write_netcdf +from compass.step import Step + + +class ProcessSMB(Step): + """ + A step for processing the ISMIP6 surface mass balance data + """ + + def __init__(self, test_case): + """ + Create the step + + Parameters + ---------- + test_case : compass.landice.tests.ismip6_forcing.atmosphere.Atmosphere + The test case this step belongs to + """ + super().__init__(test_case=test_case, name='process_smb') + + def setup(self): + """ + Set up this step of the test case + """ + config = self.config + section = config['ismip6_ais_atmosphere'] + + input_path = section.get('input_path') + input_file = section.get('input_file') + self.add_input_file(filename=input_file, + target=os.path.join(input_path, input_file)) + mali_mesh_file = section.get('mali_mesh_file') + self.add_input_file(filename=mali_mesh_file, + target=os.path.join(input_path, mali_mesh_file)) + self.add_output_file(filename= f"output_{input_file}") + + def run(self): + """ + Run this step of the test case + """ + logger = self.logger + config = self.config + section = config['ismip6_ais_atmosphere'] + + input_file = section.get('input_file') + mali_mesh_name = section.get('mali_mesh_name') + mali_mesh_file = section.get('mali_mesh_file') + method_remap = section.get('method_remap') + output_file = f"output_{input_file}" + + # interpolate and rename the ismip6 thermal forcing data + remapped_file_temp = "remapped.nc" # temporary file name + + # call the function that reads in, remap and rename the file. + logger.info("Calling a remapping function...") + self.remap_ismip6smb_to_mali(input_file, remapped_file_temp, + mali_mesh_name, mali_mesh_file, + method_remap) + + # call the function that renames the ismip6 variables to MALI variables + logger.info("Renaming the ismip6 variables to mali variable names...") + self.rename_ismip6smb_to_mali_vars(remapped_file_temp, output_file) + + logger.info("Remapping and renamping process done successfully. " + "Removing the temporary file 'remapped.nc'") + + # remove the temporary combined file + os.remove(remapped_file_temp) + + def remap_ismip6smb_to_mali(self, input_file, output_file, mali_mesh_name, + mali_mesh_file, method_remap): + """ + Remap the input ismip6 thermal forcing data onto mali mesh + + Parameters + ---------- + input_file: str + ismip6 smb data on its native polarstereo 8km grid + output_file : str + ismip6 data remapped on mali mesh + mali_mesh_name : str + name of the mali mesh used to name mapping files + mali_mesh_file : str, optional + The MALI mesh file if mapping file does not exist + method_remap : str, optional + Remapping method used in building a mapping file + """ + + # check if a mapfile + mapping_file = f"map_ismip6_8km_to_{mali_mesh_name}_{method_remap}.nc" + + if not os.path.exists(mapping_file): + # build a mapping file if it doesn't already exist + build_mapping_file(input_file, mapping_file, mali_mesh_file, + method_remap) + else: + self.logger.info("Mapping file exists. " + "Remapping the input data...") + + # remap the input data + args = ["ncremap", + "-i", input_file, + "-o", output_file, + "-m", mapping_file, + "-v", "smb_anomaly"] + + subprocess.check_call(args) + + def rename_ismip6smb_to_mali_vars(self, remapped_file_temp, output_file): + """ + Rename variables in the remapped ismip6 input data + to the ones that MALI uses. + + Parameters + ---------- + remapped_file_temp : str + temporary ismip6 data remapped on mali mesh + output_file : str + remapped ismip6 data renamed on mali mesh + """ + + # open dataset in 20 years chunk + ds = xr.open_dataset(remapped_file_temp, chunks=dict(time=20), + engine="netcdf4") + + # build dictionary for ismip6 variables that MALI takes in + ismip6_to_mali_dims = dict( + time="Time", + ncol="nCells") + ds = ds.rename(ismip6_to_mali_dims) + + ismip6_to_mali_vars = dict( + smb_anomaly="sfcMassBal") + ds = ds.rename(ismip6_to_mali_vars) + + # add xtime variable + xtime = [] + for t_index in range(ds.sizes["Time"]): + date = ds.Time[t_index].values + date = pd.to_datetime(str(date)) + date = date.strftime("%Y-%m-%d_00:00:00").ljust(64) + xtime.append(date) + + ds["xtime"] = ("Time", xtime) + ds["xtime"] = ds.xtime.astype('S') + + # drop unnecessary variables + ds = ds.drop_vars(["lon", "lon_vertices", "lat", "lat_vertices", + "area"]) + + # write to a new netCDF file + write_netcdf(ds, output_file) + ds.close() From c5b15182efc3640bd97e2be2039de8786ede43f3 Mon Sep 17 00:00:00 2001 From: hollyhan Date: Fri, 15 Apr 2022 15:10:41 -0600 Subject: [PATCH 04/58] Add a testcase for processing ismip6 AIS ocean forcing data --- .../tests/ismip6_forcing/ocean/__init__.py | 44 +++++ .../ismip6_forcing/ocean/create_mapfile.py | 84 ++++++++ .../ocean/process_basal_melt.py | 185 ++++++++++++++++++ .../ocean/process_thermal_forcing.py | 172 ++++++++++++++++ 4 files changed, 485 insertions(+) create mode 100644 compass/landice/tests/ismip6_forcing/ocean/__init__.py create mode 100644 compass/landice/tests/ismip6_forcing/ocean/create_mapfile.py create mode 100644 compass/landice/tests/ismip6_forcing/ocean/process_basal_melt.py create mode 100644 compass/landice/tests/ismip6_forcing/ocean/process_thermal_forcing.py diff --git a/compass/landice/tests/ismip6_forcing/ocean/__init__.py b/compass/landice/tests/ismip6_forcing/ocean/__init__.py new file mode 100644 index 0000000000..04d3f98dc4 --- /dev/null +++ b/compass/landice/tests/ismip6_forcing/ocean/__init__.py @@ -0,0 +1,44 @@ +from compass.validate import compare_variables +from compass.testcase import TestCase +from compass.landice.tests.ismip6_forcing.ocean.process_basal_melt \ + import ProcessBasalMelt +from compass.landice.tests.ismip6_forcing.ocean.process_thermal_forcing \ + import ProcessThermalForcing + + +class Ocean(TestCase): + """ + A test case for processing the ISMIP6 ocean forcing data. + The test case builds a mapping file for interpolation between the + ISMIP6 8km polarstereo grid and MALI mesh, regrids the forcing data + and rename the ISMIP6 variables to corresponding MALI variables. + """ + + def __init__(self, test_group): + """ + Create the test case + + Parameters + ---------- + test_group : compass.landice.tests.ismip6_forcing.Ismip6Forcing + The test group that this test case belongs to + """ + name = 'ocean' + super().__init__(test_group=test_group, name=name) + + step = ProcessBasalMelt(test_case=self) + self.add_step(step) + + step = ProcessThermalForcing(test_case=self) + self.add_step(step) + + def configure(self): + """ + Configures test case + """ + input_path = self.config.get(section="ismip6_ais_ocean", + option="input_path") + if input_path == "NotAvailable": + raise ValueError("You need to supply a user config file, which " + "should contain the ismip6_ais_ocean section" + "with the input_path option") \ No newline at end of file diff --git a/compass/landice/tests/ismip6_forcing/ocean/create_mapfile.py b/compass/landice/tests/ismip6_forcing/ocean/create_mapfile.py new file mode 100644 index 0000000000..66f194e3e3 --- /dev/null +++ b/compass/landice/tests/ismip6_forcing/ocean/create_mapfile.py @@ -0,0 +1,84 @@ +import os +import subprocess +from mpas_tools.scrip.from_mpas import scrip_from_mpas + + +# function that creates a mapping file from ismip6 grid to mali mesh +def build_mapping_file(ismip6_grid_file, mapping_file, mali_mesh_file=None, + method_remap=None): + """ + Build a mapping file if it does not exist. + + Parameters + ---------- + ismip6_grid_file : str + ismip6 grid file + mapping_file : str + weights for interpolation from ismip6_grid_file to mali_mesh_file + mali_mesh_file : str, optional + The MALI mesh file if mapping file does not exist + method_remap : str, optional + Remapping method used in building a mapping file + """ + + if os.path.exists(mapping_file): + print("Mapping file exists. Not building a new one.") + return + + if mali_mesh_file is None: + raise ValueError("Mapping file does not exist. To build one, Mali " + "mesh file with '-f' should be provided. " + "Type --help for info") + + ismip6_scripfile = "temp_ismip6_8km_scrip.nc" + mali_scripfile = "temp_mali_scrip.nc" + ismip6_projection = "ais-bedmap2" + + # create the ismip6 scripfile if mapping file does not exist + # this is the projection of ismip6 data for Antarctica + print("Mapping file does not exist. Building one based on the " + "input/ouptut meshes") + print("Creating temporary scripfiles for ismip6 grid and mali mesh...") + + args = ["create_SCRIP_file_from_planar_rectangular_grid.py", + "--input", ismip6_grid_file, + "--scrip", ismip6_scripfile, + "--proj", ismip6_projection, + "--rank", "2"] + + subprocess.check_call(args) + + # make sure the input file uses the longitude convention of [0 2pi] + args = ["set_lat_lon_fields_in_planar_grid.py", + "--file", mali_mesh_file, + "--proj", ismip6_projection] + + subprocess.check_call(args) + + # create a MALI mesh scripfile if mapping file does not exist + scrip_from_mpas(mali_mesh_file, mali_scripfile) + + # create a mapping file using ESMF weight gen + print("Creating a mapping file...") + + if method_remap is None: + raise ValueError("Desired remapping option should be provided with " + "--method. Available options are 'bilinear'," + "'neareststod', 'conserve'.") + + args = ["ESMF_RegridWeightGen", + "-s", ismip6_scripfile, + "-d", mali_scripfile, + "-w", mapping_file, + "-m", method_remap, + "-i", "-64bit_offset", + "--dst_regional", "--src_regional"] + + + # include flag and input and output file names + subprocess.check_call(args) + + # remove the temporary scripfiles once the mapping file is generated + print("Removing the temporary scripfiles...") + os.remove(ismip6_scripfile) + os.remove(mali_scripfile) diff --git a/compass/landice/tests/ismip6_forcing/ocean/process_basal_melt.py b/compass/landice/tests/ismip6_forcing/ocean/process_basal_melt.py new file mode 100644 index 0000000000..36736288b2 --- /dev/null +++ b/compass/landice/tests/ismip6_forcing/ocean/process_basal_melt.py @@ -0,0 +1,185 @@ +import os +import subprocess +import xarray as xr +from compass.landice.tests.ismip6_forcing.ocean.create_mapfile import build_mapping_file +from mpas_tools.io import write_netcdf +from mpas_tools.logging import check_call +from compass.step import Step + + +class ProcessBasalMelt(Step): + """ + A step for processing (combine, remap and rename) the ISMIP6 basalmelt + data + """ + + def __init__(self, test_case): + """ + Create the step + + Parameters + ---------- + test_case : compass.landice.tests.ismip6_forcing.ocean.Ocean + The test case this step belongs to + """ + super().__init__(test_case=test_case, name='process_basal_melt') + + def setup(self): + """ + Set up this step of the test case + """ + config = self.config + section = config['ismip6_ais_ocean'] + + input_path = section.get('input_path') + basin_file = section.get('basin_file') + self.add_input_file(filename=basin_file, + target=os.path.join(input_path, + basin_file)) + coeff_file = section.get('coeff_file') + self.add_input_file(filename=coeff_file, + target=os.path.join(input_path, + coeff_file)) + mali_mesh_file = section.get('mali_mesh_file') + self.add_input_file(filename=mali_mesh_file, + target=os.path.join(input_path, mali_mesh_file)) + self.add_output_file(filename=f"output_basin_and_{coeff_file}") + + def run(self): + """ + Run this step of the test case + """ + # logger = self.logger + config = self.config + section = config['ismip6_ais_ocean'] + + basin_file = section.get('basin_file') + coeff_file = section.get('coeff_file') + mali_mesh_name = section.get('mali_mesh_name') + mali_mesh_file = section.get('mali_mesh_file') + method_remap = section.get('method_remap') + output_file = f"output_basin_and_{coeff_file}" + + # combine, interpolate and rename the basin file and deltaT0_gamma0 + # ismip6 input files + combined_file_temp = "combined.nc" # temporary file names + remapped_file_temp = "remapped.nc" + + # call the function that combines data + # logger.info = ('calling combine_ismip6_inputfiles') + self.combine_ismip6_inputfiles(basin_file, coeff_file, + combined_file_temp) + + # call the function that reads in, remap and rename the file. + print("Calling a remapping function...") + self.remap_ismip6BasalMelt_to_mali(combined_file_temp, + remapped_file_temp, mali_mesh_name, + mali_mesh_file, method_remap) + + # call the function that renames the ismip6 variables to MALI variables + print("Renaming the ismip6 variables to mali variable names...") + self.rename_ismip6BasalMelt_to_mali_vars(remapped_file_temp, + output_file) + + print("Remapping and renamping process done successfully. " + "Removing the temporary files 'combined.nc' and 'remapped.nc'") + + # remove the temporary combined file + os.remove(combined_file_temp) + os.remove(remapped_file_temp) + + def combine_ismip6_inputfiles(self, basin_file, coeff_gamma0_deltaT_file, + combined_file_temp): + """ + Combine ismip6 input files before regridding onto mali mesh + + Parameters + ---------- + basin_file : str + imbie2 basin numbers in ismip6 grid + coeff_gamma0_deltaT_file : str + uniform melt rate coefficient (gamma0) and temperature + correction per basin + combined_file_temp : str + temporary output file that has all the variables combined + """ + + ds_basin = xr.open_dataset(basin_file, engine="netcdf4") + ds = xr.open_dataset(coeff_gamma0_deltaT_file, engine="netcdf4") + + ds["ismip6shelfMelt_basin"] = ds_basin.basinNumber + write_netcdf(ds, combined_file_temp) + + def remap_ismip6BasalMelt_to_mali(self, input_file, output_file, + mali_mesh_name, mali_mesh_file, + method_remap): + """ + Remap the input ismip6 basal melt data onto mali mesh + + Parameters + ---------- + input_file: str + temporary output file that has all the variables combined + combined_file_temp generated in the above function + output_file : str + ismip6 data remapped on mali mesh + mali_mesh_name : str + name of the mali mesh used to name mapping files + mali_mesh_file : str, optional + The MALI mesh file if mapping file does not exist + method_remap : str, optional + Remapping method used in building a mapping file + """ + + # check if a mapfile + mapping_file = f"map_ismip6_8km_to_{mali_mesh_name}_{method_remap}.nc" + + if not os.path.exists(mapping_file): + # build a mapping file if it doesn't already exist + build_mapping_file(input_file, mapping_file, mali_mesh_file, + method_remap) + else: + print("Mapping file exists. Remapping the input data...") + + # remap the input data + args = ["ncremap", + "-i", input_file, + "-o", output_file, + "-m", mapping_file] + + subprocess.check_call(args) + + def rename_ismip6BasalMelt_to_mali_vars(self, remapped_file_temp, + output_file): + """ + Rename variables in the remapped ismip6 input data + to the ones that MALI uses. + + Parameters + ---------- + remapped_file_temp : str + temporary ismip6 data remapped on mali mesh + output_file : str + remapped ismip6 data renamed on mali mesh + """ + + # open dataset in 20 years chunk + ds = xr.open_dataset(remapped_file_temp, chunks=dict(time=20)) + + # build dictionary and rename the ismip6 dimension and variables + ismip6_to_mali_dims = dict( + ncol="nCells") + ds = ds.rename(ismip6_to_mali_dims) + + ismip6_to_mali_vars = dict( + deltaT_basin="ismip6shelfMelt_deltaT", + gamma0="ismip6shelfMelt_gamma0") + + # drop unnecessary variables on the regridded data on MALI mesh + ds = ds.rename(ismip6_to_mali_vars) + ds = ds.drop_vars(["lat_vertices", "lon_vertices", + "lat", "lon", "area"]) + + # write to a new netCDF file + write_netcdf(ds, output_file) + ds.close() diff --git a/compass/landice/tests/ismip6_forcing/ocean/process_thermal_forcing.py b/compass/landice/tests/ismip6_forcing/ocean/process_thermal_forcing.py new file mode 100644 index 0000000000..1050b1aca9 --- /dev/null +++ b/compass/landice/tests/ismip6_forcing/ocean/process_thermal_forcing.py @@ -0,0 +1,172 @@ +import os +import pandas as pd +import subprocess +import xarray as xr +from compass.landice.tests.ismip6_forcing.ocean.create_mapfile import build_mapping_file +from mpas_tools.io import write_netcdf +from mpas_tools.logging import check_call +from compass.step import Step + + +class ProcessThermalForcing(Step): + """ + A step for creating a mesh and initial condition for dome test cases + """ + def __init__(self, test_case): + """ + Create the step + + Parameters + ---------- + test_case : compass.landice.tests.ismip6_forcing.ocean.Ocean + The test case this step belongs to + """ + super().__init__(test_case=test_case, name='process_thermal_forcing') + + def setup(self): + """ + Set up this step of the test case + """ + config = self.config + section = config['ismip6_ais_ocean'] + + input_path = section.get('input_path') + thermal_forcing_file = section.get('thermal_forcing_file') + self.add_input_file(filename=thermal_forcing_file, + target=os.path.join(input_path, + thermal_forcing_file)) + mali_mesh_file = section.get('mali_mesh_file') + self.add_input_file(filename=mali_mesh_file, + target=os.path.join(input_path, mali_mesh_file)) + self.add_output_file(filename=f"output_{thermal_forcing_file}") + + def run(self): + """ + Run this step of the test case + """ + # logger = self.logger + config = self.config + section = config['ismip6_ais_ocean'] + + input_path = section.get('input_path') + thermal_forcing_file = section.get('thermal_forcing_file') + self.add_input_file(filename=thermal_forcing_file, + target=os.path.join(input_path, + thermal_forcing_file)) + mali_mesh_file = section.get('mali_mesh_file') + self.add_input_file(filename=mali_mesh_file, + target=os.path.join(input_path, mali_mesh_file)) + mali_mesh_name = section.get('mali_mesh_name') + mali_mesh_file = section.get('mali_mesh_file') + method_remap = section.get('method_remap') + output_file = f"output_{thermal_forcing_file}" + + # interpolate and rename the ismip6 thermal forcing data + remapped_file_temp = "remapped.nc" # temporary file name + + # call the function that reads in, remap and rename the file. + print("Calling a remapping function...") + self.remap_ismip6thermalforcing_to_mali(thermal_forcing_file, + remapped_file_temp, + mali_mesh_name, + mali_mesh_file, method_remap) + + # call the function that renames the ismip6 variables to MALI variables + print("Renaming the ismip6 variables to mali variable names...") + self.rename_ismip6thermalforcing_to_mali_vars(remapped_file_temp, + output_file) + + print("Remapping and renamping process done successfully. " + "Removing the temporary file 'remapped.nc'") + + # remove the temporary combined file + os.remove(remapped_file_temp) + + def remap_ismip6thermalforcing_to_mali(self, input_file, + output_file, mali_mesh_name, + mali_mesh_file, method_remap): + """ + Remap the input ismip6 thermal forcing data onto mali mesh + + Parameters + ---------- + input_file: str + ismip6 thermal forcing data on its native polarstereo 8km grid + output_file : str + ismip6 data remapped on mali mesh + mali_mesh_name : str + name of the mali mesh used to name mapping files + mali_mesh_file : str, optional + The MALI mesh file if mapping file does not exist + method_remap : str, optional + Remapping method used in building a mapping file + """ + + # check if a mapfile + mapping_file = f"map_ismip6_8km_to_{mali_mesh_name}_{method_remap}.nc" + + if not os.path.exists(mapping_file): + # build a mapping file if it doesn't already exist + build_mapping_file(input_file, mapping_file, mali_mesh_file, + method_remap) + else: + print("Mapping file exists. Remapping the input data...") + + # remap the input data + args = ["ncremap", + "-i", input_file, + "-o", output_file, + "-m", mapping_file] + + subprocess.check_call(args) + + def rename_ismip6thermalforcing_to_mali_vars(self, remapped_file_temp, + output_file): + """ + Rename variables in the remapped ismip6 input data + to the ones that MALI uses. + + Parameters + ---------- + remapped_file_temp : str + temporary ismip6 data remapped on mali mesh + output_file : str + remapped ismip6 data renamed on mali mesh + """ + + # open dataset in 20 years chunk + ds = xr.open_dataset(remapped_file_temp, chunks=dict(time=20), + engine="netcdf4") + + ds["ismip6shelfMelt_zOcean"] = ds.z + ds = ds.drop_vars('z') # dropping 'z' while it's still called 'z' + + # build dictionary for ismip6 variables that MALI takes in + ismip6_to_mali_dims = dict( + z="nISMIP6OceanLayers", + time="Time", + ncol="nCells") + ds = ds.rename(ismip6_to_mali_dims) + + ismip6_to_mali_vars = dict( + thermal_forcing="ismip6shelfMelt_3dThermalForcing") + ds = ds.rename(ismip6_to_mali_vars) + + # add xtime variable + xtime = [] + for t_index in range(ds.sizes["Time"]): + date = ds.Time[t_index].values + date = pd.to_datetime(str(date)) + date = date.strftime("%Y-%m-%d_00:00:00").ljust(64) + xtime.append(date) + + ds["xtime"] = ("Time", xtime) + ds["xtime"] = ds.xtime.astype('S') + + # drop unnecessary variables + ds = ds.drop_vars(["z_bnds", "lat_vertices", "Time", + "lon_vertices", "lat", "lon", "area"]) + + # write to a new netCDF file + write_netcdf(ds, output_file) + ds.close() From 5380c07a30aadd87ff971d565c85ed0645eacad4 Mon Sep 17 00:00:00 2001 From: hollyhan Date: Fri, 15 Apr 2022 15:27:21 -0600 Subject: [PATCH 05/58] Add a config file for the ismip6forcing test case --- .../landice/tests/ismip6_forcing/__init__.py | 2 +- .../ismip6_forcing/atmosphere/__init__.py | 2 +- .../atmosphere/create_mapfile_smb.py | 2 +- .../ismip6_forcing/atmosphere/process_smb.py | 5 +- .../landice/tests/ismip6_forcing/ismip6.cfg | 75 +++++++++++++++++++ .../tests/ismip6_forcing/ocean/__init__.py | 2 +- .../ismip6_forcing/ocean/create_mapfile.py | 1 - .../ocean/process_basal_melt.py | 3 +- .../ocean/process_thermal_forcing.py | 3 +- 9 files changed, 86 insertions(+), 9 deletions(-) create mode 100644 compass/landice/tests/ismip6_forcing/ismip6.cfg diff --git a/compass/landice/tests/ismip6_forcing/__init__.py b/compass/landice/tests/ismip6_forcing/__init__.py index e033e8fac8..38d2441d6f 100644 --- a/compass/landice/tests/ismip6_forcing/__init__.py +++ b/compass/landice/tests/ismip6_forcing/__init__.py @@ -14,6 +14,6 @@ def __init__(self, mpas_core): the MPAS core that this test group belongs to """ super().__init__(mpas_core=mpas_core, name='ismip6_forcing') - + self.add_test_case(Atmosphere(test_group=self)) self.add_test_case(Ocean(test_group=self)) diff --git a/compass/landice/tests/ismip6_forcing/atmosphere/__init__.py b/compass/landice/tests/ismip6_forcing/atmosphere/__init__.py index 432982f68f..04c67b6973 100644 --- a/compass/landice/tests/ismip6_forcing/atmosphere/__init__.py +++ b/compass/landice/tests/ismip6_forcing/atmosphere/__init__.py @@ -3,6 +3,7 @@ from compass.landice.tests.ismip6_forcing.atmosphere.process_smb \ import ProcessSMB + class Atmosphere(TestCase): """ A test case for processing the ISMIP6 atmosphere forcing data. @@ -42,4 +43,3 @@ def configure(self): raise ValueError("You need to supply a user config file, which " "should contain the ismip6_ais_atmosphere section" "with the input_path option") - diff --git a/compass/landice/tests/ismip6_forcing/atmosphere/create_mapfile_smb.py b/compass/landice/tests/ismip6_forcing/atmosphere/create_mapfile_smb.py index 354a0184d7..7fa03459c0 100644 --- a/compass/landice/tests/ismip6_forcing/atmosphere/create_mapfile_smb.py +++ b/compass/landice/tests/ismip6_forcing/atmosphere/create_mapfile_smb.py @@ -129,7 +129,7 @@ def create_atm_scrip(ismip6_grid_file, ismip6_scripfile): out_file.variables['grid_center_lat'][:] = ds.lat.values.flat out_file.variables['grid_center_lon'][:] = ds.lon.values.flat - out_file.variables['grid_dims'][:] = [nx, ny] # => check the scrip format docs + out_file.variables['grid_dims'][:] = [nx, ny] out_file.variables['grid_imask'][:] = 1 lat_corner = ds.lat_bnds diff --git a/compass/landice/tests/ismip6_forcing/atmosphere/process_smb.py b/compass/landice/tests/ismip6_forcing/atmosphere/process_smb.py index 086f959eb7..f2220348c6 100644 --- a/compass/landice/tests/ismip6_forcing/atmosphere/process_smb.py +++ b/compass/landice/tests/ismip6_forcing/atmosphere/process_smb.py @@ -4,7 +4,8 @@ import subprocess import xarray as xr import numpy as np -from compass.landice.tests.ismip6_forcing.atmosphere.create_mapfile_smb import build_mapping_file +from compass.landice.tests.ismip6_forcing.atmosphere.create_mapfile_smb \ + import build_mapping_file from mpas_tools.scrip.from_mpas import scrip_from_mpas from mpas_tools.io import write_netcdf from compass.step import Step @@ -40,7 +41,7 @@ def setup(self): mali_mesh_file = section.get('mali_mesh_file') self.add_input_file(filename=mali_mesh_file, target=os.path.join(input_path, mali_mesh_file)) - self.add_output_file(filename= f"output_{input_file}") + self.add_output_file(filename=f"output_{input_file}") def run(self): """ diff --git a/compass/landice/tests/ismip6_forcing/ismip6.cfg b/compass/landice/tests/ismip6_forcing/ismip6.cfg new file mode 100644 index 0000000000..e2bce833ac --- /dev/null +++ b/compass/landice/tests/ismip6_forcing/ismip6.cfg @@ -0,0 +1,75 @@ +# This file contains some common config options you might want to set +# if you're working with the compass landice core and MALI. + +# The paths section describes paths that are used within the landice core test +# cases. +[paths] + +# The root to a location where data files for MALI will be cached +landice_database_root = /Users/hollyhan/Desktop/RESEARCH/MALI/MALI_Database + + +# The parallel section describes options related to running tests in parallel +[parallel] + +# parallel system of execution: slurm or single_node +system = single_node + +# whether to use mpirun or srun to run the model +parallel_executable = mpirun -host localhost + +# cores per node on the machine +cores_per_node = 8 + +# the number of multiprocessing or dask threads to use +threads = 8 + +# config options for ismip6 antarctic ice sheet SMB forcing data test cases +[ismip6_ais_atmosphere] + +# Base path to the input ismip6 atmosphere data files. User has to supply. +input_path = /Users/hollyhan/Desktop/ISMIP6_tempfile/ + +# input surface mass balance file +input_file = NorESM1-M_8km_anomaly_1950-1994.nc + +# name of the mali mesh used to name mapping files +mali_mesh_name = Antarctica_8to80km + +# MALI mesh file to be used to build mapping file +mali_mesh_file = Antarctic_8to80km_20220407.nc + +# Remapping method used in building a mapping file. Options include: bilinear, neareststod, conserve +method_remap = bilinear + +# name of the processed ISMIP6 ocean data on MALI mesh +output_file = output_smb.nc + +# config options for ismip6 ocean data test cases +[ismip6_ais_ocean] + +# Base path to the input ismip6 ocean data files +input_path = /Users/hollyhan/Desktop/ISMIP6_tempfile/ + +# input thermal forcing file +thermal_forcing_file = NorESM1-M_thermal_forcing_8km_x_60m.nc + +# input imbie2 basin number file +basin_file = imbie2_basin_numbers_8km.nc + +# input uniform melt rate coefficient (gamma0) +# and temperature correction per basin file +coeff_file = coeff_gamma0_DeltaT_quadratic_non_local_median.nc + +# name of the mali mesh used to name mapping files +mali_mesh_name = Antarctica_8to80km + +# MALI mesh file to be used to build mapping file +mali_mesh_file = Antarctic_8to80km_20220407.nc + +# Remapping method used in building a mapping file. Options include: bilinear, neareststod, conserve +method_remap = bilinear + +# name of the processed ISMIP6 ocean data on MALI mesh +output_thermalforcing_file = output_thermalforcing.nc +output_basalmelt_file = output_basalmelt.nc \ No newline at end of file diff --git a/compass/landice/tests/ismip6_forcing/ocean/__init__.py b/compass/landice/tests/ismip6_forcing/ocean/__init__.py index 04d3f98dc4..c4a8ad275f 100644 --- a/compass/landice/tests/ismip6_forcing/ocean/__init__.py +++ b/compass/landice/tests/ismip6_forcing/ocean/__init__.py @@ -41,4 +41,4 @@ def configure(self): if input_path == "NotAvailable": raise ValueError("You need to supply a user config file, which " "should contain the ismip6_ais_ocean section" - "with the input_path option") \ No newline at end of file + "with the input_path option") diff --git a/compass/landice/tests/ismip6_forcing/ocean/create_mapfile.py b/compass/landice/tests/ismip6_forcing/ocean/create_mapfile.py index 66f194e3e3..05df40f32a 100644 --- a/compass/landice/tests/ismip6_forcing/ocean/create_mapfile.py +++ b/compass/landice/tests/ismip6_forcing/ocean/create_mapfile.py @@ -74,7 +74,6 @@ def build_mapping_file(ismip6_grid_file, mapping_file, mali_mesh_file=None, "-i", "-64bit_offset", "--dst_regional", "--src_regional"] - # include flag and input and output file names subprocess.check_call(args) diff --git a/compass/landice/tests/ismip6_forcing/ocean/process_basal_melt.py b/compass/landice/tests/ismip6_forcing/ocean/process_basal_melt.py index 36736288b2..28b4196a53 100644 --- a/compass/landice/tests/ismip6_forcing/ocean/process_basal_melt.py +++ b/compass/landice/tests/ismip6_forcing/ocean/process_basal_melt.py @@ -1,7 +1,8 @@ import os import subprocess import xarray as xr -from compass.landice.tests.ismip6_forcing.ocean.create_mapfile import build_mapping_file +from compass.landice.tests.ismip6_forcing.ocean.create_mapfile \ + import build_mapping_file from mpas_tools.io import write_netcdf from mpas_tools.logging import check_call from compass.step import Step diff --git a/compass/landice/tests/ismip6_forcing/ocean/process_thermal_forcing.py b/compass/landice/tests/ismip6_forcing/ocean/process_thermal_forcing.py index 1050b1aca9..17c74a48c8 100644 --- a/compass/landice/tests/ismip6_forcing/ocean/process_thermal_forcing.py +++ b/compass/landice/tests/ismip6_forcing/ocean/process_thermal_forcing.py @@ -2,7 +2,8 @@ import pandas as pd import subprocess import xarray as xr -from compass.landice.tests.ismip6_forcing.ocean.create_mapfile import build_mapping_file +from compass.landice.tests.ismip6_forcing.ocean.create_mapfile \ + import build_mapping_file from mpas_tools.io import write_netcdf from mpas_tools.logging import check_call from compass.step import Step From 6d2403dc8e0e7e63f84972f81a4eb875002dd7cf Mon Sep 17 00:00:00 2001 From: hollyhan Date: Thu, 8 Sep 2022 14:29:45 -0600 Subject: [PATCH 06/58] Separate the ocean test case into two: ocean_basal and ocean_thermal --- .../landice/tests/ismip6_forcing/__init__.py | 7 +- .../tests/ismip6_forcing/ocean/__init__.py | 44 -------- .../ismip6_forcing/ocean_basal/__init__.py | 86 +++++++++++++++ .../{ocean => ocean_basal}/create_mapfile.py | 0 .../process_basal_melt.py | 31 +++--- .../ismip6_forcing/ocean_thermal/__init__.py | 101 ++++++++++++++++++ .../ocean_thermal/create_mapfile.py | 83 ++++++++++++++ .../process_thermal_forcing.py | 7 +- 8 files changed, 294 insertions(+), 65 deletions(-) delete mode 100644 compass/landice/tests/ismip6_forcing/ocean/__init__.py create mode 100644 compass/landice/tests/ismip6_forcing/ocean_basal/__init__.py rename compass/landice/tests/ismip6_forcing/{ocean => ocean_basal}/create_mapfile.py (100%) rename compass/landice/tests/ismip6_forcing/{ocean => ocean_basal}/process_basal_melt.py (91%) create mode 100644 compass/landice/tests/ismip6_forcing/ocean_thermal/__init__.py create mode 100644 compass/landice/tests/ismip6_forcing/ocean_thermal/create_mapfile.py rename compass/landice/tests/ismip6_forcing/{ocean => ocean_thermal}/process_thermal_forcing.py (96%) diff --git a/compass/landice/tests/ismip6_forcing/__init__.py b/compass/landice/tests/ismip6_forcing/__init__.py index 38d2441d6f..fab34d27a0 100644 --- a/compass/landice/tests/ismip6_forcing/__init__.py +++ b/compass/landice/tests/ismip6_forcing/__init__.py @@ -1,6 +1,8 @@ from compass.testgroup import TestGroup from compass.landice.tests.ismip6_forcing.atmosphere import Atmosphere -from compass.landice.tests.ismip6_forcing.ocean import Ocean +from compass.landice.tests.ismip6_forcing.ocean_thermal import OceanThermal +from compass.landice.tests.ismip6_forcing.ocean_basal import OceanBasal + class Ismip6Forcing(TestGroup): @@ -16,4 +18,5 @@ def __init__(self, mpas_core): super().__init__(mpas_core=mpas_core, name='ismip6_forcing') self.add_test_case(Atmosphere(test_group=self)) - self.add_test_case(Ocean(test_group=self)) + self.add_test_case(OceanBasal(test_group=self)) + self.add_test_case(OceanThermal(test_group=self)) diff --git a/compass/landice/tests/ismip6_forcing/ocean/__init__.py b/compass/landice/tests/ismip6_forcing/ocean/__init__.py deleted file mode 100644 index c4a8ad275f..0000000000 --- a/compass/landice/tests/ismip6_forcing/ocean/__init__.py +++ /dev/null @@ -1,44 +0,0 @@ -from compass.validate import compare_variables -from compass.testcase import TestCase -from compass.landice.tests.ismip6_forcing.ocean.process_basal_melt \ - import ProcessBasalMelt -from compass.landice.tests.ismip6_forcing.ocean.process_thermal_forcing \ - import ProcessThermalForcing - - -class Ocean(TestCase): - """ - A test case for processing the ISMIP6 ocean forcing data. - The test case builds a mapping file for interpolation between the - ISMIP6 8km polarstereo grid and MALI mesh, regrids the forcing data - and rename the ISMIP6 variables to corresponding MALI variables. - """ - - def __init__(self, test_group): - """ - Create the test case - - Parameters - ---------- - test_group : compass.landice.tests.ismip6_forcing.Ismip6Forcing - The test group that this test case belongs to - """ - name = 'ocean' - super().__init__(test_group=test_group, name=name) - - step = ProcessBasalMelt(test_case=self) - self.add_step(step) - - step = ProcessThermalForcing(test_case=self) - self.add_step(step) - - def configure(self): - """ - Configures test case - """ - input_path = self.config.get(section="ismip6_ais_ocean", - option="input_path") - if input_path == "NotAvailable": - raise ValueError("You need to supply a user config file, which " - "should contain the ismip6_ais_ocean section" - "with the input_path option") diff --git a/compass/landice/tests/ismip6_forcing/ocean_basal/__init__.py b/compass/landice/tests/ismip6_forcing/ocean_basal/__init__.py new file mode 100644 index 0000000000..987849d158 --- /dev/null +++ b/compass/landice/tests/ismip6_forcing/ocean_basal/__init__.py @@ -0,0 +1,86 @@ +from compass.testcase import TestCase +from compass.landice.tests.ismip6_forcing.ocean_basal.process_basal_melt \ + import ProcessBasalMelt + + +class OceanBasal(TestCase): + """ + A test case for processing the ISMIP6 ocean basalmelt param. coeff. data. + The test case builds a mapping file for interpolation between the + ISMIP6 8km polarstereo grid and MALI mesh, regrids the forcing data + and renames the ISMIP6 variables to corresponding MALI variables. + """ + + def __init__(self, test_group): + """ + Create the test case + + Parameters + ---------- + test_group : compass.landice.tests.ismip6_forcing.Ismip6Forcing + The test group that this test case belongs to + """ + name = 'ocean_basal' + super().__init__(test_group=test_group, name=name) + + step = ProcessBasalMelt(test_case=self) + self.add_step(step) + + def configure(self): + """ + Configures test case + """ + base_path = self.config.get(section="ismip6_ais", + option="base_path") + model = self.config.get(section="ismip6_ais", + option="model") + scenario = self.config.get(section="ismip6_ais", + option="scenario") + period_endyear = self.config.get(section="ismip6_ais", + option="period_endyear") + mali_mesh_name = self.config.get(section="ismip6_ais", + option="mali_mesh_name") + mali_mesh_file = self.config.get(section="ismip6_ais", + option="mali_mesh_file") + + if base_path == "NotAvailable": + raise ValueError("You need to supply a user config file, which " + "should contain the ismip6_ais " + "section with the base_path option") + if model == "NotAvailable": + raise ValueError("You need to supply a user config file, which " + "should contain the ismip6_ais " + "section with the model option") + if scenario == "NotAvailable": + raise ValueError("You need to supply a user config file, which " + "should contain the ismip6_ais " + "section with the scenario option") + if period_endyear == "NotAvailable": + raise ValueError("You need to supply a user config file, which " + "should contain the ismip6_ais " + "section with the period_endyear option") + if mali_mesh_name == "NotAvailable": + raise ValueError("You need to supply a user config file, which " + "should contain the ismip6_ais " + "section with the mali_mesh_name option") + if mali_mesh_file == "NotAvailable": + raise ValueError("You need to supply a user config file, which " + "should contain the ismip6_ais " + "section with the mali_mesh_file option") + + # input files: input uniform melt rate coefficient (gamma0) + # and temperature correction per basin + _files = { + "2100":["coeff_gamma0_DeltaT_quadratic_local_5th_pct_PIGL_gamma_calibration.nc", + "coeff_gamma0_DeltaT_quadratic_local_5th_percentile.nc", + "coeff_gamma0_DeltaT_quadratic_local_95th_pct_PIGL_gamma_calibration.nc", + "coeff_gamma0_DeltaT_quadratic_local_95th_percentile.nc", + "coeff_gamma0_DeltaT_quadratic_local_median_PIGL_gamma_calibration.nc", + "coeff_gamma0_DeltaT_quadratic_local_median.nc", + "coeff_gamma0_DeltaT_quadratic_non_local_5th_pct_PIGL_gamma_calibration.nc", + "coeff_gamma0_DeltaT_quadratic_non_local_5th_percentile.nc", + "coeff_gamma0_DeltaT_quadratic_non_local_95th_pct_PIGL_gamma_calibration.nc", + "coeff_gamma0_DeltaT_quadratic_non_local_95th_percentile.nc", + "coeff_gamma0_DeltaT_quadratic_non_local_median_PIGL_gamma_calibration.nc", + "coeff_gamma0_DeltaT_quadratic_non_local_median.nc"] + } \ No newline at end of file diff --git a/compass/landice/tests/ismip6_forcing/ocean/create_mapfile.py b/compass/landice/tests/ismip6_forcing/ocean_basal/create_mapfile.py similarity index 100% rename from compass/landice/tests/ismip6_forcing/ocean/create_mapfile.py rename to compass/landice/tests/ismip6_forcing/ocean_basal/create_mapfile.py diff --git a/compass/landice/tests/ismip6_forcing/ocean/process_basal_melt.py b/compass/landice/tests/ismip6_forcing/ocean_basal/process_basal_melt.py similarity index 91% rename from compass/landice/tests/ismip6_forcing/ocean/process_basal_melt.py rename to compass/landice/tests/ismip6_forcing/ocean_basal/process_basal_melt.py index 28b4196a53..1f8a62736b 100644 --- a/compass/landice/tests/ismip6_forcing/ocean/process_basal_melt.py +++ b/compass/landice/tests/ismip6_forcing/ocean_basal/process_basal_melt.py @@ -1,10 +1,9 @@ import os import subprocess import xarray as xr -from compass.landice.tests.ismip6_forcing.ocean.create_mapfile \ +from compass.landice.tests.ismip6_forcing.ocean_basal.create_mapfile \ import build_mapping_file from mpas_tools.io import write_netcdf -from mpas_tools.logging import check_call from compass.step import Step @@ -20,7 +19,7 @@ def __init__(self, test_case): Parameters ---------- - test_case : compass.landice.tests.ismip6_forcing.ocean.Ocean + test_case : compass.landice.tests.ismip6_forcing.ocean_basal.OceanBasal The test case this step belongs to """ super().__init__(test_case=test_case, name='process_basal_melt') @@ -30,20 +29,20 @@ def setup(self): Set up this step of the test case """ config = self.config - section = config['ismip6_ais_ocean'] - - input_path = section.get('input_path') + section = config['ismip6_ais'] + base_path = section.get('base_path') + mali_mesh_file = section.get('mali_mesh_file') + section = config['ismip6_ais_ocean_basal'] basin_file = section.get('basin_file') + coeff_file = section.get('coeff_file') self.add_input_file(filename=basin_file, - target=os.path.join(input_path, + target=os.path.join(base_path, basin_file)) - coeff_file = section.get('coeff_file') self.add_input_file(filename=coeff_file, - target=os.path.join(input_path, + target=os.path.join(base_path, coeff_file)) - mali_mesh_file = section.get('mali_mesh_file') self.add_input_file(filename=mali_mesh_file, - target=os.path.join(input_path, mali_mesh_file)) + target=os.path.join(base_path, mali_mesh_file)) self.add_output_file(filename=f"output_basin_and_{coeff_file}") def run(self): @@ -52,12 +51,12 @@ def run(self): """ # logger = self.logger config = self.config - section = config['ismip6_ais_ocean'] - - basin_file = section.get('basin_file') - coeff_file = section.get('coeff_file') + section = config['ismip6_ais'] mali_mesh_name = section.get('mali_mesh_name') mali_mesh_file = section.get('mali_mesh_file') + section = config['ismip6_ais_ocean_basal'] + basin_file = section.get('basin_file') + coeff_file = section.get('coeff_file') method_remap = section.get('method_remap') output_file = f"output_basin_and_{coeff_file}" @@ -132,7 +131,7 @@ def remap_ismip6BasalMelt_to_mali(self, input_file, output_file, Remapping method used in building a mapping file """ - # check if a mapfile + # check if mapfile exists mapping_file = f"map_ismip6_8km_to_{mali_mesh_name}_{method_remap}.nc" if not os.path.exists(mapping_file): diff --git a/compass/landice/tests/ismip6_forcing/ocean_thermal/__init__.py b/compass/landice/tests/ismip6_forcing/ocean_thermal/__init__.py new file mode 100644 index 0000000000..e7a4528375 --- /dev/null +++ b/compass/landice/tests/ismip6_forcing/ocean_thermal/__init__.py @@ -0,0 +1,101 @@ +from compass.testcase import TestCase +from compass.landice.tests.ismip6_forcing.ocean_thermal.process_thermal_forcing\ + import ProcessThermalForcing + + +class OceanThermal(TestCase): + """ + A test case for processing the ISMIP6 ocean thermal forcing data. + The test case builds a mapping file for interpolation between the + ISMIP6 8km polarstereo grid and MALI mesh, regrids the forcing data + and renames the ISMIP6 variables to corresponding MALI variables. + """ + + def __init__(self, test_group): + """ + Create the test case + + Parameters + ---------- + test_group : compass.landice.tests.ismip6_forcing.Ismip6Forcing + The test group that this test case belongs to + """ + name = 'ocean_thermal' + super().__init__(test_group=test_group, name=name) + + step = ProcessThermalForcing(test_case=self) + self.add_step(step) + + def configure(self): + """ + Configures test case + """ + base_path = self.config.get(section="ismip6_ais", + option="base_path") + model = self.config.get(section="ismip6_ais", + option="model") + scenario = self.config.get(section="ismip6_ais", + option="scenario") + period_endyear = self.config.get(section="ismip6_ais", + option="period_endyear") + mali_mesh_name = self.config.get(section="ismip6_ais", + option="mali_mesh_name") + mali_mesh_file = self.config.get(section="ismip6_ais", + option="mali_mesh_file") + + if base_path == "NotAvailable": + raise ValueError("You need to supply a user config file, which " + "should contain the ismip6_ais " + "section with the base_path option") + if model == "NotAvailable": + raise ValueError("You need to supply a user config file, which " + "should contain the ismip6_ais " + "section with the model option") + if scenario == "NotAvailable": + raise ValueError("You need to supply a user config file, which " + "should contain the ismip6_ais " + "section with the scenario option") + if period_endyear == "NotAvailable": + raise ValueError("You need to supply a user config file, which " + "should contain the ismip6_ais " + "section with the period_endyear option") + if mali_mesh_name == "NotAvailable": + raise ValueError("You need to supply a user config file, which " + "should contain the ismip6_ais " + "section with the mali_mesh_name option") + if mali_mesh_file == "NotAvailable": + raise ValueError("You need to supply a user config file, which " + "should contain the ismip6_ais " + "section with the mali_mesh_file option") + + _files = { + "2100": { + "HadGEM2-ES":{ + "RCP85": ["file1", "file2"], + "RCP26": ["file1", "file2"] + }, + "CESM2-WACCM":{} + }, + "2300": { + "CCSM4":{ + "RCP85":["AIS/Ocean_forcing/ccsm4_RCP85/1995-2300/CCSM4_RCP85_thermal_forcing_8km_x_60m.nc"] + }, + "CESM2-WACCM":{ + "SSP585":["AIS/Ocean_forcing/CESM2-WACCM_ssp585/1995-2299/CESM2-WACCM_SSP585_thermal_forcing_8km_x_60m.nc"] #repeat: "1AIS/Ocean_forcing/cesm2-waccm_ssp585-repeat/1995-2300/CESM2-WACCM_SSP585_thermal_forcing_8km_x_60m.nc" + }, + # "CSIRO-Mk3-6-0" does not exist for ocean forcing + "HadGEM2-ES":{ + "RCP85":["AIS/Ocean_forcing/hadgem2-es_RCP85/1995-2299/HadGEM2-ES_RCP85_thermal_forcing_8km_x_60m.nc", + "AIS/Ocean_forcing/hadgem2-es_RCP85-repeat/1995-2299/HadGEM2-ES_RCP85_thermal_forcing_8km_x_60m.nc"] + }, + "NorESM1-M":{ + "RCP26":["AIS/Ocean_forcing/noresm1-m_RCP26-repeat/1995-2300/NorESM1-M_RCP26_thermal_forcing_8km_x_60m.nc"], #this is repeat forcing + "RCP85":["/AIS/Ocean_forcing/noresm1-m_RCP85-repeat/1995-2300/NorESM1-M_thermal_forcing_8km_x_60m.nc"] + }, + "UKESM1-0-LL":{ + "SSP126":["AIS/Ocean_forcing/ukesm1-0-ll_ssp126/1995-2300/UKESM1-0-LL_thermal_forcing_8km_x_60m.nc"], + "SSP585":["AIS/Ocean_forcing/ukesm1-0-ll_ssp585/1995-2300/UKESM1-0-LL_SSP585_thermal_forcing_8km_x_60m.nc", + "AIS/Ocean_forcing/ukesm1-0-ll_ssp585-repeat/1995-2300/UKESM1-0-LL_SSP585_thermal_forcing_8km_x_60m.nc"] #this is repeat forcing again + } + } + } \ No newline at end of file diff --git a/compass/landice/tests/ismip6_forcing/ocean_thermal/create_mapfile.py b/compass/landice/tests/ismip6_forcing/ocean_thermal/create_mapfile.py new file mode 100644 index 0000000000..05df40f32a --- /dev/null +++ b/compass/landice/tests/ismip6_forcing/ocean_thermal/create_mapfile.py @@ -0,0 +1,83 @@ +import os +import subprocess +from mpas_tools.scrip.from_mpas import scrip_from_mpas + + +# function that creates a mapping file from ismip6 grid to mali mesh +def build_mapping_file(ismip6_grid_file, mapping_file, mali_mesh_file=None, + method_remap=None): + """ + Build a mapping file if it does not exist. + + Parameters + ---------- + ismip6_grid_file : str + ismip6 grid file + mapping_file : str + weights for interpolation from ismip6_grid_file to mali_mesh_file + mali_mesh_file : str, optional + The MALI mesh file if mapping file does not exist + method_remap : str, optional + Remapping method used in building a mapping file + """ + + if os.path.exists(mapping_file): + print("Mapping file exists. Not building a new one.") + return + + if mali_mesh_file is None: + raise ValueError("Mapping file does not exist. To build one, Mali " + "mesh file with '-f' should be provided. " + "Type --help for info") + + ismip6_scripfile = "temp_ismip6_8km_scrip.nc" + mali_scripfile = "temp_mali_scrip.nc" + ismip6_projection = "ais-bedmap2" + + # create the ismip6 scripfile if mapping file does not exist + # this is the projection of ismip6 data for Antarctica + print("Mapping file does not exist. Building one based on the " + "input/ouptut meshes") + print("Creating temporary scripfiles for ismip6 grid and mali mesh...") + + args = ["create_SCRIP_file_from_planar_rectangular_grid.py", + "--input", ismip6_grid_file, + "--scrip", ismip6_scripfile, + "--proj", ismip6_projection, + "--rank", "2"] + + subprocess.check_call(args) + + # make sure the input file uses the longitude convention of [0 2pi] + args = ["set_lat_lon_fields_in_planar_grid.py", + "--file", mali_mesh_file, + "--proj", ismip6_projection] + + subprocess.check_call(args) + + # create a MALI mesh scripfile if mapping file does not exist + scrip_from_mpas(mali_mesh_file, mali_scripfile) + + # create a mapping file using ESMF weight gen + print("Creating a mapping file...") + + if method_remap is None: + raise ValueError("Desired remapping option should be provided with " + "--method. Available options are 'bilinear'," + "'neareststod', 'conserve'.") + + args = ["ESMF_RegridWeightGen", + "-s", ismip6_scripfile, + "-d", mali_scripfile, + "-w", mapping_file, + "-m", method_remap, + "-i", "-64bit_offset", + "--dst_regional", "--src_regional"] + + # include flag and input and output file names + subprocess.check_call(args) + + # remove the temporary scripfiles once the mapping file is generated + print("Removing the temporary scripfiles...") + os.remove(ismip6_scripfile) + os.remove(mali_scripfile) diff --git a/compass/landice/tests/ismip6_forcing/ocean/process_thermal_forcing.py b/compass/landice/tests/ismip6_forcing/ocean_thermal/process_thermal_forcing.py similarity index 96% rename from compass/landice/tests/ismip6_forcing/ocean/process_thermal_forcing.py rename to compass/landice/tests/ismip6_forcing/ocean_thermal/process_thermal_forcing.py index 17c74a48c8..c96e316742 100644 --- a/compass/landice/tests/ismip6_forcing/ocean/process_thermal_forcing.py +++ b/compass/landice/tests/ismip6_forcing/ocean_thermal/process_thermal_forcing.py @@ -2,7 +2,7 @@ import pandas as pd import subprocess import xarray as xr -from compass.landice.tests.ismip6_forcing.ocean.create_mapfile \ +from compass.landice.tests.ismip6_forcing.ocean_thermal.create_mapfile \ import build_mapping_file from mpas_tools.io import write_netcdf from mpas_tools.logging import check_call @@ -19,7 +19,8 @@ def __init__(self, test_case): Parameters ---------- - test_case : compass.landice.tests.ismip6_forcing.ocean.Ocean + test_case : compass.landice.tests.ismip6_forcing.ocean_thermal. + OceanThermal The test case this step belongs to """ super().__init__(test_case=test_case, name='process_thermal_forcing') @@ -45,7 +46,7 @@ def run(self): """ Run this step of the test case """ - # logger = self.logger + #logger = self.logger config = self.config section = config['ismip6_ais_ocean'] From 02d110bdb2bb4e2a78ccf800994f81b611a3d524 Mon Sep 17 00:00:00 2001 From: hollyhan Date: Tue, 3 May 2022 22:05:59 -0600 Subject: [PATCH 07/58] Make major edits to the ismip6_forcing/atmosphere testcase 1. add/get different config options for input and output files and their paths 2. add a nested dictionary for loading different isimip6 forcing files 3. add a method that applies the MALI base SMB to the ismip6 smb anomaly field 4. fix how to add xtime variable; use `xarray.DataArray.dt` instead of `pandas.to_datetime` --- .../ismip6_forcing/atmosphere/__init__.py | 30 ++- .../ismip6_forcing/atmosphere/process_smb.py | 214 ++++++++++++++++-- 2 files changed, 215 insertions(+), 29 deletions(-) diff --git a/compass/landice/tests/ismip6_forcing/atmosphere/__init__.py b/compass/landice/tests/ismip6_forcing/atmosphere/__init__.py index 04c67b6973..c1af8f0d9d 100644 --- a/compass/landice/tests/ismip6_forcing/atmosphere/__init__.py +++ b/compass/landice/tests/ismip6_forcing/atmosphere/__init__.py @@ -1,4 +1,3 @@ -from compass.validate import compare_variables from compass.testcase import TestCase from compass.landice.tests.ismip6_forcing.atmosphere.process_smb \ import ProcessSMB @@ -37,9 +36,28 @@ def configure(self): """ Configures test case """ - input_path = self.config.get(section="ismip6_ais_atmosphere", - option="input_path") - if input_path == "NotAvailable": + base_path_ismip6 = self.config.get(section="ismip6_ais", + option="base_path_ismip6") + base_path_mali = self.config.get(section="ismip6_ais", + option="base_path_mali") + mali_mesh_name = self.config.get(section="ismip6_ais", + option="mali_mesh_name") + mali_mesh_file = self.config.get(section="ismip6_ais", + option="mali_mesh_file") + + if base_path_ismip6 == "NotAvailable": + raise ValueError("You need to supply a user config file, which " + "should contain the ismip6_ais " + "section with the base_path_ismip6 option") + if base_path_mali == "NotAvailable": + raise ValueError("You need to supply a user config file, which " + "should contain the ismip6_ais " + "section with the base_path_mali option") + if mali_mesh_name == "NotAvailable": + raise ValueError("You need to supply a user config file, which " + "should contain the ismip6_ais " + "section with the mali_mesh_name option") + if mali_mesh_file == "NotAvailable": raise ValueError("You need to supply a user config file, which " - "should contain the ismip6_ais_atmosphere section" - "with the input_path option") + "should contain the ismip6_ais " + "section with the mali_mesh_file option") diff --git a/compass/landice/tests/ismip6_forcing/atmosphere/process_smb.py b/compass/landice/tests/ismip6_forcing/atmosphere/process_smb.py index f2220348c6..170e1917dd 100644 --- a/compass/landice/tests/ismip6_forcing/atmosphere/process_smb.py +++ b/compass/landice/tests/ismip6_forcing/atmosphere/process_smb.py @@ -1,12 +1,8 @@ import os -import pandas as pd -import shutil import subprocess import xarray as xr -import numpy as np from compass.landice.tests.ismip6_forcing.atmosphere.create_mapfile_smb \ import build_mapping_file -from mpas_tools.scrip.from_mpas import scrip_from_mpas from mpas_tools.io import write_netcdf from compass.step import Step @@ -16,7 +12,7 @@ class ProcessSMB(Step): A step for processing the ISMIP6 surface mass balance data """ - def __init__(self, test_case): + def __init__(self, test_case, input_file=None): """ Create the step @@ -24,7 +20,10 @@ def __init__(self, test_case): ---------- test_case : compass.landice.tests.ismip6_forcing.atmosphere.Atmosphere The test case this step belongs to + + input_file : file name of ismip6 forcing data processed by this step """ + self.input_file = input_file super().__init__(test_case=test_case, name='process_smb') def setup(self): @@ -32,16 +31,43 @@ def setup(self): Set up this step of the test case """ config = self.config - section = config['ismip6_ais_atmosphere'] + section = config['ismip6_ais'] - input_path = section.get('input_path') - input_file = section.get('input_file') - self.add_input_file(filename=input_file, - target=os.path.join(input_path, input_file)) + base_path_ismip6 = section.get('base_path_ismip6') + base_path_mali = section.get('base_path_mali') mali_mesh_file = section.get('mali_mesh_file') + self.add_input_file(filename=mali_mesh_file, - target=os.path.join(input_path, mali_mesh_file)) - self.add_output_file(filename=f"output_{input_file}") + target=os.path.join(base_path_mali, + mali_mesh_file)) + + section = config['ismip6_ais'] + period_endyear = section.get("period_endyear") + model = section.get("model") + scenario = section.get("scenario") + + if period_endyear == "NotAvailable": + raise ValueError("You need to supply a user config file, which " + "should contain the ismip6_ais " + "section with the period_endyear option") + if model == "NotAvailable": + raise ValueError("You need to supply a user config file, which " + "should contain the ismip6_ais " + "section with the model option") + if scenario == "NotAvailable": + raise ValueError("You need to supply a user config file, which " + "should contain the ismip6_ais " + "section with the scenario option") + + input_file_list = self._files[period_endyear][model][scenario] + for file in input_file_list: + print(base_path_ismip6) + print(os.path.join(base_path_ismip6, file)) + self.add_input_file(filename=os.path.basename(file), + target=os.path.join(base_path_ismip6, file)) + + output_file = f"processed_{model}_{scenario}_{period_endyear}.nc" + self.add_output_file(filename=output_file) def run(self): """ @@ -49,32 +75,55 @@ def run(self): """ logger = self.logger config = self.config - section = config['ismip6_ais_atmosphere'] - input_file = section.get('input_file') + section = config['ismip6_ais'] + # base_path_ismip6 = section.get('base_path_ismip6') mali_mesh_name = section.get('mali_mesh_name') mali_mesh_file = section.get('mali_mesh_file') + period_endyear = section.get("period_endyear") + model = section.get("model") + scenario = section.get("scenario") + + section = config['ismip6_ais_atmosphere'] method_remap = section.get('method_remap') - output_file = f"output_{input_file}" + + input_file_list = self._files[period_endyear][model][scenario] + i = 0 + for file in input_file_list: + input_file_list[i] = os.path.basename(file) + i += 1 + + input_file_combined = xr.open_mfdataset(input_file_list, + concat_dim='time', + combine='nested', + engine='netcdf4') + combined_file_temp = "combined.nc" + write_netcdf(input_file_combined, combined_file_temp) # interpolate and rename the ismip6 thermal forcing data remapped_file_temp = "remapped.nc" # temporary file name # call the function that reads in, remap and rename the file. logger.info("Calling a remapping function...") - self.remap_ismip6smb_to_mali(input_file, remapped_file_temp, + self.remap_ismip6smb_to_mali(combined_file_temp, remapped_file_temp, mali_mesh_name, mali_mesh_file, method_remap) # call the function that renames the ismip6 variables to MALI variables logger.info("Renaming the ismip6 variables to mali variable names...") + output_file = f"processed_{model}_{scenario}_{period_endyear}.nc" self.rename_ismip6smb_to_mali_vars(remapped_file_temp, output_file) - logger.info("Remapping and renamping process done successfully. " - "Removing the temporary file 'remapped.nc'") + # correct the SMB anomaly field with mali base SMB field + logger.info("Correcting the SMB anomaly field with MALI base SMB...") + self.correct_SMB_anomaly_for_baseSMB(output_file, mali_mesh_file) - # remove the temporary combined file + logger.info("Processing done successfully. " + "Removing the temporary files 'remapped.nc' and " + "'combined.nc'...") + # remove the temporary remapped and combined files os.remove(remapped_file_temp) + os.remove(combined_file_temp) def remap_ismip6smb_to_mali(self, input_file, output_file, mali_mesh_name, mali_mesh_file, method_remap): @@ -95,7 +144,7 @@ def remap_ismip6smb_to_mali(self, input_file, output_file, mali_mesh_name, Remapping method used in building a mapping file """ - # check if a mapfile + # check if a mapfile exists mapping_file = f"map_ismip6_8km_to_{mali_mesh_name}_{method_remap}.nc" if not os.path.exists(mapping_file): @@ -145,9 +194,9 @@ def rename_ismip6smb_to_mali_vars(self, remapped_file_temp, output_file): # add xtime variable xtime = [] for t_index in range(ds.sizes["Time"]): - date = ds.Time[t_index].values - date = pd.to_datetime(str(date)) - date = date.strftime("%Y-%m-%d_00:00:00").ljust(64) + date = ds.Time[t_index] + date = date.dt.strftime("%Y-%m-%d_00:00:00") + date = str(date.values).ljust(64) xtime.append(date) ds["xtime"] = ("Time", xtime) @@ -160,3 +209,122 @@ def rename_ismip6smb_to_mali_vars(self, remapped_file_temp, output_file): # write to a new netCDF file write_netcdf(ds, output_file) ds.close() + + def correct_SMB_anomaly_for_baseSMB(self, output_file, mali_mesh_file): + """ + Apply the MALI base SMB to the ismip6 SMB anomaly field + + Parameters + ---------- + output_file : str + remapped ismip6 data renamed on mali mesh + mali_mesh_file : str + initialized MALI mesh file in which the base SMB field exists + """ + + ds = xr.open_dataset(output_file) + ds_base = xr.open_dataset(mali_mesh_file) + for i in range(len(ds["sfcMassBal"])): + ds["sfcMassBal"][i] = ds["sfcMassBal"][i] \ + + ds_base["sfcMassBal"][0,:] + + # write to a new netCDF file + write_netcdf(ds, output_file) + ds.close() + + # create a nested dictionary for the ISMIP6 original forcing files including relative path + _files = { + "2100": { + "CCSM4": { + "RCP26": ["AIS/Atmosphere_forcing/ccsm4_rcp2.6/Regridded_8km/CCSM4_8km_anomaly_rcp26_1995-2100.nc"], + "RCP85": ["AIS/Atmosphere_Forcing/ccsm4_rcp8.5/Regridded_8km/CCSM4_8km_anomaly_1995-2100.nc"] + }, + "CESM2": { + "SSP585v1": [ + "AIS/Atmosphere_Forcing/CESM2_ssp585/Regridded_8km/CESM2_anomaly_ssp585_1995-2100_8km_v1.nc"], + "SSP585v2": [ + "AIS/Atmosphere_forcing/CESM2_ssp585/Regridded_8km/CESM2_anomaly_ssp585_1995-2100_8km_v2.nc"] + }, + "CNRM_CM6": { + "SSP126": [ + "AIS/Atmosphere_forcing/CNRM_CM6_ssp126/Regridded_8km/CNRM-CM6-1_anomaly_ssp126_1995-2100_8km_ISMIP6.nc"], + "SSP585": [ + "AIS/Atmosphere_forcing/CNRM_CM6_ssp585/Regridded_8km/CNRM-CM6-1_anomaly_ssp585_1995-2100_8km_ISMIP6.nc"] + }, + "CNRM_ESM2": { + "SSP585": [ + "AIS/Atmosphere_forcing/CNRM_ESM2_ssp585/Regridded_8km/CNRM-ESM2-1_anomaly_ssp585_1995-2100_8km_ISMIP6.nc"] + }, + "CSIRO-Mk3-6-0": { + "RCP85": [ + "AIS/Atmosphere_forcing/CSIRO-Mk3-6-0_rcp85/Regridded_8km/CSIRO-Mk3-6-0_8km_anomaly_rcp85_1995-2100.nc"] + }, + "HadGEM2-ES": { + "RCP85": [ + "AIS/Atmosphere_forcing/HadGEM2-ES_rcp85/Regridded_8km/HadGEM2-ES_8km_anomaly_rcp85_1995-2100.nc"] + }, + "IPSL-CM5A-MR": { + "RCP26": [ + "AIS/Atmosphere_Forcing/IPSL-CM5A-MR_rcp26/Regridded_8km/IPSL-CM5A-MR_8km_anomaly_rcp26_1995-2100.nc"], + "RCP85": [ + "AIS/Atmosphere_Forcing/IPSL-CM5A-MR_rcp85/Regridded_8km/IPSL-CM5A-MR_8km_anomaly_rcp85_1995-2100.nc"] + }, + "MIROC-ESM-CHEM": { + "RCP26": [ + "AIS/Atmosphere_Forcing/miroc-esm-chem_rcp2.6/Regridded_8km/MIROC-ESM-CHEM_8km_anomaly_rcp26_1995-2100.nc"], + "RCP85": [ + "AIS/Atmosphere_Forcing/miroc-esm-chem_rcp8.5/Regridded_8km/MIROC-ESM-CHEM_8km_anomaly_1995-2100.nc"] + }, + "NorESM1-M": { + "RCP26": [ + "AIS/Atmosphere_Forcing/noresm1-m_rcp2.6/Regridded_8km/NorESM-M_8km_anomaly_rcp26_1995-2100.nc"], + "RCP85": ["AIS/Atmosphere_Forcing/noresm1-m_rcp8.5/Regridded_8km/NorESM-M_8km_anomaly_1995-2100.nc"] + }, + "UKESM1-0-LL": { + "SSP585": [ + "AIS/Atmosphere_Forcing/UKESM1-0-LL/Regridded_8km/UKESM1-0-LL_anomaly_ssp585_1995-2100_8km.nc"] + } + }, + "2300": { + "CCSM4": { + "RCP85": ["AIS/Atmospheric_forcing/CCSM4_RCP85/Regridded_08km/CCSM4_8km_anomaly_1995-2100.nc", + "AIS/Atmospheric_forcing/CCSM4_RCP85/Regridded_08km/CCSM4_8km_anomaly_2101-2300.nc"] + }, + "CESM2-WACCM": { + "SSP585": [ + "AIS/Atmospheric_forcing/CESM2-WACCM_ssp585/Regridded_8km/CESM2-WACCM_8km_anomaly_ssp585_1995-2100.nc", + "AIS/Atmospheric_forcing/CESM2-WACCM_ssp585/Regridded_8km/CESM2-WACCM_8km_anomaly_ssp585_2101-2299.nc"], + "SSP585-repeat": [ + "AIS/Atmospheric_forcing/CESM2-WACCM_ssp585-repeat/Regridded_8km/CESM2-WACCM_8km_anomaly_ssp585_1995-2300.nc"] + }, + "CSIRO-Mk3-6-0": { + "RCP85": [ + "AIS/Atmospheric_forcing/CSIRO-Mk3-6-0_RCP85/Regridded_8km/CSIRO-Mk3-6-0_8km_anomaly_rcp85_1995-2100.nc", + "AIS/Atmospheric_forcing/CSIRO-Mk3-6-0_RCP85/Regridded_8km/CSIRO-Mk3-6-0_8km_anomaly_rcp85_2101-2300.nc"] + }, + "HadGEM2-ES": { + "RCP85": [ + "AIS/Atmospheric_forcing/HadGEM2-ES_RCP85/Regridded_8km/HadGEM2-ES_8km_anomaly_rcp85_1995-2100.nc", + "AIS/Atmospheric_forcing/HadGEM2-ES_RCP85/Regridded_8km/HadGEM2-ES_8km_anomaly_rcp85_2101-2299.nc"], + "RCP85-repeat": [ + "AIS/Atmospheric_forcing/HadGEM2-ES-RCP85-repeat/Regridded_8km/HadGEM2-ES_8km_anomaly_rcp85_1995-2300.nc"] + }, + "NorESM1-M": { + "RCP26-repeat": [ + "AIS/Atmospheric_forcing/NorESM1-M_RCP26-repeat/Regridded_8km/NorESM1-M_8km_anomaly_rcp26_1995-2300.nc"], + "RCP85-repeat": [ + "AIS/Atmospheric_forcing/NorESM1-M_RCP85-repeat/Regridded_8km/NorESM1-M_anomaly_1995-2300.nc"] + + }, + "UKESM1-0-LL": { + "SSP126": [ + "AIS/Atmospheric_forcing/UKESM1-0-LL_ssp126/Regridded_8km/UKESM1-0-LL_8km_anomaly_ssp126_1995-2100.nc" + "AIS/Atmospheric_forcing/UKESM1-0-LL_ssp126/Regridded_8km/UKESM1-0-LL_8km_anomaly_ssp126_2101-2300.nc"], + "SSP585": [ + "AIS/Atmospheric_forcing/UKESM1-0-LL_ssp585/Regridding_8km/UKESM1-0-LL_8km_anomaly_ssp585_1995-2100.nc", + "AIS/Atmospheric_forcing/UKESM1-0-LL_ssp585/Regridding_8km/UKESM1-0-LL_8km_anomaly_ssp585_2101-2300.nc"], + "SSP585-repeat": [ + "AIS/Atmospheric_forcing/UKESM1-0-LL_ssp585-repeat/Regridding_8km/UKESM1-0-LL_8km_anomaly_ssp585_1995-2300.nc"] + } + } + } From ba140ad5ebf9342026f790c91afed700405ca267 Mon Sep 17 00:00:00 2001 From: hollyhan Date: Wed, 4 May 2022 08:49:11 -0600 Subject: [PATCH 08/58] Change to the use of xarray.Dataarary.dt in adding xtime string --- .../ocean_thermal/process_thermal_forcing.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/compass/landice/tests/ismip6_forcing/ocean_thermal/process_thermal_forcing.py b/compass/landice/tests/ismip6_forcing/ocean_thermal/process_thermal_forcing.py index c96e316742..e1784a0679 100644 --- a/compass/landice/tests/ismip6_forcing/ocean_thermal/process_thermal_forcing.py +++ b/compass/landice/tests/ismip6_forcing/ocean_thermal/process_thermal_forcing.py @@ -1,11 +1,9 @@ import os -import pandas as pd import subprocess import xarray as xr from compass.landice.tests.ismip6_forcing.ocean_thermal.create_mapfile \ import build_mapping_file from mpas_tools.io import write_netcdf -from mpas_tools.logging import check_call from compass.step import Step @@ -157,9 +155,9 @@ def rename_ismip6thermalforcing_to_mali_vars(self, remapped_file_temp, # add xtime variable xtime = [] for t_index in range(ds.sizes["Time"]): - date = ds.Time[t_index].values - date = pd.to_datetime(str(date)) - date = date.strftime("%Y-%m-%d_00:00:00").ljust(64) + date = ds.Time[t_index] + date = date.dt.strftime("%Y-%m-%d_00:00:00") + date = str(date.values).ljust(64) xtime.append(date) ds["xtime"] = ("Time", xtime) From 55f056f0f38d1888dee51c5cb57e0004076e350c Mon Sep 17 00:00:00 2001 From: hollyhan Date: Wed, 4 May 2022 17:15:15 -0600 Subject: [PATCH 09/58] Edit config options & I/O filenames & nested dictionaries --- .../ismip6_forcing/atmosphere/process_smb.py | 30 ++-- .../ismip6_forcing/ocean_basal/__init__.py | 45 +---- .../ocean_basal/process_basal_melt.py | 17 +- .../ismip6_forcing/ocean_thermal/__init__.py | 60 +------ .../ocean_thermal/process_thermal_forcing.py | 160 +++++++++++++++--- 5 files changed, 181 insertions(+), 131 deletions(-) diff --git a/compass/landice/tests/ismip6_forcing/atmosphere/process_smb.py b/compass/landice/tests/ismip6_forcing/atmosphere/process_smb.py index 170e1917dd..5a5d9f8784 100644 --- a/compass/landice/tests/ismip6_forcing/atmosphere/process_smb.py +++ b/compass/landice/tests/ismip6_forcing/atmosphere/process_smb.py @@ -32,16 +32,9 @@ def setup(self): """ config = self.config section = config['ismip6_ais'] - base_path_ismip6 = section.get('base_path_ismip6') base_path_mali = section.get('base_path_mali') mali_mesh_file = section.get('mali_mesh_file') - - self.add_input_file(filename=mali_mesh_file, - target=os.path.join(base_path_mali, - mali_mesh_file)) - - section = config['ismip6_ais'] period_endyear = section.get("period_endyear") model = section.get("model") scenario = section.get("scenario") @@ -59,6 +52,10 @@ def setup(self): "should contain the ismip6_ais " "section with the scenario option") + self.add_input_file(filename=mali_mesh_file, + target=os.path.join(base_path_mali, + mali_mesh_file)) + input_file_list = self._files[period_endyear][model][scenario] for file in input_file_list: print(base_path_ismip6) @@ -66,7 +63,7 @@ def setup(self): self.add_input_file(filename=os.path.basename(file), target=os.path.join(base_path_ismip6, file)) - output_file = f"processed_{model}_{scenario}_{period_endyear}.nc" + output_file = f"processed_SMB_{model}_{scenario}_{period_endyear}.nc" self.add_output_file(filename=output_file) def run(self): @@ -77,7 +74,6 @@ def run(self): config = self.config section = config['ismip6_ais'] - # base_path_ismip6 = section.get('base_path_ismip6') mali_mesh_name = section.get('mali_mesh_name') mali_mesh_file = section.get('mali_mesh_file') period_endyear = section.get("period_endyear") @@ -111,7 +107,7 @@ def run(self): # call the function that renames the ismip6 variables to MALI variables logger.info("Renaming the ismip6 variables to mali variable names...") - output_file = f"processed_{model}_{scenario}_{period_endyear}.nc" + output_file = f"processed_SMB_{model}_{scenario}_{period_endyear}.nc" self.rename_ismip6smb_to_mali_vars(remapped_file_temp, output_file) # correct the SMB anomaly field with mali base SMB field @@ -236,8 +232,10 @@ def correct_SMB_anomaly_for_baseSMB(self, output_file, mali_mesh_file): _files = { "2100": { "CCSM4": { - "RCP26": ["AIS/Atmosphere_forcing/ccsm4_rcp2.6/Regridded_8km/CCSM4_8km_anomaly_rcp26_1995-2100.nc"], - "RCP85": ["AIS/Atmosphere_Forcing/ccsm4_rcp8.5/Regridded_8km/CCSM4_8km_anomaly_1995-2100.nc"] + "RCP26": [ + "AIS/Atmosphere_forcing/ccsm4_rcp2.6/Regridded_8km/CCSM4_8km_anomaly_rcp26_1995-2100.nc"], + "RCP85": [ + "AIS/Atmosphere_Forcing/ccsm4_rcp8.5/Regridded_8km/CCSM4_8km_anomaly_1995-2100.nc"] }, "CESM2": { "SSP585v1": [ @@ -278,7 +276,8 @@ def correct_SMB_anomaly_for_baseSMB(self, output_file, mali_mesh_file): "NorESM1-M": { "RCP26": [ "AIS/Atmosphere_Forcing/noresm1-m_rcp2.6/Regridded_8km/NorESM-M_8km_anomaly_rcp26_1995-2100.nc"], - "RCP85": ["AIS/Atmosphere_Forcing/noresm1-m_rcp8.5/Regridded_8km/NorESM-M_8km_anomaly_1995-2100.nc"] + "RCP85": [ + "AIS/Atmosphere_Forcing/noresm1-m_rcp8.5/Regridded_8km/NorESM-M_8km_anomaly_1995-2100.nc"] }, "UKESM1-0-LL": { "SSP585": [ @@ -287,8 +286,9 @@ def correct_SMB_anomaly_for_baseSMB(self, output_file, mali_mesh_file): }, "2300": { "CCSM4": { - "RCP85": ["AIS/Atmospheric_forcing/CCSM4_RCP85/Regridded_08km/CCSM4_8km_anomaly_1995-2100.nc", - "AIS/Atmospheric_forcing/CCSM4_RCP85/Regridded_08km/CCSM4_8km_anomaly_2101-2300.nc"] + "RCP85": [ + "AIS/Atmospheric_forcing/CCSM4_RCP85/Regridded_08km/CCSM4_8km_anomaly_1995-2100.nc", + "AIS/Atmospheric_forcing/CCSM4_RCP85/Regridded_08km/CCSM4_8km_anomaly_2101-2300.nc"] }, "CESM2-WACCM": { "SSP585": [ diff --git a/compass/landice/tests/ismip6_forcing/ocean_basal/__init__.py b/compass/landice/tests/ismip6_forcing/ocean_basal/__init__.py index 987849d158..e75a5821d6 100644 --- a/compass/landice/tests/ismip6_forcing/ocean_basal/__init__.py +++ b/compass/landice/tests/ismip6_forcing/ocean_basal/__init__.py @@ -30,35 +30,23 @@ def configure(self): """ Configures test case """ - base_path = self.config.get(section="ismip6_ais", - option="base_path") - model = self.config.get(section="ismip6_ais", - option="model") - scenario = self.config.get(section="ismip6_ais", - option="scenario") - period_endyear = self.config.get(section="ismip6_ais", - option="period_endyear") + base_path_ismip6 = self.config.get(section="ismip6_ais", + option="base_path_ismip6") + base_path_mali = self.config.get(section="ismip6_ais", + option="base_path_mali") mali_mesh_name = self.config.get(section="ismip6_ais", option="mali_mesh_name") mali_mesh_file = self.config.get(section="ismip6_ais", option="mali_mesh_file") - if base_path == "NotAvailable": + if base_path_ismip6 == "NotAvailable": raise ValueError("You need to supply a user config file, which " "should contain the ismip6_ais " - "section with the base_path option") - if model == "NotAvailable": + "section with the base_path_ismip6 option") + if base_path_mali == "NotAvailable": raise ValueError("You need to supply a user config file, which " "should contain the ismip6_ais " - "section with the model option") - if scenario == "NotAvailable": - raise ValueError("You need to supply a user config file, which " - "should contain the ismip6_ais " - "section with the scenario option") - if period_endyear == "NotAvailable": - raise ValueError("You need to supply a user config file, which " - "should contain the ismip6_ais " - "section with the period_endyear option") + "section with the base_path_mali option") if mali_mesh_name == "NotAvailable": raise ValueError("You need to supply a user config file, which " "should contain the ismip6_ais " @@ -67,20 +55,3 @@ def configure(self): raise ValueError("You need to supply a user config file, which " "should contain the ismip6_ais " "section with the mali_mesh_file option") - - # input files: input uniform melt rate coefficient (gamma0) - # and temperature correction per basin - _files = { - "2100":["coeff_gamma0_DeltaT_quadratic_local_5th_pct_PIGL_gamma_calibration.nc", - "coeff_gamma0_DeltaT_quadratic_local_5th_percentile.nc", - "coeff_gamma0_DeltaT_quadratic_local_95th_pct_PIGL_gamma_calibration.nc", - "coeff_gamma0_DeltaT_quadratic_local_95th_percentile.nc", - "coeff_gamma0_DeltaT_quadratic_local_median_PIGL_gamma_calibration.nc", - "coeff_gamma0_DeltaT_quadratic_local_median.nc", - "coeff_gamma0_DeltaT_quadratic_non_local_5th_pct_PIGL_gamma_calibration.nc", - "coeff_gamma0_DeltaT_quadratic_non_local_5th_percentile.nc", - "coeff_gamma0_DeltaT_quadratic_non_local_95th_pct_PIGL_gamma_calibration.nc", - "coeff_gamma0_DeltaT_quadratic_non_local_95th_percentile.nc", - "coeff_gamma0_DeltaT_quadratic_non_local_median_PIGL_gamma_calibration.nc", - "coeff_gamma0_DeltaT_quadratic_non_local_median.nc"] - } \ No newline at end of file diff --git a/compass/landice/tests/ismip6_forcing/ocean_basal/process_basal_melt.py b/compass/landice/tests/ismip6_forcing/ocean_basal/process_basal_melt.py index 1f8a62736b..41317e82b4 100644 --- a/compass/landice/tests/ismip6_forcing/ocean_basal/process_basal_melt.py +++ b/compass/landice/tests/ismip6_forcing/ocean_basal/process_basal_melt.py @@ -30,21 +30,26 @@ def setup(self): """ config = self.config section = config['ismip6_ais'] - base_path = section.get('base_path') + base_path_ismip6 = section.get('base_path_ismip6') + base_path_mali = section.get('base_path_mali') mali_mesh_file = section.get('mali_mesh_file') + section = config['ismip6_ais_ocean_basal'] basin_file = section.get('basin_file') coeff_file = section.get('coeff_file') + + self.add_input_file(filename=mali_mesh_file, + target=os.path.join(base_path_mali, + mali_mesh_file)) self.add_input_file(filename=basin_file, - target=os.path.join(base_path, + target=os.path.join(base_path_ismip6, basin_file)) self.add_input_file(filename=coeff_file, - target=os.path.join(base_path, + target=os.path.join(base_path_ismip6, coeff_file)) - self.add_input_file(filename=mali_mesh_file, - target=os.path.join(base_path, mali_mesh_file)) self.add_output_file(filename=f"output_basin_and_{coeff_file}") + def run(self): """ Run this step of the test case @@ -58,7 +63,6 @@ def run(self): basin_file = section.get('basin_file') coeff_file = section.get('coeff_file') method_remap = section.get('method_remap') - output_file = f"output_basin_and_{coeff_file}" # combine, interpolate and rename the basin file and deltaT0_gamma0 # ismip6 input files @@ -76,6 +80,7 @@ def run(self): remapped_file_temp, mali_mesh_name, mali_mesh_file, method_remap) + output_file = f"output_basin_and_{coeff_file}" # call the function that renames the ismip6 variables to MALI variables print("Renaming the ismip6 variables to mali variable names...") self.rename_ismip6BasalMelt_to_mali_vars(remapped_file_temp, diff --git a/compass/landice/tests/ismip6_forcing/ocean_thermal/__init__.py b/compass/landice/tests/ismip6_forcing/ocean_thermal/__init__.py index e7a4528375..582d0fdcf8 100644 --- a/compass/landice/tests/ismip6_forcing/ocean_thermal/__init__.py +++ b/compass/landice/tests/ismip6_forcing/ocean_thermal/__init__.py @@ -30,35 +30,23 @@ def configure(self): """ Configures test case """ - base_path = self.config.get(section="ismip6_ais", - option="base_path") - model = self.config.get(section="ismip6_ais", - option="model") - scenario = self.config.get(section="ismip6_ais", - option="scenario") - period_endyear = self.config.get(section="ismip6_ais", - option="period_endyear") + base_path_ismip6 = self.config.get(section="ismip6_ais", + option="base_path_ismip6") + base_path_mali = self.config.get(section="ismip6_ais", + option="base_path_mali") mali_mesh_name = self.config.get(section="ismip6_ais", option="mali_mesh_name") mali_mesh_file = self.config.get(section="ismip6_ais", option="mali_mesh_file") - if base_path == "NotAvailable": + if base_path_ismip6 == "NotAvailable": raise ValueError("You need to supply a user config file, which " "should contain the ismip6_ais " - "section with the base_path option") - if model == "NotAvailable": + "section with the base_path_ismip6 option") + if base_path_mali == "NotAvailable": raise ValueError("You need to supply a user config file, which " "should contain the ismip6_ais " - "section with the model option") - if scenario == "NotAvailable": - raise ValueError("You need to supply a user config file, which " - "should contain the ismip6_ais " - "section with the scenario option") - if period_endyear == "NotAvailable": - raise ValueError("You need to supply a user config file, which " - "should contain the ismip6_ais " - "section with the period_endyear option") + "section with the base_path_mali option") if mali_mesh_name == "NotAvailable": raise ValueError("You need to supply a user config file, which " "should contain the ismip6_ais " @@ -67,35 +55,3 @@ def configure(self): raise ValueError("You need to supply a user config file, which " "should contain the ismip6_ais " "section with the mali_mesh_file option") - - _files = { - "2100": { - "HadGEM2-ES":{ - "RCP85": ["file1", "file2"], - "RCP26": ["file1", "file2"] - }, - "CESM2-WACCM":{} - }, - "2300": { - "CCSM4":{ - "RCP85":["AIS/Ocean_forcing/ccsm4_RCP85/1995-2300/CCSM4_RCP85_thermal_forcing_8km_x_60m.nc"] - }, - "CESM2-WACCM":{ - "SSP585":["AIS/Ocean_forcing/CESM2-WACCM_ssp585/1995-2299/CESM2-WACCM_SSP585_thermal_forcing_8km_x_60m.nc"] #repeat: "1AIS/Ocean_forcing/cesm2-waccm_ssp585-repeat/1995-2300/CESM2-WACCM_SSP585_thermal_forcing_8km_x_60m.nc" - }, - # "CSIRO-Mk3-6-0" does not exist for ocean forcing - "HadGEM2-ES":{ - "RCP85":["AIS/Ocean_forcing/hadgem2-es_RCP85/1995-2299/HadGEM2-ES_RCP85_thermal_forcing_8km_x_60m.nc", - "AIS/Ocean_forcing/hadgem2-es_RCP85-repeat/1995-2299/HadGEM2-ES_RCP85_thermal_forcing_8km_x_60m.nc"] - }, - "NorESM1-M":{ - "RCP26":["AIS/Ocean_forcing/noresm1-m_RCP26-repeat/1995-2300/NorESM1-M_RCP26_thermal_forcing_8km_x_60m.nc"], #this is repeat forcing - "RCP85":["/AIS/Ocean_forcing/noresm1-m_RCP85-repeat/1995-2300/NorESM1-M_thermal_forcing_8km_x_60m.nc"] - }, - "UKESM1-0-LL":{ - "SSP126":["AIS/Ocean_forcing/ukesm1-0-ll_ssp126/1995-2300/UKESM1-0-LL_thermal_forcing_8km_x_60m.nc"], - "SSP585":["AIS/Ocean_forcing/ukesm1-0-ll_ssp585/1995-2300/UKESM1-0-LL_SSP585_thermal_forcing_8km_x_60m.nc", - "AIS/Ocean_forcing/ukesm1-0-ll_ssp585-repeat/1995-2300/UKESM1-0-LL_SSP585_thermal_forcing_8km_x_60m.nc"] #this is repeat forcing again - } - } - } \ No newline at end of file diff --git a/compass/landice/tests/ismip6_forcing/ocean_thermal/process_thermal_forcing.py b/compass/landice/tests/ismip6_forcing/ocean_thermal/process_thermal_forcing.py index e1784a0679..a831b74981 100644 --- a/compass/landice/tests/ismip6_forcing/ocean_thermal/process_thermal_forcing.py +++ b/compass/landice/tests/ismip6_forcing/ocean_thermal/process_thermal_forcing.py @@ -28,49 +28,71 @@ def setup(self): Set up this step of the test case """ config = self.config - section = config['ismip6_ais_ocean'] - - input_path = section.get('input_path') - thermal_forcing_file = section.get('thermal_forcing_file') - self.add_input_file(filename=thermal_forcing_file, - target=os.path.join(input_path, - thermal_forcing_file)) + section = config['ismip6_ais'] + base_path_ismip6 = section.get('base_path_ismip6') + base_path_mali = section.get('base_path_mali') mali_mesh_file = section.get('mali_mesh_file') + period_endyear = section.get("period_endyear") + model = section.get("model") + scenario = section.get("scenario") + + if period_endyear == "NotAvailable": + raise ValueError("You need to supply a user config file, which " + "should contain the ismip6_ais " + "section with the period_endyear option") + if model == "NotAvailable": + raise ValueError("You need to supply a user config file, which " + "should contain the ismip6_ais " + "section with the model option") + if scenario == "NotAvailable": + raise ValueError("You need to supply a user config file, which " + "should contain the ismip6_ais " + "section with the scenario option") + self.add_input_file(filename=mali_mesh_file, - target=os.path.join(input_path, mali_mesh_file)) - self.add_output_file(filename=f"output_{thermal_forcing_file}") + target=os.path.join(base_path_mali, + mali_mesh_file)) + + + input_file_list = self._files[period_endyear][model][scenario] + for file in input_file_list: + self.add_input_file(filename=os.path.basename(file), + target=os.path.join(base_path_ismip6, file)) + + output_file = f"processed_TF_{model}_{scenario}_{period_endyear}.nc" + self.add_output_file(filename=output_file) def run(self): """ Run this step of the test case """ - #logger = self.logger + logger = self.logger config = self.config - section = config['ismip6_ais_ocean'] - input_path = section.get('input_path') - thermal_forcing_file = section.get('thermal_forcing_file') - self.add_input_file(filename=thermal_forcing_file, - target=os.path.join(input_path, - thermal_forcing_file)) - mali_mesh_file = section.get('mali_mesh_file') - self.add_input_file(filename=mali_mesh_file, - target=os.path.join(input_path, mali_mesh_file)) + section = config['ismip6_ais'] mali_mesh_name = section.get('mali_mesh_name') mali_mesh_file = section.get('mali_mesh_file') + period_endyear = section.get("period_endyear") + model = section.get("model") + scenario = section.get("scenario") + + section = config['ismip6_ais_ocean_thermal'] method_remap = section.get('method_remap') - output_file = f"output_{thermal_forcing_file}" + + input_file_list = self._files[period_endyear][model][scenario] + input_file = os.path.basename(input_file_list[0]) # interpolate and rename the ismip6 thermal forcing data remapped_file_temp = "remapped.nc" # temporary file name # call the function that reads in, remap and rename the file. print("Calling a remapping function...") - self.remap_ismip6thermalforcing_to_mali(thermal_forcing_file, + self.remap_ismip6thermalforcing_to_mali(input_file, remapped_file_temp, mali_mesh_name, mali_mesh_file, method_remap) + output_file = f"processed_TF_{model}_{scenario}_{period_endyear}.nc" # call the function that renames the ismip6 variables to MALI variables print("Renaming the ismip6 variables to mali variable names...") self.rename_ismip6thermalforcing_to_mali_vars(remapped_file_temp, @@ -170,3 +192,99 @@ def rename_ismip6thermalforcing_to_mali_vars(self, remapped_file_temp, # write to a new netCDF file write_netcdf(ds, output_file) ds.close() + + + # create a nested dictionary for the ISMIP6 original forcing files including relative path + _files = { + "2100": { + "CCSM4": { + # "RCP26": ["AIS/Atmosphere_forcing/ccsm4_rcp2.6/Regridded_8km/CCSM4_8km_anomaly_rcp26_1995-2100.nc"], + "RCP85": ["AIS/Ocean_Forcing/ccsm4_rcp8.5/1995-2100/CCSM4_thermal_forcing_8km_x_60m.nc"] + }, + "CESM2": { + "SSP585": [ + "AIS/Ocean_Forcing/cesm2_ssp585/1995-2100/CESM2_ssp585_thermal_forcing_8km_x_60m.nc"] + # "SSP585v1": [ + # "AIS/Atmosphere_Forcing/CESM2_ssp585/Regridded_8km/CESM2_anomaly_ssp585_1995-2100_8km_v1.nc"], + # "SSP585v2": [ + # "AIS/Atmosphere_forcing/CESM2_ssp585/Regridded_8km/CESM2_anomaly_ssp585_1995-2100_8km_v2.nc"] + }, + "CNRM_CM6": { + "SSP126": [ + "AIS/Ocean_Forcing/cnrm-cm6-1_ssp126/1995-2100/CNRM-CM6-1_ssp126_thermal_forcing_8km_x_60m.nc"], + "SSP585": [ + "AIS/Ocean_Forcing/cnrm-cm6-1_ssp585/1995-2100/CNRM-CM6-1_ssp585_thermal_forcing_8km_x_60m.nc"] + }, + "CNRM_ESM2": { + "SSP585": [ + "AIS/Ocean_Forcing/cnrm-esm2-1_ssp585/1995-2100/CNRM-ESM2-1_ssp585_thermal_forcing_8km_x_60m.nc"] + }, + "CSIRO-Mk3-6-0": { + "RCP85": [ + "AIS/Ocean_Forcing/csiro-mk3-6-0_rcp8.5/1995-2100/CSIRO-Mk3-6-0_RCP85_thermal_forcing_8km_x_60m.nc"] + }, + "HadGEM2-ES": { + "RCP85": [ + "AIS/Ocean_Forcing/hadgem2-es_rcp8.5/1995-2100/HadGEM2-ES_RCP85_thermal_forcing_8km_x_60m.nc"] + }, + "IPSL-CM5A-MR": { + "RCP26": [ + "AIS/Ocean_Forcing/ipsl-cm5a-mr_rcp2.6/1995-2100/IPSL-CM5A-MR_RCP26_thermal_forcing_8km_x_60m.nc"], + "RCP85": [ + "AIS/Ocean_Forcing/ipsl-cm5a-mr_rcp8.5/1995-2100/IPSL-CM5A-MR_RCP85_thermal_forcing_8km_x_60m.nc"] + }, + "MIROC-ESM-CHEM": { + # "RCP26": [ + # "AIS/Atmosphere_Forcing/miroc-esm-chem_rcp2.6/Regridded_8km/MIROC-ESM-CHEM_8km_anomaly_rcp26_1995-2100.nc"], + "RCP85": [ + "AIS/Ocean_Forcing/miroc-esm-chem_rcp8.5/1995-2100/MIROC-ESM-CHEM_thermal_forcing_8km_x_60m.nc"] + }, + "NorESM1-M": { + "RCP26": [ + "AIS/Ocean_Forcing/noresm1-m_rcp2.6/1995-2100/NorESM1-M_RCP26_thermal_forcing_8km_x_60m.nc"], + "RCP85": [ + "AIS/Ocean_Forcing/noresm1-m_rcp8.5/1995-2100/NorESM1-M_thermal_forcing_8km_x_60m.nc"] + }, + "UKESM1-0-LL": { + "SSP585":[ + "AIS/Ocean_Forcing/ukesm1-0-ll_ssp585/1995-2100/UKESM1-0-LL_ssp585_thermal_forcing_8km_x_60m.nc"] + } + }, + "2300": { + "CCSM4": { + "RCP85": [ + "AIS/Ocean_forcing/ccsm4_RCP85/1995-2300/CCSM4_RCP85_thermal_forcing_8km_x_60m.nc"] + }, + "CESM2-WACCM": { + "SSP585":[ + "AIS/Ocean_forcing/cesm2-waccm_ssp585/1995-2299/CESM2-WACCM_SSP585_thermal_forcing_8km_x_60m.nc"], + "SSP585-repeat": [ + "AIS/Ocean_forcing/cesm2-waccm_ssp585-repeat/1995-2300/CESM2-WACCM_ssp585_thermal_forcing_8km_x_60m.nc"] + }, + # "CSIRO-Mk3-6-0": { + # "RCP85": [ + # "AIS/Atmospheric_forcing/CSIRO-Mk3-6-0_RCP85/Regridded_8km/CSIRO-Mk3-6-0_8km_anomaly_rcp85_1995-2100.nc", + # "AIS/Atmospheric_forcing/CSIRO-Mk3-6-0_RCP85/Regridded_8km/CSIRO-Mk3-6-0_8km_anomaly_rcp85_2101-2300.nc"] +# }, + "HadGEM2-ES": { + "RCP85": [ + "AIS/Ocean_forcing/hadgem2-es_RCP85/1995-2299/HadGEM2-ES_RCP85_thermal_forcing_8km_x_60m.nc"], + "RCP85-repeat": [ + "AIS/Ocean_forcing/hadgem2-es_RCP85-repeat/1995-2300/HadGEM2-ES_rcp85_thermal_forcing_8km_x_60m.nc"] + }, + "NorESM1-M": { + "RCP26-repeat": [ + "AIS/Ocean_forcing/noresm1-m_RCP26-repeat/1995-2300/NorESM1-M_RCP26_thermal_forcing_8km_x_60m.nc"], + "RCP85-repeat": [ + "AIS/Ocean_forcing/noresm1-m_RCP85-repeat/1995-2300/NorESM1-M_thermal_forcing_8km_x_60m.nc"] + }, + "UKESM1-0-LL": { + "SSP126": [ + "AIS/Ocean_forcing/ukesm1-0-ll_ssp126/1995-2300/UKESM1-0-LL_thermal_forcing_8km_x_60m.nc"], + "SSP585": [ + "AIS/Ocean_forcing/ukesm1-0-ll_ssp585/1995-2300/UKESM1-0-LL_SSP585_thermal_forcing_8km_x_60m.nc"], + "SSP585-repeat": [ + "AIS/Ocean_forcing/ukesm1-0-ll_ssp585-repeat/1995-2300/UKESM1-0-LL_ssp585_thermal_forcing_8km_x_60m.nc"] + } + } + } From 3f1e21f8f6ea9f4e1bf5085c6b4b755506553fae Mon Sep 17 00:00:00 2001 From: hollyhan Date: Mon, 9 May 2022 23:02:55 -0600 Subject: [PATCH 10/58] Add documentation of the test group --- docs/developers_guide/landice/api.rst | 16 +++ .../landice/test_groups/index.rst | 1 + .../landice/test_groups/ismip6_forcing.rst | 50 ++++++++ .../users_guide/landice/test_groups/index.rst | 1 + .../landice/test_groups/ismip6_forcing.rst | 118 ++++++++++++++++++ 5 files changed, 186 insertions(+) create mode 100644 docs/developers_guide/landice/test_groups/ismip6_forcing.rst create mode 100644 docs/users_guide/landice/test_groups/ismip6_forcing.rst diff --git a/docs/developers_guide/landice/api.rst b/docs/developers_guide/landice/api.rst index 8499c2f410..9d4a7fcaf9 100644 --- a/docs/developers_guide/landice/api.rst +++ b/docs/developers_guide/landice/api.rst @@ -231,6 +231,22 @@ hydro_radial visualize.Visualize.run visualize.visualize_hydro_radial +ismip6_forcing +~~~~~~~~~~~~~~ + +.. currentmodule:: compass.landice.tests.ismip6_forcing + +.. autosummary:: + :toctree: generated/ + + Ismip6Forcing + + atmosphere.Atmosphere + atmosphere.Atmosphere.run + + ocean_thermal.OceanThermal + ocean_thermal.OceanThermal.run + kangerlussuaq ~~~~~~~~~~~~~ diff --git a/docs/developers_guide/landice/test_groups/index.rst b/docs/developers_guide/landice/test_groups/index.rst index 94760fc976..02e83acaa6 100644 --- a/docs/developers_guide/landice/test_groups/index.rst +++ b/docs/developers_guide/landice/test_groups/index.rst @@ -15,6 +15,7 @@ Test groups greenland humboldt hydro_radial + ismip6_forcing kangerlussuaq koge_bugt_s mismipplus diff --git a/docs/developers_guide/landice/test_groups/ismip6_forcing.rst b/docs/developers_guide/landice/test_groups/ismip6_forcing.rst new file mode 100644 index 0000000000..4907493f0c --- /dev/null +++ b/docs/developers_guide/landice/test_groups/ismip6_forcing.rst @@ -0,0 +1,50 @@ +.. _dev_landice_ismip6_forcing: + +ismip6_forcing +============== + +The ``ismip6_forcing`` test group (:py:class:`compass.landice.tests.ismip6_ +forcing.Ismip6Forcing`) processes the atmospheric and oceaninc forcing data of +the Ice Sheet Model Intercomparison for CMIP6 (ISMIP6) protocol. Here, +we describe the shared framework for this test group and the 3 test cases. + +.. _dev_landice_ismip6_forcing_framework: + +framework +--------- + +The shared config options for the ``ismip6_forcing`` test group are described +in :ref:`landice_ismip6_forcing` in the User's Guide. + + +.. _dev_landice_ismip6_forcing_atmosphere: + +atmosphere +---------- + +The :py:class:`compass.landice.tests.ismip6_forcing.atmosphere.Atmosphere` +performs processing of the surface mass balance (SMB) forcing. +Processing data includes regridding the original ISMIP6 SMB data from its +native polarstereo grid to MALI's unstructured grid, renaming variables and +correcting the SMB anomaly field for the MALI base SMB. + +.. _dev_landice_ocean_thermal: + +ocean_thermal +------------- + +The :py:class:`compass.landice.tests.ismip6_forcing.ocean_thermal.OceanThermal` +performs the processing of ocean thermal forcing. Processing data includes +regridding the original ISMIP6 thermal forcing data from its native +polarstereo grid to MALI's unstructured grid and renaming variables. + +.. _dev_landice_ocean_basal: + +ocean_basal +------------ + +The :py:class:`compass.landice.tests.ismip6_forcing.ocean_basal.OceanBasal` +performs processing of the coefficients for the basal melt parametrization +utilized by the ISMIP6 protocol. Processing data includes combining the +IMBIE2 basin number file and parametrization coefficients and remapping onto +the MALI mesh. diff --git a/docs/users_guide/landice/test_groups/index.rst b/docs/users_guide/landice/test_groups/index.rst index 044b85404f..4123aae7c7 100644 --- a/docs/users_guide/landice/test_groups/index.rst +++ b/docs/users_guide/landice/test_groups/index.rst @@ -20,6 +20,7 @@ physics but that are not run routinely. greenland humboldt hydro_radial + ismip6_forcing kangerlussuaq koge_bugt_s mismipplus diff --git a/docs/users_guide/landice/test_groups/ismip6_forcing.rst b/docs/users_guide/landice/test_groups/ismip6_forcing.rst new file mode 100644 index 0000000000..62bb9d00f6 --- /dev/null +++ b/docs/users_guide/landice/test_groups/ismip6_forcing.rst @@ -0,0 +1,118 @@ +.. _landice_ismip6_forcing: + +hydro_radial +============ + +The ``landice/ismip6_forcing`` test group implements processing of atmospheric +and ocean forcing data of the Ice Sheet Model Intercomparison for CMIP6 +(ISMIP6) protocol. + +The test group includes 3 test cases, ``atmosphere``, ``ocean_thermal`` and +``ocean_basal``. All test cases are made up of a single main step, +``ProcessSMB``,``ProcessThermalForcing`` and ``ProcessBasalMelt``, +respectively. Each step has the local methods (functions) of remapping and +renaming the original ismip6 data to the format that MALI can incorporate in +its forward simulations. In remapping the data, all test cases import the +method ``build_mapping_file`` to create or use scrip files +of the source (ISMIP6) and destination (MALI) mesh depending on the existence +of a mapping file. + +config options +-------------- + +All three test cases share some set of default config options under the section +``[ismip6_ais]`` and separate config options under the corresponding section: + +.. code-block:: cfg + + # config options for ismip6 antarctic ice sheet SMB forcing data set + [ismip6_ais] + + # Base path to the input ismip6 ocean and smb forcing files. User has to supply. + base_path_ismip6 = NotAvailable + + # Base path to the the MALI mesh. User has to supply. + base_path_mali = NotAvailable + + # Forcing end year of the ISMIP6 data. User has to supply. + # Available end years are 2100 and 2300. + period_endyear = NotAvailable + + # Name of climate model name used to generate ISMIP6 forcing data. User has to supply. + # Available model names are the following: CCSM4, CESM2-WACCM, CSIRO-Mk3-6-0, HadGEM2-ES, NorESM1-M, UKESM1-0-LL + model = NotAvailable + + # Scenarios used by climate model. User has to supply. + # Available scenarios are the following: RCP26, RCP85, SSP126, SSP585 + scenario = NotAvailable + + # name of the mali mesh used to name mapping files (e,g. Antarctica_8to80km). User has to supply. + mali_mesh_name = NotAvailable + + # MALI mesh file to be used to build mapping file (netCDF format). User has to supply. + mali_mesh_file = NotAvailable + + + # config options for ismip6 antarctic ice sheet SMB forcing data test cases + [ismip6_ais_atmosphere] + + # Remapping method used in building a mapping file. Options include: bilinear, neareststod, conserve + method_remap = bilinear + + # Path to which processed output SMB data is saved. User has to supply. + output_path = NotAvailable + + + # config options for ismip6 ocean thermal forcing data test cases + [ismip6_ais_ocean_thermal] + + # Remapping method used in building a mapping file. Options include: bilinear, neareststod, conserve + method_remap = bilinear + + # Path to which processed output thermal forcing data is saved. User has to supply. + output_path = NotAvailable + + + # config options for ismip6 ocean basal test case + [ismip6_ais_ocean_basal] + + # Input imbie2 basin number file. + basin_file = imbie2_basin_numbers_8km.nc + + # Input uniform melt rate coefficient (gamma0) + # and temperature correction per basin file + coeff_file = coeff_gamma0_DeltaT_quadratic_non_local_median.nc + + # Remapping method used in building a mapping file. Options include: bilinear, neareststod, conserve + method_remap = neareststod + + # Path to which processed output basal param coeff data is saved. User has to supply. + output_path = NotAvailable + + +atmosphere +---------- + +The ``landice/ismip6_forcing/atmosphere`` +performs processing of the surface mass balance (SMB) forcing. +Processing data includes regridding the original ISMIP6 SMB data from its +native polarstereo grid to MALI's unstructured grid, renaming variables and +correcting the SMB anomaly field for the MALI base SMB. + +ocean_thermal +------------- + +The ``landice/ismip6_forcing/ocean_thermal`` +performs the processing of ocean thermal forcing. Processing data includes +regridding the original ISMIP6 thermal forcing data from its native +polarstereo grid to MALI's unstructured grid and renaming variables. + +ocean_basal +------------ + +The ``landice.tests.ismip6_forcing.ocean_basal`` +performs processing of the coefficients for the basal melt parametrization +utilized by the ISMIP6 protocol. Processing data includes combining the +IMBIE2 basin number file and parametrization coefficients and remapping onto +the MALI mesh. + From 1e3a61dddee7dba7f9eda976d12c72689c6925f4 Mon Sep 17 00:00:00 2001 From: hollyhan Date: Tue, 10 May 2022 11:30:16 -0600 Subject: [PATCH 11/58] Edit config option --- docs/users_guide/landice/test_groups/ismip6_forcing.rst | 7 ------- 1 file changed, 7 deletions(-) diff --git a/docs/users_guide/landice/test_groups/ismip6_forcing.rst b/docs/users_guide/landice/test_groups/ismip6_forcing.rst index 62bb9d00f6..d59de33db6 100644 --- a/docs/users_guide/landice/test_groups/ismip6_forcing.rst +++ b/docs/users_guide/landice/test_groups/ismip6_forcing.rst @@ -76,13 +76,6 @@ All three test cases share some set of default config options under the section # config options for ismip6 ocean basal test case [ismip6_ais_ocean_basal] - # Input imbie2 basin number file. - basin_file = imbie2_basin_numbers_8km.nc - - # Input uniform melt rate coefficient (gamma0) - # and temperature correction per basin file - coeff_file = coeff_gamma0_DeltaT_quadratic_non_local_median.nc - # Remapping method used in building a mapping file. Options include: bilinear, neareststod, conserve method_remap = neareststod From bf4aeaf342dcda81dd3f0f7f26696d10baf3390b Mon Sep 17 00:00:00 2001 From: hollyhan Date: Tue, 10 May 2022 11:32:27 -0600 Subject: [PATCH 12/58] Edit print statement to include remapping method --- .../tests/ismip6_forcing/atmosphere/create_mapfile_smb.py | 2 +- .../landice/tests/ismip6_forcing/ocean_basal/create_mapfile.py | 2 +- .../tests/ismip6_forcing/ocean_thermal/create_mapfile.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/compass/landice/tests/ismip6_forcing/atmosphere/create_mapfile_smb.py b/compass/landice/tests/ismip6_forcing/atmosphere/create_mapfile_smb.py index 7fa03459c0..59321ec7e0 100644 --- a/compass/landice/tests/ismip6_forcing/atmosphere/create_mapfile_smb.py +++ b/compass/landice/tests/ismip6_forcing/atmosphere/create_mapfile_smb.py @@ -57,7 +57,7 @@ def build_mapping_file(ismip6_grid_file, mapping_file, mali_mesh_file=None, scrip_from_mpas(mali_mesh_file, mali_scripfile) # create a mapping file using ESMF weight gen - print("Creating a mapping file...") + print(f"Creating a mapping file. Mapping method used: {method_remap}") if method_remap is None: raise ValueError("Desired remapping option should be provided with " diff --git a/compass/landice/tests/ismip6_forcing/ocean_basal/create_mapfile.py b/compass/landice/tests/ismip6_forcing/ocean_basal/create_mapfile.py index 05df40f32a..b5f86452a2 100644 --- a/compass/landice/tests/ismip6_forcing/ocean_basal/create_mapfile.py +++ b/compass/landice/tests/ismip6_forcing/ocean_basal/create_mapfile.py @@ -59,7 +59,7 @@ def build_mapping_file(ismip6_grid_file, mapping_file, mali_mesh_file=None, scrip_from_mpas(mali_mesh_file, mali_scripfile) # create a mapping file using ESMF weight gen - print("Creating a mapping file...") + print(f"Creating a mapping file. Mapping method used: {method_remap}") if method_remap is None: raise ValueError("Desired remapping option should be provided with " diff --git a/compass/landice/tests/ismip6_forcing/ocean_thermal/create_mapfile.py b/compass/landice/tests/ismip6_forcing/ocean_thermal/create_mapfile.py index 05df40f32a..b5f86452a2 100644 --- a/compass/landice/tests/ismip6_forcing/ocean_thermal/create_mapfile.py +++ b/compass/landice/tests/ismip6_forcing/ocean_thermal/create_mapfile.py @@ -59,7 +59,7 @@ def build_mapping_file(ismip6_grid_file, mapping_file, mali_mesh_file=None, scrip_from_mpas(mali_mesh_file, mali_scripfile) # create a mapping file using ESMF weight gen - print("Creating a mapping file...") + print(f"Creating a mapping file. Mapping method used: {method_remap}") if method_remap is None: raise ValueError("Desired remapping option should be provided with " From 604d88476e5246e72eb9752b7a201dcf44e6daf5 Mon Sep 17 00:00:00 2001 From: hollyhan Date: Tue, 10 May 2022 11:33:44 -0600 Subject: [PATCH 13/58] Modify the ocean_basal testcase to process all files at once Also set path to output data from config option --- .../ocean_basal/process_basal_melt.py | 104 ++++++++++++------ 1 file changed, 72 insertions(+), 32 deletions(-) diff --git a/compass/landice/tests/ismip6_forcing/ocean_basal/process_basal_melt.py b/compass/landice/tests/ismip6_forcing/ocean_basal/process_basal_melt.py index 41317e82b4..defb33412d 100644 --- a/compass/landice/tests/ismip6_forcing/ocean_basal/process_basal_melt.py +++ b/compass/landice/tests/ismip6_forcing/ocean_basal/process_basal_melt.py @@ -1,4 +1,5 @@ import os +import shutil import subprocess import xarray as xr from compass.landice.tests.ismip6_forcing.ocean_basal.create_mapfile \ @@ -35,20 +36,19 @@ def setup(self): mali_mesh_file = section.get('mali_mesh_file') section = config['ismip6_ais_ocean_basal'] - basin_file = section.get('basin_file') - coeff_file = section.get('coeff_file') - self.add_input_file(filename=mali_mesh_file, target=os.path.join(base_path_mali, mali_mesh_file)) - self.add_input_file(filename=basin_file, - target=os.path.join(base_path_ismip6, - basin_file)) - self.add_input_file(filename=coeff_file, + self.add_input_file(filename=os.path.basename(self._file_basin), target=os.path.join(base_path_ismip6, - coeff_file)) - self.add_output_file(filename=f"output_basin_and_{coeff_file}") + self._file_basin)) + input_file_list = self._files_coeff + for file in input_file_list: + self.add_input_file(filename=os.path.basename(file), + target=os.path.join(base_path_ismip6, file)) + self.add_output_file(filename=f"processed_basin_and_" + f"{os.path.basename(file)}") def run(self): """ @@ -59,10 +59,10 @@ def run(self): section = config['ismip6_ais'] mali_mesh_name = section.get('mali_mesh_name') mali_mesh_file = section.get('mali_mesh_file') + section = config['ismip6_ais_ocean_basal'] - basin_file = section.get('basin_file') - coeff_file = section.get('coeff_file') method_remap = section.get('method_remap') + output_path = section.get('output_path') # combine, interpolate and rename the basin file and deltaT0_gamma0 # ismip6 input files @@ -71,27 +71,51 @@ def run(self): # call the function that combines data # logger.info = ('calling combine_ismip6_inputfiles') - self.combine_ismip6_inputfiles(basin_file, coeff_file, - combined_file_temp) - - # call the function that reads in, remap and rename the file. - print("Calling a remapping function...") - self.remap_ismip6BasalMelt_to_mali(combined_file_temp, - remapped_file_temp, mali_mesh_name, - mali_mesh_file, method_remap) - - output_file = f"output_basin_and_{coeff_file}" - # call the function that renames the ismip6 variables to MALI variables - print("Renaming the ismip6 variables to mali variable names...") - self.rename_ismip6BasalMelt_to_mali_vars(remapped_file_temp, - output_file) - - print("Remapping and renamping process done successfully. " - "Removing the temporary files 'combined.nc' and 'remapped.nc'") - - # remove the temporary combined file - os.remove(combined_file_temp) - os.remove(remapped_file_temp) + input_file_list = self._files_coeff + i = 0 + for file in input_file_list: + print(f"processing the input file {os.path.basename(file)}") + i += 1 + self.combine_ismip6_inputfiles(os.path.basename(self._file_basin), + os.path.basename(file), + combined_file_temp) + + # remap the input forcing file. + print("Calling the remapping function...") + self.remap_ismip6BasalMelt_to_mali(combined_file_temp, + remapped_file_temp, + mali_mesh_name, + mali_mesh_file, method_remap) + + output_file = f"processed_basin_and_{os.path.basename(file)}" + + # rename the ismip6 variables to MALI variables + print("Renaming the ismip6 variables to mali variable names...") + self.rename_ismip6BasalMelt_to_mali_vars(remapped_file_temp, + output_file) + + print("Remapping and renamping process done successfully. " + "Removing the temporary files 'combined.nc' " + "and 'remapped.nc'") + + # remove the temporary combined file + os.remove(combined_file_temp) + os.remove(remapped_file_temp) + + # place the output file in appropriate directory + print(output_path) + if output_path == "NotAvailable": + pass + else: + if not os.path.exists(output_path): + print("Creating a new directory for the output data") + os.makedirs(output_path) + + src = os.path.join(os.getcwd(), output_file) + dst = os.path.join(output_path, output_file) + shutil.copy(src, dst) + + print("") def combine_ismip6_inputfiles(self, basin_file, coeff_gamma0_deltaT_file, combined_file_temp): @@ -188,3 +212,19 @@ def rename_ismip6BasalMelt_to_mali_vars(self, remapped_file_temp, # write to a new netCDF file write_netcdf(ds, output_file) ds.close() + + # input files: input uniform melt rate coefficient (gamma0) + # and temperature correction per basin + _file_basin = "AIS/Ocean_Forcing/imbie2/imbie2_basin_numbers_8km.nc" + _files_coeff = {"AIS/Ocean_Forcing/parametrizations/coeff_gamma0_DeltaT_quadratic_local_5th_pct_PIGL_gamma_calibration.nc", + "AIS/Ocean_Forcing/parametrizations/coeff_gamma0_DeltaT_quadratic_local_5th_percentile.nc", + "AIS/Ocean_Forcing/parametrizations/coeff_gamma0_DeltaT_quadratic_local_95th_pct_PIGL_gamma_calibration.nc", + "AIS/Ocean_Forcing/parametrizations/coeff_gamma0_DeltaT_quadratic_local_95th_percentile.nc", + "AIS/Ocean_Forcing/parametrizations/coeff_gamma0_DeltaT_quadratic_local_median_PIGL_gamma_calibration.nc", + "AIS/Ocean_Forcing/parametrizations/coeff_gamma0_DeltaT_quadratic_local_median.nc", + "AIS/Ocean_Forcing/parametrizations/coeff_gamma0_DeltaT_quadratic_non_local_5th_pct_PIGL_gamma_calibration.nc", + "AIS/Ocean_Forcing/parametrizations/coeff_gamma0_DeltaT_quadratic_non_local_5th_percentile.nc", + "AIS/Ocean_Forcing/parametrizations/coeff_gamma0_DeltaT_quadratic_non_local_95th_pct_PIGL_gamma_calibration.nc", + "AIS/Ocean_Forcing/parametrizations/coeff_gamma0_DeltaT_quadratic_non_local_95th_percentile.nc", + "AIS/Ocean_Forcing/parametrizations/coeff_gamma0_DeltaT_quadratic_non_local_median_PIGL_gamma_calibration.nc", + "AIS/Ocean_Forcing/parametrizations/coeff_gamma0_DeltaT_quadratic_non_local_median.nc"} \ No newline at end of file From a39f8513c7c592ab2e3129ac82a08e37a75696f4 Mon Sep 17 00:00:00 2001 From: hollyhan Date: Tue, 10 May 2022 11:45:10 -0600 Subject: [PATCH 14/58] Set path to output data from config options Also fix the dictionary list --- .../ismip6_forcing/atmosphere/process_smb.py | 68 ++++++++++--------- .../ocean_thermal/process_thermal_forcing.py | 52 +++++++------- 2 files changed, 65 insertions(+), 55 deletions(-) diff --git a/compass/landice/tests/ismip6_forcing/atmosphere/process_smb.py b/compass/landice/tests/ismip6_forcing/atmosphere/process_smb.py index 5a5d9f8784..9e520e8d35 100644 --- a/compass/landice/tests/ismip6_forcing/atmosphere/process_smb.py +++ b/compass/landice/tests/ismip6_forcing/atmosphere/process_smb.py @@ -1,4 +1,5 @@ import os +import shutil import subprocess import xarray as xr from compass.landice.tests.ismip6_forcing.atmosphere.create_mapfile_smb \ @@ -82,6 +83,7 @@ def run(self): section = config['ismip6_ais_atmosphere'] method_remap = section.get('method_remap') + output_path = section.get('output_path') input_file_list = self._files[period_endyear][model][scenario] i = 0 @@ -121,6 +123,20 @@ def run(self): os.remove(remapped_file_temp) os.remove(combined_file_temp) + # place the output file in appropriate directory + if output_path == "NotAvailable": + pass + else: + if not os.path.exists(output_path): + print("Creating a new directory for the output data") + os.makedirs(output_path) + + src = os.path.join(os.getcwd(), output_file) + dst = os.path.join(output_path, output_file) + print(src) + print(dst) + shutil.copy(src, dst) + def remap_ismip6smb_to_mali(self, input_file, output_file, mali_mesh_name, mali_mesh_file, method_remap): """ @@ -232,8 +248,6 @@ def correct_SMB_anomaly_for_baseSMB(self, output_file, mali_mesh_file): _files = { "2100": { "CCSM4": { - "RCP26": [ - "AIS/Atmosphere_forcing/ccsm4_rcp2.6/Regridded_8km/CCSM4_8km_anomaly_rcp26_1995-2100.nc"], "RCP85": [ "AIS/Atmosphere_Forcing/ccsm4_rcp8.5/Regridded_8km/CCSM4_8km_anomaly_1995-2100.nc"] }, @@ -241,25 +255,25 @@ def correct_SMB_anomaly_for_baseSMB(self, output_file, mali_mesh_file): "SSP585v1": [ "AIS/Atmosphere_Forcing/CESM2_ssp585/Regridded_8km/CESM2_anomaly_ssp585_1995-2100_8km_v1.nc"], "SSP585v2": [ - "AIS/Atmosphere_forcing/CESM2_ssp585/Regridded_8km/CESM2_anomaly_ssp585_1995-2100_8km_v2.nc"] + "AIS/Atmosphere_Forcing/CESM2_ssp585/Regridded_8km/CESM2_anomaly_ssp585_1995-2100_8km_v2.nc"] }, "CNRM_CM6": { "SSP126": [ - "AIS/Atmosphere_forcing/CNRM_CM6_ssp126/Regridded_8km/CNRM-CM6-1_anomaly_ssp126_1995-2100_8km_ISMIP6.nc"], + "AIS/Atmosphere_Forcing/CNRM_CM6_ssp126/Regridded_8km/CNRM-CM6-1_anomaly_ssp126_1995-2100_8km_ISMIP6.nc"], "SSP585": [ - "AIS/Atmosphere_forcing/CNRM_CM6_ssp585/Regridded_8km/CNRM-CM6-1_anomaly_ssp585_1995-2100_8km_ISMIP6.nc"] + "AIS/Atmosphere_Forcing/CNRM_CM6_ssp585/Regridded_8km/CNRM-CM6-1_anomaly_ssp585_1995-2100_8km_ISMIP6.nc"] }, "CNRM_ESM2": { "SSP585": [ - "AIS/Atmosphere_forcing/CNRM_ESM2_ssp585/Regridded_8km/CNRM-ESM2-1_anomaly_ssp585_1995-2100_8km_ISMIP6.nc"] + "AIS/Atmosphere_Forcing/CNRM_ESM2_ssp585/Regridded_8km/CNRM-ESM2-1_anomaly_ssp585_1995-2100_8km_ISMIP6.nc"] }, "CSIRO-Mk3-6-0": { "RCP85": [ - "AIS/Atmosphere_forcing/CSIRO-Mk3-6-0_rcp85/Regridded_8km/CSIRO-Mk3-6-0_8km_anomaly_rcp85_1995-2100.nc"] + "AIS/Atmosphere_Forcing/CSIRO-Mk3-6-0_rcp85/Regridded_8km/CSIRO-Mk3-6-0_8km_anomaly_rcp85_1995-2100.nc"] }, "HadGEM2-ES": { "RCP85": [ - "AIS/Atmosphere_forcing/HadGEM2-ES_rcp85/Regridded_8km/HadGEM2-ES_8km_anomaly_rcp85_1995-2100.nc"] + "AIS/Atmosphere_Forcing/HadGEM2-ES_rcp85/Regridded_8km/HadGEM2-ES_8km_anomaly_rcp85_1995-2100.nc"] }, "IPSL-CM5A-MR": { "RCP26": [ @@ -268,8 +282,6 @@ def correct_SMB_anomaly_for_baseSMB(self, output_file, mali_mesh_file): "AIS/Atmosphere_Forcing/IPSL-CM5A-MR_rcp85/Regridded_8km/IPSL-CM5A-MR_8km_anomaly_rcp85_1995-2100.nc"] }, "MIROC-ESM-CHEM": { - "RCP26": [ - "AIS/Atmosphere_Forcing/miroc-esm-chem_rcp2.6/Regridded_8km/MIROC-ESM-CHEM_8km_anomaly_rcp26_1995-2100.nc"], "RCP85": [ "AIS/Atmosphere_Forcing/miroc-esm-chem_rcp8.5/Regridded_8km/MIROC-ESM-CHEM_8km_anomaly_1995-2100.nc"] }, @@ -287,44 +299,38 @@ def correct_SMB_anomaly_for_baseSMB(self, output_file, mali_mesh_file): "2300": { "CCSM4": { "RCP85": [ - "AIS/Atmospheric_forcing/CCSM4_RCP85/Regridded_08km/CCSM4_8km_anomaly_1995-2100.nc", - "AIS/Atmospheric_forcing/CCSM4_RCP85/Regridded_08km/CCSM4_8km_anomaly_2101-2300.nc"] + "AIS/Atmospheric_Forcing/CCSM4_RCP85/Regridded_08km/CCSM4_8km_anomaly_1995-2100.nc", + "AIS/Atmospheric_Forcing/CCSM4_RCP85/Regridded_08km/CCSM4_8km_anomaly_2101-2300.nc"] }, "CESM2-WACCM": { "SSP585": [ - "AIS/Atmospheric_forcing/CESM2-WACCM_ssp585/Regridded_8km/CESM2-WACCM_8km_anomaly_ssp585_1995-2100.nc", - "AIS/Atmospheric_forcing/CESM2-WACCM_ssp585/Regridded_8km/CESM2-WACCM_8km_anomaly_ssp585_2101-2299.nc"], + "AIS/Atmospheric_Forcing/CESM2-WACCM_ssp585/Regridded_8km/CESM2-WACCM_8km_anomaly_ssp585_1995-2100.nc", + "AIS/Atmospheric_Forcing/CESM2-WACCM_ssp585/Regridded_8km/CESM2-WACCM_8km_anomaly_ssp585_2101-2299.nc"], "SSP585-repeat": [ - "AIS/Atmospheric_forcing/CESM2-WACCM_ssp585-repeat/Regridded_8km/CESM2-WACCM_8km_anomaly_ssp585_1995-2300.nc"] - }, - "CSIRO-Mk3-6-0": { - "RCP85": [ - "AIS/Atmospheric_forcing/CSIRO-Mk3-6-0_RCP85/Regridded_8km/CSIRO-Mk3-6-0_8km_anomaly_rcp85_1995-2100.nc", - "AIS/Atmospheric_forcing/CSIRO-Mk3-6-0_RCP85/Regridded_8km/CSIRO-Mk3-6-0_8km_anomaly_rcp85_2101-2300.nc"] + "AIS/Atmospheric_Forcing/CESM2-WACCM_ssp585-repeat/Regridded_8km/CESM2-WACCM_8km_anomaly_ssp585_1995-2300.nc"] }, "HadGEM2-ES": { "RCP85": [ - "AIS/Atmospheric_forcing/HadGEM2-ES_RCP85/Regridded_8km/HadGEM2-ES_8km_anomaly_rcp85_1995-2100.nc", - "AIS/Atmospheric_forcing/HadGEM2-ES_RCP85/Regridded_8km/HadGEM2-ES_8km_anomaly_rcp85_2101-2299.nc"], + "AIS/Atmospheric_Forcing/HadGEM2-ES_RCP85/Regridded_8km/HadGEM2-ES_8km_anomaly_rcp85_1995-2100.nc", + "AIS/Atmospheric_Forcing/HadGEM2-ES_RCP85/Regridded_8km/HadGEM2-ES_8km_anomaly_rcp85_2101-2299.nc"], "RCP85-repeat": [ - "AIS/Atmospheric_forcing/HadGEM2-ES-RCP85-repeat/Regridded_8km/HadGEM2-ES_8km_anomaly_rcp85_1995-2300.nc"] + "AIS/Atmospheric_Forcing/HadGEM2-ES-RCP85-repeat/Regridded_8km/HadGEM2-ES_8km_anomaly_rcp85_1995-2300.nc"] }, "NorESM1-M": { "RCP26-repeat": [ - "AIS/Atmospheric_forcing/NorESM1-M_RCP26-repeat/Regridded_8km/NorESM1-M_8km_anomaly_rcp26_1995-2300.nc"], + "AIS/Atmospheric_Forcing/NorESM1-M_RCP26-repeat/Regridded_8km/NorESM1-M_8km_anomaly_rcp26_1995-2300.nc"], "RCP85-repeat": [ - "AIS/Atmospheric_forcing/NorESM1-M_RCP85-repeat/Regridded_8km/NorESM1-M_anomaly_1995-2300.nc"] - + "AIS/Atmospheric_Forcing/NorESM1-M_RCP85-repeat/Regridded_8km/NorESM1-M_anomaly_1995-2300.nc"] }, "UKESM1-0-LL": { "SSP126": [ - "AIS/Atmospheric_forcing/UKESM1-0-LL_ssp126/Regridded_8km/UKESM1-0-LL_8km_anomaly_ssp126_1995-2100.nc" - "AIS/Atmospheric_forcing/UKESM1-0-LL_ssp126/Regridded_8km/UKESM1-0-LL_8km_anomaly_ssp126_2101-2300.nc"], + "AIS/Atmospheric_Forcing/UKESM1-0-LL_ssp126/Regridded_8km/UKESM1-0-LL_8km_anomaly_ssp126_1995-2100.nc" + "AIS/Atmospheric_Forcing/UKESM1-0-LL_ssp126/Regridded_8km/UKESM1-0-LL_8km_anomaly_ssp126_2101-2300.nc"], "SSP585": [ - "AIS/Atmospheric_forcing/UKESM1-0-LL_ssp585/Regridding_8km/UKESM1-0-LL_8km_anomaly_ssp585_1995-2100.nc", - "AIS/Atmospheric_forcing/UKESM1-0-LL_ssp585/Regridding_8km/UKESM1-0-LL_8km_anomaly_ssp585_2101-2300.nc"], + "AIS/Atmospheric_Forcing/UKESM1-0-LL_ssp585/Regridding_8km/UKESM1-0-LL_8km_anomaly_ssp585_1995-2100.nc", + "AIS/Atmospheric_Forcing/UKESM1-0-LL_ssp585/Regridding_8km/UKESM1-0-LL_8km_anomaly_ssp585_2101-2300.nc"], "SSP585-repeat": [ - "AIS/Atmospheric_forcing/UKESM1-0-LL_ssp585-repeat/Regridding_8km/UKESM1-0-LL_8km_anomaly_ssp585_1995-2300.nc"] + "AIS/Atmospheric_Forcing/UKESM1-0-LL_ssp585-repeat/Regridding_8km/UKESM1-0-LL_8km_anomaly_ssp585_1995-2300.nc"] } } } diff --git a/compass/landice/tests/ismip6_forcing/ocean_thermal/process_thermal_forcing.py b/compass/landice/tests/ismip6_forcing/ocean_thermal/process_thermal_forcing.py index a831b74981..704376eccf 100644 --- a/compass/landice/tests/ismip6_forcing/ocean_thermal/process_thermal_forcing.py +++ b/compass/landice/tests/ismip6_forcing/ocean_thermal/process_thermal_forcing.py @@ -1,4 +1,5 @@ import os +import shutil import subprocess import xarray as xr from compass.landice.tests.ismip6_forcing.ocean_thermal.create_mapfile \ @@ -78,6 +79,7 @@ def run(self): section = config['ismip6_ais_ocean_thermal'] method_remap = section.get('method_remap') + output_path = section.get('output_path') input_file_list = self._files[period_endyear][model][scenario] input_file = os.path.basename(input_file_list[0]) @@ -104,6 +106,18 @@ def run(self): # remove the temporary combined file os.remove(remapped_file_temp) + # place the output file in appropriate directory + if output_path == "NotAvailable": + pass + else: + if not os.path.exists(output_path): + print("Creating a new directory for the output data") + os.makedirs(output_path) + + src = os.path.join(os.getcwd(), output_file) + dst = os.path.join(output_path, output_file) + shutil.copy(src, dst) + def remap_ismip6thermalforcing_to_mali(self, input_file, output_file, mali_mesh_name, mali_mesh_file, method_remap): @@ -198,16 +212,13 @@ def rename_ismip6thermalforcing_to_mali_vars(self, remapped_file_temp, _files = { "2100": { "CCSM4": { - # "RCP26": ["AIS/Atmosphere_forcing/ccsm4_rcp2.6/Regridded_8km/CCSM4_8km_anomaly_rcp26_1995-2100.nc"], "RCP85": ["AIS/Ocean_Forcing/ccsm4_rcp8.5/1995-2100/CCSM4_thermal_forcing_8km_x_60m.nc"] }, "CESM2": { - "SSP585": [ - "AIS/Ocean_Forcing/cesm2_ssp585/1995-2100/CESM2_ssp585_thermal_forcing_8km_x_60m.nc"] - # "SSP585v1": [ - # "AIS/Atmosphere_Forcing/CESM2_ssp585/Regridded_8km/CESM2_anomaly_ssp585_1995-2100_8km_v1.nc"], - # "SSP585v2": [ - # "AIS/Atmosphere_forcing/CESM2_ssp585/Regridded_8km/CESM2_anomaly_ssp585_1995-2100_8km_v2.nc"] + "SSP585v1": [ + "AIS/Ocean_Forcing/cesm2_ssp585/1995-2100/CESM2_ssp585_thermal_forcing_8km_x_60m.nc"], + "SSP585v2": [ + "AIS/Ocean_Forcing/cesm2_ssp585/1995-2100/CESM2_ssp585_thermal_forcing_8km_x_60m.nc"], }, "CNRM_CM6": { "SSP126": [ @@ -234,8 +245,6 @@ def rename_ismip6thermalforcing_to_mali_vars(self, remapped_file_temp, "AIS/Ocean_Forcing/ipsl-cm5a-mr_rcp8.5/1995-2100/IPSL-CM5A-MR_RCP85_thermal_forcing_8km_x_60m.nc"] }, "MIROC-ESM-CHEM": { - # "RCP26": [ - # "AIS/Atmosphere_Forcing/miroc-esm-chem_rcp2.6/Regridded_8km/MIROC-ESM-CHEM_8km_anomaly_rcp26_1995-2100.nc"], "RCP85": [ "AIS/Ocean_Forcing/miroc-esm-chem_rcp8.5/1995-2100/MIROC-ESM-CHEM_thermal_forcing_8km_x_60m.nc"] }, @@ -253,38 +262,33 @@ def rename_ismip6thermalforcing_to_mali_vars(self, remapped_file_temp, "2300": { "CCSM4": { "RCP85": [ - "AIS/Ocean_forcing/ccsm4_RCP85/1995-2300/CCSM4_RCP85_thermal_forcing_8km_x_60m.nc"] + "AIS/Ocean_Forcing/ccsm4_RCP85/1995-2300/CCSM4_RCP85_thermal_forcing_8km_x_60m.nc"] }, "CESM2-WACCM": { "SSP585":[ - "AIS/Ocean_forcing/cesm2-waccm_ssp585/1995-2299/CESM2-WACCM_SSP585_thermal_forcing_8km_x_60m.nc"], + "AIS/Ocean_Forcing/cesm2-waccm_ssp585/1995-2299/CESM2-WACCM_SSP585_thermal_forcing_8km_x_60m.nc"], "SSP585-repeat": [ - "AIS/Ocean_forcing/cesm2-waccm_ssp585-repeat/1995-2300/CESM2-WACCM_ssp585_thermal_forcing_8km_x_60m.nc"] + "AIS/Ocean_Forcing/cesm2-waccm_ssp585-repeat/1995-2300/CESM2-WACCM_ssp585_thermal_forcing_8km_x_60m.nc"] }, - # "CSIRO-Mk3-6-0": { - # "RCP85": [ - # "AIS/Atmospheric_forcing/CSIRO-Mk3-6-0_RCP85/Regridded_8km/CSIRO-Mk3-6-0_8km_anomaly_rcp85_1995-2100.nc", - # "AIS/Atmospheric_forcing/CSIRO-Mk3-6-0_RCP85/Regridded_8km/CSIRO-Mk3-6-0_8km_anomaly_rcp85_2101-2300.nc"] -# }, "HadGEM2-ES": { "RCP85": [ - "AIS/Ocean_forcing/hadgem2-es_RCP85/1995-2299/HadGEM2-ES_RCP85_thermal_forcing_8km_x_60m.nc"], + "AIS/Ocean_Forcing/hadgem2-es_RCP85/1995-2299/HadGEM2-ES_RCP85_thermal_forcing_8km_x_60m.nc"], "RCP85-repeat": [ - "AIS/Ocean_forcing/hadgem2-es_RCP85-repeat/1995-2300/HadGEM2-ES_rcp85_thermal_forcing_8km_x_60m.nc"] + "AIS/Ocean_Forcing/hadgem2-es_RCP85-repeat/1995-2300/HadGEM2-ES_rcp85_thermal_forcing_8km_x_60m.nc"] }, "NorESM1-M": { "RCP26-repeat": [ - "AIS/Ocean_forcing/noresm1-m_RCP26-repeat/1995-2300/NorESM1-M_RCP26_thermal_forcing_8km_x_60m.nc"], + "AIS/Ocean_Forcing/noresm1-m_RCP26-repeat/1995-2300/NorESM1-M_RCP26_thermal_forcing_8km_x_60m.nc"], "RCP85-repeat": [ - "AIS/Ocean_forcing/noresm1-m_RCP85-repeat/1995-2300/NorESM1-M_thermal_forcing_8km_x_60m.nc"] + "AIS/Ocean_Forcing/noresm1-m_RCP85-repeat/1995-2300/NorESM1-M_thermal_forcing_8km_x_60m.nc"] }, "UKESM1-0-LL": { "SSP126": [ - "AIS/Ocean_forcing/ukesm1-0-ll_ssp126/1995-2300/UKESM1-0-LL_thermal_forcing_8km_x_60m.nc"], + "AIS/Ocean_Forcing/ukesm1-0-ll_ssp126/1995-2300/UKESM1-0-LL_thermal_forcing_8km_x_60m.nc"], "SSP585": [ - "AIS/Ocean_forcing/ukesm1-0-ll_ssp585/1995-2300/UKESM1-0-LL_SSP585_thermal_forcing_8km_x_60m.nc"], + "AIS/Ocean_Forcing/ukesm1-0-ll_ssp585/1995-2300/UKESM1-0-LL_SSP585_thermal_forcing_8km_x_60m.nc"], "SSP585-repeat": [ - "AIS/Ocean_forcing/ukesm1-0-ll_ssp585-repeat/1995-2300/UKESM1-0-LL_ssp585_thermal_forcing_8km_x_60m.nc"] + "AIS/Ocean_Forcing/ukesm1-0-ll_ssp585-repeat/1995-2300/UKESM1-0-LL_ssp585_thermal_forcing_8km_x_60m.nc"] } } } From 2663c7f1429c2e1bbbcb7ae1639887560db1f8f0 Mon Sep 17 00:00:00 2001 From: hollyhan Date: Tue, 10 May 2022 11:53:28 -0600 Subject: [PATCH 15/58] Fix PEP8 issues --- compass/landice/tests/ismip6_forcing/__init__.py | 2 +- .../tests/ismip6_forcing/atmosphere/__init__.py | 8 ++++---- .../tests/ismip6_forcing/atmosphere/process_smb.py | 2 +- .../tests/ismip6_forcing/ocean_basal/__init__.py | 8 ++++---- .../ismip6_forcing/ocean_basal/process_basal_melt.py | 2 +- .../tests/ismip6_forcing/ocean_thermal/__init__.py | 12 ++++++------ .../ocean_thermal/process_thermal_forcing.py | 6 ++---- 7 files changed, 19 insertions(+), 21 deletions(-) diff --git a/compass/landice/tests/ismip6_forcing/__init__.py b/compass/landice/tests/ismip6_forcing/__init__.py index fab34d27a0..3272adee26 100644 --- a/compass/landice/tests/ismip6_forcing/__init__.py +++ b/compass/landice/tests/ismip6_forcing/__init__.py @@ -4,12 +4,12 @@ from compass.landice.tests.ismip6_forcing.ocean_basal import OceanBasal - class Ismip6Forcing(TestGroup): """ A test group for processing the ISMIP6 ocean and atmosphere forcing data """ + def __init__(self, mpas_core): """ mpas_core : compass.landice.Landice diff --git a/compass/landice/tests/ismip6_forcing/atmosphere/__init__.py b/compass/landice/tests/ismip6_forcing/atmosphere/__init__.py index c1af8f0d9d..f8d4eb1ad5 100644 --- a/compass/landice/tests/ismip6_forcing/atmosphere/__init__.py +++ b/compass/landice/tests/ismip6_forcing/atmosphere/__init__.py @@ -37,13 +37,13 @@ def configure(self): Configures test case """ base_path_ismip6 = self.config.get(section="ismip6_ais", - option="base_path_ismip6") + option="base_path_ismip6") base_path_mali = self.config.get(section="ismip6_ais", - option="base_path_mali") + option="base_path_mali") mali_mesh_name = self.config.get(section="ismip6_ais", - option="mali_mesh_name") + option="mali_mesh_name") mali_mesh_file = self.config.get(section="ismip6_ais", - option="mali_mesh_file") + option="mali_mesh_file") if base_path_ismip6 == "NotAvailable": raise ValueError("You need to supply a user config file, which " diff --git a/compass/landice/tests/ismip6_forcing/atmosphere/process_smb.py b/compass/landice/tests/ismip6_forcing/atmosphere/process_smb.py index 9e520e8d35..4d5c280460 100644 --- a/compass/landice/tests/ismip6_forcing/atmosphere/process_smb.py +++ b/compass/landice/tests/ismip6_forcing/atmosphere/process_smb.py @@ -238,7 +238,7 @@ def correct_SMB_anomaly_for_baseSMB(self, output_file, mali_mesh_file): ds_base = xr.open_dataset(mali_mesh_file) for i in range(len(ds["sfcMassBal"])): ds["sfcMassBal"][i] = ds["sfcMassBal"][i] \ - + ds_base["sfcMassBal"][0,:] + + ds_base["sfcMassBal"][0, :] # write to a new netCDF file write_netcdf(ds, output_file) diff --git a/compass/landice/tests/ismip6_forcing/ocean_basal/__init__.py b/compass/landice/tests/ismip6_forcing/ocean_basal/__init__.py index e75a5821d6..3a46b7eea9 100644 --- a/compass/landice/tests/ismip6_forcing/ocean_basal/__init__.py +++ b/compass/landice/tests/ismip6_forcing/ocean_basal/__init__.py @@ -31,13 +31,13 @@ def configure(self): Configures test case """ base_path_ismip6 = self.config.get(section="ismip6_ais", - option="base_path_ismip6") + option="base_path_ismip6") base_path_mali = self.config.get(section="ismip6_ais", - option="base_path_mali") + option="base_path_mali") mali_mesh_name = self.config.get(section="ismip6_ais", - option="mali_mesh_name") + option="mali_mesh_name") mali_mesh_file = self.config.get(section="ismip6_ais", - option="mali_mesh_file") + option="mali_mesh_file") if base_path_ismip6 == "NotAvailable": raise ValueError("You need to supply a user config file, which " diff --git a/compass/landice/tests/ismip6_forcing/ocean_basal/process_basal_melt.py b/compass/landice/tests/ismip6_forcing/ocean_basal/process_basal_melt.py index defb33412d..fb9c0ad34e 100644 --- a/compass/landice/tests/ismip6_forcing/ocean_basal/process_basal_melt.py +++ b/compass/landice/tests/ismip6_forcing/ocean_basal/process_basal_melt.py @@ -227,4 +227,4 @@ def rename_ismip6BasalMelt_to_mali_vars(self, remapped_file_temp, "AIS/Ocean_Forcing/parametrizations/coeff_gamma0_DeltaT_quadratic_non_local_95th_pct_PIGL_gamma_calibration.nc", "AIS/Ocean_Forcing/parametrizations/coeff_gamma0_DeltaT_quadratic_non_local_95th_percentile.nc", "AIS/Ocean_Forcing/parametrizations/coeff_gamma0_DeltaT_quadratic_non_local_median_PIGL_gamma_calibration.nc", - "AIS/Ocean_Forcing/parametrizations/coeff_gamma0_DeltaT_quadratic_non_local_median.nc"} \ No newline at end of file + "AIS/Ocean_Forcing/parametrizations/coeff_gamma0_DeltaT_quadratic_non_local_median.nc"} diff --git a/compass/landice/tests/ismip6_forcing/ocean_thermal/__init__.py b/compass/landice/tests/ismip6_forcing/ocean_thermal/__init__.py index 582d0fdcf8..0e88257996 100644 --- a/compass/landice/tests/ismip6_forcing/ocean_thermal/__init__.py +++ b/compass/landice/tests/ismip6_forcing/ocean_thermal/__init__.py @@ -1,6 +1,6 @@ from compass.testcase import TestCase -from compass.landice.tests.ismip6_forcing.ocean_thermal.process_thermal_forcing\ - import ProcessThermalForcing +from compass.landice.tests.ismip6_forcing.ocean_thermal.\ + process_thermal_forcing import ProcessThermalForcing class OceanThermal(TestCase): @@ -31,13 +31,13 @@ def configure(self): Configures test case """ base_path_ismip6 = self.config.get(section="ismip6_ais", - option="base_path_ismip6") + option="base_path_ismip6") base_path_mali = self.config.get(section="ismip6_ais", - option="base_path_mali") + option="base_path_mali") mali_mesh_name = self.config.get(section="ismip6_ais", - option="mali_mesh_name") + option="mali_mesh_name") mali_mesh_file = self.config.get(section="ismip6_ais", - option="mali_mesh_file") + option="mali_mesh_file") if base_path_ismip6 == "NotAvailable": raise ValueError("You need to supply a user config file, which " diff --git a/compass/landice/tests/ismip6_forcing/ocean_thermal/process_thermal_forcing.py b/compass/landice/tests/ismip6_forcing/ocean_thermal/process_thermal_forcing.py index 704376eccf..98807aad92 100644 --- a/compass/landice/tests/ismip6_forcing/ocean_thermal/process_thermal_forcing.py +++ b/compass/landice/tests/ismip6_forcing/ocean_thermal/process_thermal_forcing.py @@ -54,7 +54,6 @@ def setup(self): target=os.path.join(base_path_mali, mali_mesh_file)) - input_file_list = self._files[period_endyear][model][scenario] for file in input_file_list: self.add_input_file(filename=os.path.basename(file), @@ -207,7 +206,6 @@ def rename_ismip6thermalforcing_to_mali_vars(self, remapped_file_temp, write_netcdf(ds, output_file) ds.close() - # create a nested dictionary for the ISMIP6 original forcing files including relative path _files = { "2100": { @@ -255,7 +253,7 @@ def rename_ismip6thermalforcing_to_mali_vars(self, remapped_file_temp, "AIS/Ocean_Forcing/noresm1-m_rcp8.5/1995-2100/NorESM1-M_thermal_forcing_8km_x_60m.nc"] }, "UKESM1-0-LL": { - "SSP585":[ + "SSP585": [ "AIS/Ocean_Forcing/ukesm1-0-ll_ssp585/1995-2100/UKESM1-0-LL_ssp585_thermal_forcing_8km_x_60m.nc"] } }, @@ -265,7 +263,7 @@ def rename_ismip6thermalforcing_to_mali_vars(self, remapped_file_temp, "AIS/Ocean_Forcing/ccsm4_RCP85/1995-2300/CCSM4_RCP85_thermal_forcing_8km_x_60m.nc"] }, "CESM2-WACCM": { - "SSP585":[ + "SSP585": [ "AIS/Ocean_Forcing/cesm2-waccm_ssp585/1995-2299/CESM2-WACCM_SSP585_thermal_forcing_8km_x_60m.nc"], "SSP585-repeat": [ "AIS/Ocean_Forcing/cesm2-waccm_ssp585-repeat/1995-2300/CESM2-WACCM_ssp585_thermal_forcing_8km_x_60m.nc"] From 2d0111eb4b09c1b10f87f3de7fac813fc6c1e3c6 Mon Sep 17 00:00:00 2001 From: hollyhan Date: Wed, 11 May 2022 10:54:01 -0600 Subject: [PATCH 16/58] Edit config file and docs --- .../landice/tests/ismip6_forcing/ismip6.cfg | 83 +++++++------------ .../landice/test_groups/ismip6_forcing.rst | 4 +- .../landice/test_groups/ismip6_forcing.rst | 20 ++--- 3 files changed, 44 insertions(+), 63 deletions(-) diff --git a/compass/landice/tests/ismip6_forcing/ismip6.cfg b/compass/landice/tests/ismip6_forcing/ismip6.cfg index e2bce833ac..c95140a0e6 100644 --- a/compass/landice/tests/ismip6_forcing/ismip6.cfg +++ b/compass/landice/tests/ismip6_forcing/ismip6.cfg @@ -1,75 +1,56 @@ -# This file contains some common config options you might want to set -# if you're working with the compass landice core and MALI. +# config options for ismip6 antarctic ice sheet data set +[ismip6_ais] -# The paths section describes paths that are used within the landice core test -# cases. -[paths] +# Base path to the input ismip6 ocean and smb forcing files. User has to supply. +base_path_ismip6 = NotAvailable -# The root to a location where data files for MALI will be cached -landice_database_root = /Users/hollyhan/Desktop/RESEARCH/MALI/MALI_Database +# Base path to the the MALI mesh. User has to supply. +base_path_mali = NotAvailable +# Forcing end year of the ISMIP6 data. User has to supply. +# Available end years are 2100 and 2300. +period_endyear = NotAvailable -# The parallel section describes options related to running tests in parallel -[parallel] +# Name of climate model name used to generate ISMIP6 forcing data. User has to supply. +# Available model names are the following: CCSM4, CESM2-WACCM, CSIRO-Mk3-6-0, HadGEM2-ES, NorESM1-M, UKESM1-0-LL +model = NotAvailable -# parallel system of execution: slurm or single_node -system = single_node +# Scenarios used by climate model. User has to supply. +# Available scenarios are the following: RCP26, RCP85, SSP126, SSP585 +scenario = NotAvailable -# whether to use mpirun or srun to run the model -parallel_executable = mpirun -host localhost +# name of the mali mesh used to name mapping files (e,g. Antarctica_8to80km) +mali_mesh_name = NotAvailable -# cores per node on the machine -cores_per_node = 8 +# MALI mesh file to be used to build mapping file (e.g.Antarctic_8to80km_20220407.nc) +mali_mesh_file = NotAvailable -# the number of multiprocessing or dask threads to use -threads = 8 # config options for ismip6 antarctic ice sheet SMB forcing data test cases [ismip6_ais_atmosphere] -# Base path to the input ismip6 atmosphere data files. User has to supply. -input_path = /Users/hollyhan/Desktop/ISMIP6_tempfile/ - -# input surface mass balance file -input_file = NorESM1-M_8km_anomaly_1950-1994.nc - -# name of the mali mesh used to name mapping files -mali_mesh_name = Antarctica_8to80km - -# MALI mesh file to be used to build mapping file -mali_mesh_file = Antarctic_8to80km_20220407.nc - # Remapping method used in building a mapping file. Options include: bilinear, neareststod, conserve method_remap = bilinear -# name of the processed ISMIP6 ocean data on MALI mesh -output_file = output_smb.nc +# Path to which processed output SMB data is saved. User has to supply. +output_path = NotAvailable -# config options for ismip6 ocean data test cases -[ismip6_ais_ocean] -# Base path to the input ismip6 ocean data files -input_path = /Users/hollyhan/Desktop/ISMIP6_tempfile/ +# config options for ismip6 ocean thermal forcing data test cases +[ismip6_ais_ocean_thermal] -# input thermal forcing file -thermal_forcing_file = NorESM1-M_thermal_forcing_8km_x_60m.nc - -# input imbie2 basin number file -basin_file = imbie2_basin_numbers_8km.nc +# Remapping method used in building a mapping file. Options include: bilinear, neareststod, conserve +method_remap = bilinear -# input uniform melt rate coefficient (gamma0) -# and temperature correction per basin file -coeff_file = coeff_gamma0_DeltaT_quadratic_non_local_median.nc +# Path to which processed output thermal forcing data is saved. User has to supply. +output_path = NotAvailable -# name of the mali mesh used to name mapping files -mali_mesh_name = Antarctica_8to80km -# MALI mesh file to be used to build mapping file -mali_mesh_file = Antarctic_8to80km_20220407.nc +# config options for ismip6 ocean basal test case +[ismip6_ais_ocean_basal] # Remapping method used in building a mapping file. Options include: bilinear, neareststod, conserve -method_remap = bilinear +method_remap = neareststod -# name of the processed ISMIP6 ocean data on MALI mesh -output_thermalforcing_file = output_thermalforcing.nc -output_basalmelt_file = output_basalmelt.nc \ No newline at end of file +# Path to which processed output basal param coeff data is saved. User has to supply. +output_path = NotAvailable diff --git a/docs/developers_guide/landice/test_groups/ismip6_forcing.rst b/docs/developers_guide/landice/test_groups/ismip6_forcing.rst index 4907493f0c..2f9b58d9cc 100644 --- a/docs/developers_guide/landice/test_groups/ismip6_forcing.rst +++ b/docs/developers_guide/landice/test_groups/ismip6_forcing.rst @@ -28,7 +28,7 @@ Processing data includes regridding the original ISMIP6 SMB data from its native polarstereo grid to MALI's unstructured grid, renaming variables and correcting the SMB anomaly field for the MALI base SMB. -.. _dev_landice_ocean_thermal: +.. _dev_landice_ismip6_forcing_ocean_thermal: ocean_thermal ------------- @@ -38,7 +38,7 @@ performs the processing of ocean thermal forcing. Processing data includes regridding the original ISMIP6 thermal forcing data from its native polarstereo grid to MALI's unstructured grid and renaming variables. -.. _dev_landice_ocean_basal: +.. _dev_landice_ismip6_forcing_ocean_basal: ocean_basal ------------ diff --git a/docs/users_guide/landice/test_groups/ismip6_forcing.rst b/docs/users_guide/landice/test_groups/ismip6_forcing.rst index d59de33db6..911b47f2d1 100644 --- a/docs/users_guide/landice/test_groups/ismip6_forcing.rst +++ b/docs/users_guide/landice/test_groups/ismip6_forcing.rst @@ -29,28 +29,28 @@ All three test cases share some set of default config options under the section [ismip6_ais] # Base path to the input ismip6 ocean and smb forcing files. User has to supply. - base_path_ismip6 = NotAvailable + base_path_ismip6 = /Users/hollyhan/Desktop/Data/ISMIP6-Projections-Forcing-2300/ # Base path to the the MALI mesh. User has to supply. - base_path_mali = NotAvailable + base_path_mali = /Users/hollyhan/Desktop/Data/MALI-mesh/ # Forcing end year of the ISMIP6 data. User has to supply. # Available end years are 2100 and 2300. - period_endyear = NotAvailable + period_endyear = 2300 # Name of climate model name used to generate ISMIP6 forcing data. User has to supply. # Available model names are the following: CCSM4, CESM2-WACCM, CSIRO-Mk3-6-0, HadGEM2-ES, NorESM1-M, UKESM1-0-LL - model = NotAvailable + model = NorESM1-M # Scenarios used by climate model. User has to supply. # Available scenarios are the following: RCP26, RCP85, SSP126, SSP585 - scenario = NotAvailable + scenario = RCP26-repeat # name of the mali mesh used to name mapping files (e,g. Antarctica_8to80km). User has to supply. - mali_mesh_name = NotAvailable + mali_mesh_name = Antarctica_8to80km # MALI mesh file to be used to build mapping file (netCDF format). User has to supply. - mali_mesh_file = NotAvailable + mali_mesh_file = Antarctica_8to80km_20220407.nc # config options for ismip6 antarctic ice sheet SMB forcing data test cases @@ -60,7 +60,7 @@ All three test cases share some set of default config options under the section method_remap = bilinear # Path to which processed output SMB data is saved. User has to supply. - output_path = NotAvailable + output_path = /Users/hollyhan/Desktop/Data/ISMIP6-Projections-Forcing-2300/AIS_Processed/Atmospheric_Forcing/ # config options for ismip6 ocean thermal forcing data test cases @@ -70,7 +70,7 @@ All three test cases share some set of default config options under the section method_remap = bilinear # Path to which processed output thermal forcing data is saved. User has to supply. - output_path = NotAvailable + output_path = /Users/hollyhan/Desktop/Data/ISMIP6-Projections-Forcing-2300/AIS_Processed/Ocean_Forcing/ # config options for ismip6 ocean basal test case @@ -80,7 +80,7 @@ All three test cases share some set of default config options under the section method_remap = neareststod # Path to which processed output basal param coeff data is saved. User has to supply. - output_path = NotAvailable + output_path = /Users/hollyhan/Desktop/Data/ISMIP6-Projections-Forcing-2300/AIS_Processed/Ocean_Forcing/parametrizations/ atmosphere From 8b85f32739d270406037ac1c9badf4088df277f6 Mon Sep 17 00:00:00 2001 From: hollyhan Date: Fri, 13 May 2022 11:55:34 -0700 Subject: [PATCH 17/58] Restructure base path of output forcing data (atmosphere testcase) --- .../ismip6_forcing/atmosphere/process_smb.py | 24 +++++++++++-------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/compass/landice/tests/ismip6_forcing/atmosphere/process_smb.py b/compass/landice/tests/ismip6_forcing/atmosphere/process_smb.py index 4d5c280460..5fdf2ab877 100644 --- a/compass/landice/tests/ismip6_forcing/atmosphere/process_smb.py +++ b/compass/landice/tests/ismip6_forcing/atmosphere/process_smb.py @@ -80,10 +80,10 @@ def run(self): period_endyear = section.get("period_endyear") model = section.get("model") scenario = section.get("scenario") + output_base_path = section.get('output_base_path') section = config['ismip6_ais_atmosphere'] method_remap = section.get('method_remap') - output_path = section.get('output_path') input_file_list = self._files[period_endyear][model][scenario] i = 0 @@ -124,12 +124,14 @@ def run(self): os.remove(combined_file_temp) # place the output file in appropriate directory - if output_path == "NotAvailable": - pass - else: - if not os.path.exists(output_path): - print("Creating a new directory for the output data") - os.makedirs(output_path) + if output_base_path == "NotAvailable": + return + + output_path = f'{output_base_path}/atmosphere_forcing/' \ + f'{model}_{scenario}/1995-{period_endyear}' + if not os.path.exists(output_path): + print("Creating a new directory for the output data") + os.makedirs(output_path) src = os.path.join(os.getcwd(), output_file) dst = os.path.join(output_path, output_file) @@ -236,9 +238,11 @@ def correct_SMB_anomaly_for_baseSMB(self, output_file, mali_mesh_file): ds = xr.open_dataset(output_file) ds_base = xr.open_dataset(mali_mesh_file) - for i in range(len(ds["sfcMassBal"])): - ds["sfcMassBal"][i] = ds["sfcMassBal"][i] \ - + ds_base["sfcMassBal"][0, :] + # get the first time index + ref_smb = ds_base["sfcMassBal"].isel(Time=0) + # broadcast so time is the same size as in ds + #ref_smb = ref_smb.broadcast(ds["sfcMassBal"]) + ds["sfcMassBal"] = ds["sfcMassBal"] + ref_smb # write to a new netCDF file write_netcdf(ds, output_file) From d435c0bf848591bd131da4dd4e3aec70d6a76e00 Mon Sep 17 00:00:00 2001 From: hollyhan Date: Fri, 13 May 2022 11:31:57 -0700 Subject: [PATCH 18/58] Restructure base path of output forcing data --- .../ocean_basal/process_basal_melt.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/compass/landice/tests/ismip6_forcing/ocean_basal/process_basal_melt.py b/compass/landice/tests/ismip6_forcing/ocean_basal/process_basal_melt.py index fb9c0ad34e..aa9339a51c 100644 --- a/compass/landice/tests/ismip6_forcing/ocean_basal/process_basal_melt.py +++ b/compass/landice/tests/ismip6_forcing/ocean_basal/process_basal_melt.py @@ -59,10 +59,10 @@ def run(self): section = config['ismip6_ais'] mali_mesh_name = section.get('mali_mesh_name') mali_mesh_file = section.get('mali_mesh_file') + output_base_path = section.get('output_base_path') section = config['ismip6_ais_ocean_basal'] method_remap = section.get('method_remap') - output_path = section.get('output_path') # combine, interpolate and rename the basin file and deltaT0_gamma0 # ismip6 input files @@ -103,13 +103,13 @@ def run(self): os.remove(remapped_file_temp) # place the output file in appropriate directory - print(output_path) - if output_path == "NotAvailable": - pass - else: - if not os.path.exists(output_path): - print("Creating a new directory for the output data") - os.makedirs(output_path) + if output_base_path == "NotAvailable": + return + + output_path = f'{output_base_path}/basal_melt/parametrizations/' + if not os.path.exists(output_path): + print("Creating a new directory for the output data") + os.makedirs(output_path) src = os.path.join(os.getcwd(), output_file) dst = os.path.join(output_path, output_file) From e22b33a9e5cb1a4d2ff0cc51fcd0185c0e68a878 Mon Sep 17 00:00:00 2001 From: hollyhan Date: Fri, 13 May 2022 11:29:15 -0700 Subject: [PATCH 19/58] Add an option to process observed thermal forcing --- .../ocean_thermal/process_thermal_forcing.py | 87 ++++++++++++------- 1 file changed, 57 insertions(+), 30 deletions(-) diff --git a/compass/landice/tests/ismip6_forcing/ocean_thermal/process_thermal_forcing.py b/compass/landice/tests/ismip6_forcing/ocean_thermal/process_thermal_forcing.py index 98807aad92..3372b2a9a3 100644 --- a/compass/landice/tests/ismip6_forcing/ocean_thermal/process_thermal_forcing.py +++ b/compass/landice/tests/ismip6_forcing/ocean_thermal/process_thermal_forcing.py @@ -37,6 +37,9 @@ def setup(self): model = section.get("model") scenario = section.get("scenario") + section = config['ismip6_ais_ocean_thermal'] + process_obs_data = section.get("process_obs_data") + if period_endyear == "NotAvailable": raise ValueError("You need to supply a user config file, which " "should contain the ismip6_ais " @@ -54,12 +57,15 @@ def setup(self): target=os.path.join(base_path_mali, mali_mesh_file)) - input_file_list = self._files[period_endyear][model][scenario] - for file in input_file_list: - self.add_input_file(filename=os.path.basename(file), - target=os.path.join(base_path_ismip6, file)) + if process_obs_data == "True": + input_file = self._file_obs + output_file = f"processed_obs_TF_1995-2017_8km_x_60m.nc" + else: + input_file = self._files[period_endyear][model][scenario] + output_file = f"processed_TF_{model}_{scenario}_{period_endyear}.nc" - output_file = f"processed_TF_{model}_{scenario}_{period_endyear}.nc" + self.add_input_file(filename=os.path.basename(input_file[0]), + target=os.path.join(base_path_ismip6, input_file[0])) self.add_output_file(filename=output_file) def run(self): @@ -75,12 +81,23 @@ def run(self): period_endyear = section.get("period_endyear") model = section.get("model") scenario = section.get("scenario") + output_base_path = section.get('output_base_path') section = config['ismip6_ais_ocean_thermal'] method_remap = section.get('method_remap') - output_path = section.get('output_path') + process_obs_data = section.get('process_obs_data') + + if process_obs_data == "True": + input_file_list = self._file_obs + output_file = f'processed_obs_TF_1995-2017_8km_x_60m.nc' + output_path = f'{output_base_path}/ocean_thermal_forcing/'\ + f'obs' + else: + input_file_list = self._files[period_endyear][model][scenario] + output_file = f'processed_TF_{model}_{scenario}_{period_endyear}.nc' + output_path = f'{output_base_path}/ocean_thermal_forcing/' \ + f'{model}_{scenario}/1995-{period_endyear}' - input_file_list = self._files[period_endyear][model][scenario] input_file = os.path.basename(input_file_list[0]) # interpolate and rename the ismip6 thermal forcing data @@ -93,7 +110,6 @@ def run(self): mali_mesh_name, mali_mesh_file, method_remap) - output_file = f"processed_TF_{model}_{scenario}_{period_endyear}.nc" # call the function that renames the ismip6 variables to MALI variables print("Renaming the ismip6 variables to mali variable names...") self.rename_ismip6thermalforcing_to_mali_vars(remapped_file_temp, @@ -106,10 +122,10 @@ def run(self): os.remove(remapped_file_temp) # place the output file in appropriate directory - if output_path == "NotAvailable": - pass - else: - if not os.path.exists(output_path): + if output_base_path == "NotAvailable": + return + + if not os.path.exists(output_path): print("Creating a new directory for the output data") os.makedirs(output_path) @@ -169,6 +185,10 @@ def rename_ismip6thermalforcing_to_mali_vars(self, remapped_file_temp, remapped ismip6 data renamed on mali mesh """ + config = self.config + section = config['ismip6_ais_ocean_thermal'] + process_obs_data = section.get('process_obs_data') + # open dataset in 20 years chunk ds = xr.open_dataset(remapped_file_temp, chunks=dict(time=20), engine="netcdf4") @@ -177,36 +197,43 @@ def rename_ismip6thermalforcing_to_mali_vars(self, remapped_file_temp, ds = ds.drop_vars('z') # dropping 'z' while it's still called 'z' # build dictionary for ismip6 variables that MALI takes in - ismip6_to_mali_dims = dict( - z="nISMIP6OceanLayers", - time="Time", - ncol="nCells") - ds = ds.rename(ismip6_to_mali_dims) + if process_obs_data == "True": + ismip6_to_mali_dims = dict( + z="nISMIP6OceanLayers", + ncol="nCells") + ds = ds.rename(ismip6_to_mali_dims) + else: + ismip6_to_mali_dims = dict( + z="nISMIP6OceanLayers", + time="Time", + ncol="nCells") + ds = ds.rename(ismip6_to_mali_dims) + # add xtime variable + xtime = [] + for t_index in range(ds.sizes["Time"]): + date = ds.Time[t_index] + date = date.dt.strftime("%Y-%m-%d_00:00:00") + date = str(date.values).ljust(64) + xtime.append(date) + + ds["xtime"] = ("Time", xtime) + ds["xtime"] = ds.xtime.astype('S') + ds = ds.drop_vars(["Time"]) ismip6_to_mali_vars = dict( thermal_forcing="ismip6shelfMelt_3dThermalForcing") ds = ds.rename(ismip6_to_mali_vars) - # add xtime variable - xtime = [] - for t_index in range(ds.sizes["Time"]): - date = ds.Time[t_index] - date = date.dt.strftime("%Y-%m-%d_00:00:00") - date = str(date.values).ljust(64) - xtime.append(date) - - ds["xtime"] = ("Time", xtime) - ds["xtime"] = ds.xtime.astype('S') - # drop unnecessary variables - ds = ds.drop_vars(["z_bnds", "lat_vertices", "Time", - "lon_vertices", "lat", "lon", "area"]) + ds = ds.drop_vars(["z_bnds", "lat_vertices", "area", + "lon_vertices", "lat", "lon"]) # write to a new netCDF file write_netcdf(ds, output_file) ds.close() # create a nested dictionary for the ISMIP6 original forcing files including relative path + _file_obs = ["AIS/Ocean_Forcing/climatology_from_obs_1995-2017/obs_thermal_forcing_1995-2017_8km_x_60m.nc"] _files = { "2100": { "CCSM4": { From 22bb5d8fcd979ab4f97b4b05a832e0ab118df39a Mon Sep 17 00:00:00 2001 From: hollyhan Date: Fri, 13 May 2022 15:42:21 -0600 Subject: [PATCH 20/58] Address xarray and PEP8 issues --- .../landice/tests/ismip6_forcing/atmosphere/process_smb.py | 2 +- .../ismip6_forcing/ocean_thermal/process_thermal_forcing.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/compass/landice/tests/ismip6_forcing/atmosphere/process_smb.py b/compass/landice/tests/ismip6_forcing/atmosphere/process_smb.py index 5fdf2ab877..679f1b4c07 100644 --- a/compass/landice/tests/ismip6_forcing/atmosphere/process_smb.py +++ b/compass/landice/tests/ismip6_forcing/atmosphere/process_smb.py @@ -241,7 +241,7 @@ def correct_SMB_anomaly_for_baseSMB(self, output_file, mali_mesh_file): # get the first time index ref_smb = ds_base["sfcMassBal"].isel(Time=0) # broadcast so time is the same size as in ds - #ref_smb = ref_smb.broadcast(ds["sfcMassBal"]) + ref_smb = ref_smb.broadcast(ds["sfcMassBal"]) ds["sfcMassBal"] = ds["sfcMassBal"] + ref_smb # write to a new netCDF file diff --git a/compass/landice/tests/ismip6_forcing/ocean_thermal/process_thermal_forcing.py b/compass/landice/tests/ismip6_forcing/ocean_thermal/process_thermal_forcing.py index 3372b2a9a3..bf905232f7 100644 --- a/compass/landice/tests/ismip6_forcing/ocean_thermal/process_thermal_forcing.py +++ b/compass/landice/tests/ismip6_forcing/ocean_thermal/process_thermal_forcing.py @@ -126,8 +126,8 @@ def run(self): return if not os.path.exists(output_path): - print("Creating a new directory for the output data") - os.makedirs(output_path) + print("Creating a new directory for the output data") + os.makedirs(output_path) src = os.path.join(os.getcwd(), output_file) dst = os.path.join(output_path, output_file) From 0bac938750045739742dd41ca5568a06de032797 Mon Sep 17 00:00:00 2001 From: hollyhan Date: Thu, 26 May 2022 10:00:52 -0700 Subject: [PATCH 21/58] Fix doc string --- compass/landice/tests/ismip6_forcing/atmosphere/__init__.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/compass/landice/tests/ismip6_forcing/atmosphere/__init__.py b/compass/landice/tests/ismip6_forcing/atmosphere/__init__.py index f8d4eb1ad5..3ac9d53049 100644 --- a/compass/landice/tests/ismip6_forcing/atmosphere/__init__.py +++ b/compass/landice/tests/ismip6_forcing/atmosphere/__init__.py @@ -9,11 +9,6 @@ class Atmosphere(TestCase): The test case builds a mapping file for interpolation between the ISMIP6 8km polarstereo grid and MALI mesh, regrids the forcing data and rename the ISMIP6 variables to corresponding MALI variables. - - Attributes - ---------- - mesh_type : str >>>>>>> what kind of attributes would we have? - The resolution or type of mesh of the test case """ def __init__(self, test_group): From dd8fae3849296058006383e46efbc30ebd199b61 Mon Sep 17 00:00:00 2001 From: hollyhan Date: Thu, 26 May 2022 06:47:50 -0700 Subject: [PATCH 22/58] Move duplicate scripts from to test group folder --- .../{ocean_basal => }/create_mapfile.py | 0 .../ocean_basal/process_basal_melt.py | 2 +- .../ocean_thermal/create_mapfile.py | 83 ------------------- .../ocean_thermal/process_thermal_forcing.py | 2 +- 4 files changed, 2 insertions(+), 85 deletions(-) rename compass/landice/tests/ismip6_forcing/{ocean_basal => }/create_mapfile.py (100%) delete mode 100644 compass/landice/tests/ismip6_forcing/ocean_thermal/create_mapfile.py diff --git a/compass/landice/tests/ismip6_forcing/ocean_basal/create_mapfile.py b/compass/landice/tests/ismip6_forcing/create_mapfile.py similarity index 100% rename from compass/landice/tests/ismip6_forcing/ocean_basal/create_mapfile.py rename to compass/landice/tests/ismip6_forcing/create_mapfile.py diff --git a/compass/landice/tests/ismip6_forcing/ocean_basal/process_basal_melt.py b/compass/landice/tests/ismip6_forcing/ocean_basal/process_basal_melt.py index aa9339a51c..9e3a6cfef8 100644 --- a/compass/landice/tests/ismip6_forcing/ocean_basal/process_basal_melt.py +++ b/compass/landice/tests/ismip6_forcing/ocean_basal/process_basal_melt.py @@ -2,7 +2,7 @@ import shutil import subprocess import xarray as xr -from compass.landice.tests.ismip6_forcing.ocean_basal.create_mapfile \ +from compass.landice.tests.ismip6_forcing.create_mapfile \ import build_mapping_file from mpas_tools.io import write_netcdf from compass.step import Step diff --git a/compass/landice/tests/ismip6_forcing/ocean_thermal/create_mapfile.py b/compass/landice/tests/ismip6_forcing/ocean_thermal/create_mapfile.py deleted file mode 100644 index b5f86452a2..0000000000 --- a/compass/landice/tests/ismip6_forcing/ocean_thermal/create_mapfile.py +++ /dev/null @@ -1,83 +0,0 @@ -import os -import subprocess -from mpas_tools.scrip.from_mpas import scrip_from_mpas - - -# function that creates a mapping file from ismip6 grid to mali mesh -def build_mapping_file(ismip6_grid_file, mapping_file, mali_mesh_file=None, - method_remap=None): - """ - Build a mapping file if it does not exist. - - Parameters - ---------- - ismip6_grid_file : str - ismip6 grid file - mapping_file : str - weights for interpolation from ismip6_grid_file to mali_mesh_file - mali_mesh_file : str, optional - The MALI mesh file if mapping file does not exist - method_remap : str, optional - Remapping method used in building a mapping file - """ - - if os.path.exists(mapping_file): - print("Mapping file exists. Not building a new one.") - return - - if mali_mesh_file is None: - raise ValueError("Mapping file does not exist. To build one, Mali " - "mesh file with '-f' should be provided. " - "Type --help for info") - - ismip6_scripfile = "temp_ismip6_8km_scrip.nc" - mali_scripfile = "temp_mali_scrip.nc" - ismip6_projection = "ais-bedmap2" - - # create the ismip6 scripfile if mapping file does not exist - # this is the projection of ismip6 data for Antarctica - print("Mapping file does not exist. Building one based on the " - "input/ouptut meshes") - print("Creating temporary scripfiles for ismip6 grid and mali mesh...") - - args = ["create_SCRIP_file_from_planar_rectangular_grid.py", - "--input", ismip6_grid_file, - "--scrip", ismip6_scripfile, - "--proj", ismip6_projection, - "--rank", "2"] - - subprocess.check_call(args) - - # make sure the input file uses the longitude convention of [0 2pi] - args = ["set_lat_lon_fields_in_planar_grid.py", - "--file", mali_mesh_file, - "--proj", ismip6_projection] - - subprocess.check_call(args) - - # create a MALI mesh scripfile if mapping file does not exist - scrip_from_mpas(mali_mesh_file, mali_scripfile) - - # create a mapping file using ESMF weight gen - print(f"Creating a mapping file. Mapping method used: {method_remap}") - - if method_remap is None: - raise ValueError("Desired remapping option should be provided with " - "--method. Available options are 'bilinear'," - "'neareststod', 'conserve'.") - - args = ["ESMF_RegridWeightGen", - "-s", ismip6_scripfile, - "-d", mali_scripfile, - "-w", mapping_file, - "-m", method_remap, - "-i", "-64bit_offset", - "--dst_regional", "--src_regional"] - - # include flag and input and output file names - subprocess.check_call(args) - - # remove the temporary scripfiles once the mapping file is generated - print("Removing the temporary scripfiles...") - os.remove(ismip6_scripfile) - os.remove(mali_scripfile) diff --git a/compass/landice/tests/ismip6_forcing/ocean_thermal/process_thermal_forcing.py b/compass/landice/tests/ismip6_forcing/ocean_thermal/process_thermal_forcing.py index bf905232f7..9b81dc809f 100644 --- a/compass/landice/tests/ismip6_forcing/ocean_thermal/process_thermal_forcing.py +++ b/compass/landice/tests/ismip6_forcing/ocean_thermal/process_thermal_forcing.py @@ -2,7 +2,7 @@ import shutil import subprocess import xarray as xr -from compass.landice.tests.ismip6_forcing.ocean_thermal.create_mapfile \ +from compass.landice.tests.ismip6_forcing.create_mapfile \ import build_mapping_file from mpas_tools.io import write_netcdf from compass.step import Step From 3f94c75fe87623d7a851d4a369abad2afe8fa76d Mon Sep 17 00:00:00 2001 From: hollyhan Date: Thu, 26 May 2022 10:22:04 -0700 Subject: [PATCH 23/58] Edit logging statement and remove xr.broadcast method --- .../tests/ismip6_forcing/atmosphere/process_smb.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/compass/landice/tests/ismip6_forcing/atmosphere/process_smb.py b/compass/landice/tests/ismip6_forcing/atmosphere/process_smb.py index 679f1b4c07..95b63888da 100644 --- a/compass/landice/tests/ismip6_forcing/atmosphere/process_smb.py +++ b/compass/landice/tests/ismip6_forcing/atmosphere/process_smb.py @@ -113,7 +113,8 @@ def run(self): self.rename_ismip6smb_to_mali_vars(remapped_file_temp, output_file) # correct the SMB anomaly field with mali base SMB field - logger.info("Correcting the SMB anomaly field with MALI base SMB...") + logger.info("Correcting the SMB anomaly field with MALI base SMB " + "from file {mali_mesh_file}...") self.correct_SMB_anomaly_for_baseSMB(output_file, mali_mesh_file) logger.info("Processing done successfully. " @@ -135,8 +136,6 @@ def run(self): src = os.path.join(os.getcwd(), output_file) dst = os.path.join(output_path, output_file) - print(src) - print(dst) shutil.copy(src, dst) def remap_ismip6smb_to_mali(self, input_file, output_file, mali_mesh_name, @@ -163,6 +162,8 @@ def remap_ismip6smb_to_mali(self, input_file, output_file, mali_mesh_name, if not os.path.exists(mapping_file): # build a mapping file if it doesn't already exist + self.logger.info("Creating a mapping file. " + "Mapping method used: {method_remap}") build_mapping_file(input_file, mapping_file, mali_mesh_file, method_remap) else: @@ -240,8 +241,7 @@ def correct_SMB_anomaly_for_baseSMB(self, output_file, mali_mesh_file): ds_base = xr.open_dataset(mali_mesh_file) # get the first time index ref_smb = ds_base["sfcMassBal"].isel(Time=0) - # broadcast so time is the same size as in ds - ref_smb = ref_smb.broadcast(ds["sfcMassBal"]) + # correct for the reference smb ds["sfcMassBal"] = ds["sfcMassBal"] + ref_smb # write to a new netCDF file @@ -249,6 +249,8 @@ def correct_SMB_anomaly_for_baseSMB(self, output_file, mali_mesh_file): ds.close() # create a nested dictionary for the ISMIP6 original forcing files including relative path + # Note: these files needed to be explicitly listed because of inconsistencies that are + # present in file naming conventions in the ISMIP6 source dataset. _files = { "2100": { "CCSM4": { From 4da36edd79eeacbb4d9ea8f1241e1763a428994f Mon Sep 17 00:00:00 2001 From: hollyhan Date: Thu, 26 May 2022 10:43:48 -0700 Subject: [PATCH 24/58] Fix users guide --- .../landice/test_groups/ismip6_forcing.rst | 32 +++++++++---------- 1 file changed, 15 insertions(+), 17 deletions(-) diff --git a/docs/users_guide/landice/test_groups/ismip6_forcing.rst b/docs/users_guide/landice/test_groups/ismip6_forcing.rst index 911b47f2d1..deff500ea5 100644 --- a/docs/users_guide/landice/test_groups/ismip6_forcing.rst +++ b/docs/users_guide/landice/test_groups/ismip6_forcing.rst @@ -1,11 +1,12 @@ .. _landice_ismip6_forcing: -hydro_radial +ismip6_forcing ============ The ``landice/ismip6_forcing`` test group implements processing of atmospheric and ocean forcing data of the Ice Sheet Model Intercomparison for CMIP6 -(ISMIP6) protocol. +(ISMIP6) protocol. The ISMIP6 source data can be obtained by directly +contacting the authors. Reference: https://www.climate-cryosphere.org/wiki/index.php?title=ISMIP6-Projections2300-Antarctica#A2.2_Retrieving_datasets_and_uploading_your_model_output The test group includes 3 test cases, ``atmosphere``, ``ocean_thermal`` and ``ocean_basal``. All test cases are made up of a single main step, @@ -15,7 +16,9 @@ renaming the original ismip6 data to the format that MALI can incorporate in its forward simulations. In remapping the data, all test cases import the method ``build_mapping_file`` to create or use scrip files of the source (ISMIP6) and destination (MALI) mesh depending on the existence -of a mapping file. +of a mapping file. Approximated time for processing a single forcing file +on Cori (single core) is 2, 1, and 5 minutes for the atmosphere, ocean basal, +and ocean thermal testcase, respectively. config options -------------- @@ -25,7 +28,7 @@ All three test cases share some set of default config options under the section .. code-block:: cfg - # config options for ismip6 antarctic ice sheet SMB forcing data set + # config options for ismip6 antarctic ice sheet data set [ismip6_ais] # Base path to the input ismip6 ocean and smb forcing files. User has to supply. @@ -43,10 +46,11 @@ All three test cases share some set of default config options under the section model = NorESM1-M # Scenarios used by climate model. User has to supply. - # Available scenarios are the following: RCP26, RCP85, SSP126, SSP585 + # Available scenarios are the following: RCP26, RCP26-repeat, RCP85, SSP126, SSP585 scenario = RCP26-repeat - # name of the mali mesh used to name mapping files (e,g. Antarctica_8to80km). User has to supply. + # name of the mali mesh. User has to supply. Note: It is used to name mapping files + # (e,g. 'map_ismip6_8km_to_{mali_mesh_name}_{method_remap}.nc'). mali_mesh_name = Antarctica_8to80km # MALI mesh file to be used to build mapping file (netCDF format). User has to supply. @@ -59,30 +63,24 @@ All three test cases share some set of default config options under the section # Remapping method used in building a mapping file. Options include: bilinear, neareststod, conserve method_remap = bilinear - # Path to which processed output SMB data is saved. User has to supply. - output_path = /Users/hollyhan/Desktop/Data/ISMIP6-Projections-Forcing-2300/AIS_Processed/Atmospheric_Forcing/ - - # config options for ismip6 ocean thermal forcing data test cases [ismip6_ais_ocean_thermal] # Remapping method used in building a mapping file. Options include: bilinear, neareststod, conserve method_remap = bilinear - # Path to which processed output thermal forcing data is saved. User has to supply. - output_path = /Users/hollyhan/Desktop/Data/ISMIP6-Projections-Forcing-2300/AIS_Processed/Ocean_Forcing/ + # Set to True if the want to process observational thermal forcing data. Set to False if want to process model thermal forcing data. + # Note: when set True, the ['ismip6_ais'] config options 'period_endyear', 'model' and 'scenario' will be ignored. + process_obs_data = False # config options for ismip6 ocean basal test case [ismip6_ais_ocean_basal] - # Remapping method used in building a mapping file. Options include: bilinear, neareststod, conserve + # Remapping method used in building a mapping file. Ocean basal testcase will always want to + # use neareststod method. method_remap = neareststod - # Path to which processed output basal param coeff data is saved. User has to supply. - output_path = /Users/hollyhan/Desktop/Data/ISMIP6-Projections-Forcing-2300/AIS_Processed/Ocean_Forcing/parametrizations/ - - atmosphere ---------- From 9dfa495c07c55096ee1ae34d446402c014f264e0 Mon Sep 17 00:00:00 2001 From: hollyhan <33441196+hollyhan@users.noreply.github.com> Date: Thu, 26 May 2022 12:09:46 -0600 Subject: [PATCH 25/58] Update the config file - Get a config option using 'getboolean' - Add a config option for output_base_path - Rename config file --- .../{ismip6.cfg => ismip6_forcing.cfg} | 22 +++++++++---------- .../ocean_thermal/process_thermal_forcing.py | 10 ++++----- 2 files changed, 15 insertions(+), 17 deletions(-) rename compass/landice/tests/ismip6_forcing/{ismip6.cfg => ismip6_forcing.cfg} (72%) diff --git a/compass/landice/tests/ismip6_forcing/ismip6.cfg b/compass/landice/tests/ismip6_forcing/ismip6_forcing.cfg similarity index 72% rename from compass/landice/tests/ismip6_forcing/ismip6.cfg rename to compass/landice/tests/ismip6_forcing/ismip6_forcing.cfg index c95140a0e6..d33eafaacd 100644 --- a/compass/landice/tests/ismip6_forcing/ismip6.cfg +++ b/compass/landice/tests/ismip6_forcing/ismip6_forcing.cfg @@ -11,6 +11,9 @@ base_path_mali = NotAvailable # Available end years are 2100 and 2300. period_endyear = NotAvailable +# Base path to which output forcing files are saved. +output_base_path = NotAvailable + # Name of climate model name used to generate ISMIP6 forcing data. User has to supply. # Available model names are the following: CCSM4, CESM2-WACCM, CSIRO-Mk3-6-0, HadGEM2-ES, NorESM1-M, UKESM1-0-LL model = NotAvailable @@ -19,10 +22,11 @@ model = NotAvailable # Available scenarios are the following: RCP26, RCP85, SSP126, SSP585 scenario = NotAvailable -# name of the mali mesh used to name mapping files (e,g. Antarctica_8to80km) +# name of the mali mesh. User has to supply. Note: It is used to name mapping files +# (e,g. 'map_ismip6_8km_to_{mali_mesh_name}_{method_remap}.nc'). mali_mesh_name = NotAvailable -# MALI mesh file to be used to build mapping file (e.g.Antarctic_8to80km_20220407.nc) +# MALI mesh file to be used to build mapping file (e.g.Antarctic_8to80km_20220407.nc). User has to supply. mali_mesh_file = NotAvailable @@ -32,9 +36,6 @@ mali_mesh_file = NotAvailable # Remapping method used in building a mapping file. Options include: bilinear, neareststod, conserve method_remap = bilinear -# Path to which processed output SMB data is saved. User has to supply. -output_path = NotAvailable - # config options for ismip6 ocean thermal forcing data test cases [ismip6_ais_ocean_thermal] @@ -42,15 +43,12 @@ output_path = NotAvailable # Remapping method used in building a mapping file. Options include: bilinear, neareststod, conserve method_remap = bilinear -# Path to which processed output thermal forcing data is saved. User has to supply. -output_path = NotAvailable - +# Set to True if the want to process observational thermal forcing data. Set to False if want to process model thermal forcing data. +process_obs_data = False # config options for ismip6 ocean basal test case [ismip6_ais_ocean_basal] -# Remapping method used in building a mapping file. Options include: bilinear, neareststod, conserve +# Remapping method used in building a mapping file. Note: ocean basal testcase will always want to +# use neareststod method. method_remap = neareststod - -# Path to which processed output basal param coeff data is saved. User has to supply. -output_path = NotAvailable diff --git a/compass/landice/tests/ismip6_forcing/ocean_thermal/process_thermal_forcing.py b/compass/landice/tests/ismip6_forcing/ocean_thermal/process_thermal_forcing.py index 9b81dc809f..0a8703acdd 100644 --- a/compass/landice/tests/ismip6_forcing/ocean_thermal/process_thermal_forcing.py +++ b/compass/landice/tests/ismip6_forcing/ocean_thermal/process_thermal_forcing.py @@ -38,7 +38,7 @@ def setup(self): scenario = section.get("scenario") section = config['ismip6_ais_ocean_thermal'] - process_obs_data = section.get("process_obs_data") + process_obs_data = section.getboolean("process_obs_data") if period_endyear == "NotAvailable": raise ValueError("You need to supply a user config file, which " @@ -57,7 +57,7 @@ def setup(self): target=os.path.join(base_path_mali, mali_mesh_file)) - if process_obs_data == "True": + if process_obs_data: input_file = self._file_obs output_file = f"processed_obs_TF_1995-2017_8km_x_60m.nc" else: @@ -85,9 +85,9 @@ def run(self): section = config['ismip6_ais_ocean_thermal'] method_remap = section.get('method_remap') - process_obs_data = section.get('process_obs_data') + process_obs_data = section.getboolean('process_obs_data') - if process_obs_data == "True": + if process_obs_data: input_file_list = self._file_obs output_file = f'processed_obs_TF_1995-2017_8km_x_60m.nc' output_path = f'{output_base_path}/ocean_thermal_forcing/'\ @@ -197,7 +197,7 @@ def rename_ismip6thermalforcing_to_mali_vars(self, remapped_file_temp, ds = ds.drop_vars('z') # dropping 'z' while it's still called 'z' # build dictionary for ismip6 variables that MALI takes in - if process_obs_data == "True": + if process_obs_data: ismip6_to_mali_dims = dict( z="nISMIP6OceanLayers", ncol="nCells") From 6cd599dd99af20cb6afc7da70fc4f2ae80e0b9ec Mon Sep 17 00:00:00 2001 From: hollyhan Date: Thu, 26 May 2022 18:39:34 -0600 Subject: [PATCH 26/58] Delete duplicate path check and function call Delete because 1) Checking the mapfile path is done in the main function `run` 2) 'set_lat_lon_fields_in_planar_grid.py' is already called once in the AIS mesh-generation test group --- .../atmosphere/create_mapfile_smb.py | 14 -------------- .../landice/tests/ismip6_forcing/create_mapfile.py | 7 ------- 2 files changed, 21 deletions(-) diff --git a/compass/landice/tests/ismip6_forcing/atmosphere/create_mapfile_smb.py b/compass/landice/tests/ismip6_forcing/atmosphere/create_mapfile_smb.py index 59321ec7e0..5bbe62b200 100644 --- a/compass/landice/tests/ismip6_forcing/atmosphere/create_mapfile_smb.py +++ b/compass/landice/tests/ismip6_forcing/atmosphere/create_mapfile_smb.py @@ -23,10 +23,6 @@ def build_mapping_file(ismip6_grid_file, mapping_file, mali_mesh_file=None, Remapping method used in building a mapping file """ - if os.path.exists(mapping_file): - print("Mapping file exists. Not building a new one.") - return - if mali_mesh_file is None: raise ValueError("Mapping file does not exist. To build one, Mali " "mesh file with '-f' should be provided. " @@ -34,7 +30,6 @@ def build_mapping_file(ismip6_grid_file, mapping_file, mali_mesh_file=None, ismip6_scripfile = "temp_ismip6_8km_scrip.nc" mali_scripfile = "temp_mali_scrip.nc" - ismip6_projection = "ais-bedmap2" # create the ismip6 scripfile if mapping file does not exist # this is the projection of ismip6 data for Antarctica @@ -45,14 +40,6 @@ def build_mapping_file(ismip6_grid_file, mapping_file, mali_mesh_file=None, # create a scripfile for the atmosphere forcing data create_atm_scrip(ismip6_grid_file, ismip6_scripfile) - # create a MALI mesh scripfile if mapping file does not exist - # make sure the mali mesh file uses the longitude convention of [0 2pi] - args = ["set_lat_lon_fields_in_planar_grid.py", - "--file", mali_mesh_file, - "--proj", ismip6_projection] - - subprocess.check_call(args) - # create a MALI mesh scripfile if mapping file does not exist scrip_from_mpas(mali_mesh_file, mali_scripfile) @@ -80,7 +67,6 @@ def build_mapping_file(ismip6_grid_file, mapping_file, mali_mesh_file=None, os.remove(ismip6_scripfile) os.remove(mali_scripfile) - def create_atm_scrip(ismip6_grid_file, ismip6_scripfile): """ create a scripfile for the ismip6 atmospheric forcing data. diff --git a/compass/landice/tests/ismip6_forcing/create_mapfile.py b/compass/landice/tests/ismip6_forcing/create_mapfile.py index b5f86452a2..43caef9d5f 100644 --- a/compass/landice/tests/ismip6_forcing/create_mapfile.py +++ b/compass/landice/tests/ismip6_forcing/create_mapfile.py @@ -48,13 +48,6 @@ def build_mapping_file(ismip6_grid_file, mapping_file, mali_mesh_file=None, subprocess.check_call(args) - # make sure the input file uses the longitude convention of [0 2pi] - args = ["set_lat_lon_fields_in_planar_grid.py", - "--file", mali_mesh_file, - "--proj", ismip6_projection] - - subprocess.check_call(args) - # create a MALI mesh scripfile if mapping file does not exist scrip_from_mpas(mali_mesh_file, mali_scripfile) From dadf323f9afb232d6829042e763eb3a2d9aea38e Mon Sep 17 00:00:00 2001 From: hollyhan Date: Mon, 6 Jun 2022 10:39:36 -0600 Subject: [PATCH 27/58] Add a shared configure function --- .../ismip6_forcing/atmosphere/__init__.py | 28 ++-------------- .../ismip6_forcing/atmosphere/process_smb.py | 13 -------- .../landice/tests/ismip6_forcing/configure.py | 27 ++++++++++++++++ .../ismip6_forcing/ocean_basal/__init__.py | 28 ++-------------- .../ismip6_forcing/ocean_thermal/__init__.py | 32 ++++--------------- .../ocean_thermal/process_thermal_forcing.py | 13 -------- 6 files changed, 40 insertions(+), 101 deletions(-) create mode 100644 compass/landice/tests/ismip6_forcing/configure.py diff --git a/compass/landice/tests/ismip6_forcing/atmosphere/__init__.py b/compass/landice/tests/ismip6_forcing/atmosphere/__init__.py index 3ac9d53049..c84431387e 100644 --- a/compass/landice/tests/ismip6_forcing/atmosphere/__init__.py +++ b/compass/landice/tests/ismip6_forcing/atmosphere/__init__.py @@ -1,6 +1,8 @@ from compass.testcase import TestCase from compass.landice.tests.ismip6_forcing.atmosphere.process_smb \ import ProcessSMB +from compass.landice.tests.ismip6_forcing.configure import configure as \ + configure_testgroup class Atmosphere(TestCase): @@ -31,28 +33,4 @@ def configure(self): """ Configures test case """ - base_path_ismip6 = self.config.get(section="ismip6_ais", - option="base_path_ismip6") - base_path_mali = self.config.get(section="ismip6_ais", - option="base_path_mali") - mali_mesh_name = self.config.get(section="ismip6_ais", - option="mali_mesh_name") - mali_mesh_file = self.config.get(section="ismip6_ais", - option="mali_mesh_file") - - if base_path_ismip6 == "NotAvailable": - raise ValueError("You need to supply a user config file, which " - "should contain the ismip6_ais " - "section with the base_path_ismip6 option") - if base_path_mali == "NotAvailable": - raise ValueError("You need to supply a user config file, which " - "should contain the ismip6_ais " - "section with the base_path_mali option") - if mali_mesh_name == "NotAvailable": - raise ValueError("You need to supply a user config file, which " - "should contain the ismip6_ais " - "section with the mali_mesh_name option") - if mali_mesh_file == "NotAvailable": - raise ValueError("You need to supply a user config file, which " - "should contain the ismip6_ais " - "section with the mali_mesh_file option") + configure_testgroup(config=self.config, check_model_options=True) diff --git a/compass/landice/tests/ismip6_forcing/atmosphere/process_smb.py b/compass/landice/tests/ismip6_forcing/atmosphere/process_smb.py index 95b63888da..b75533ac1a 100644 --- a/compass/landice/tests/ismip6_forcing/atmosphere/process_smb.py +++ b/compass/landice/tests/ismip6_forcing/atmosphere/process_smb.py @@ -40,19 +40,6 @@ def setup(self): model = section.get("model") scenario = section.get("scenario") - if period_endyear == "NotAvailable": - raise ValueError("You need to supply a user config file, which " - "should contain the ismip6_ais " - "section with the period_endyear option") - if model == "NotAvailable": - raise ValueError("You need to supply a user config file, which " - "should contain the ismip6_ais " - "section with the model option") - if scenario == "NotAvailable": - raise ValueError("You need to supply a user config file, which " - "should contain the ismip6_ais " - "section with the scenario option") - self.add_input_file(filename=mali_mesh_file, target=os.path.join(base_path_mali, mali_mesh_file)) diff --git a/compass/landice/tests/ismip6_forcing/configure.py b/compass/landice/tests/ismip6_forcing/configure.py new file mode 100644 index 0000000000..842426419e --- /dev/null +++ b/compass/landice/tests/ismip6_forcing/configure.py @@ -0,0 +1,27 @@ +def configure(config, check_model_options): + """ + A shared function for configuring options for all ismip6 forcing + test cases + + Parameters + ---------- + config : compass.config.CompassConfigParser + Configuration options for a ismip6 forcing test case + + check_model_options : bool + Whether we check ``model``, ``scenario``, ``period_endyear`` + + """ + + section = 'ismip6_ais' + options = ['base_path_ismip6', 'base_path_mali', 'mali_mesh_name', + 'mali_mesh_file'] + if check_model_options: + options = options + ['model', 'scenario', 'period_endyear'] + + for option in options: + value = config.get(section=section, option=option) + if value == "NotAvailable": + raise ValueError(f"You need to supply a user config file, which " + f"should contain the {section} " + f"section with the {option} option") diff --git a/compass/landice/tests/ismip6_forcing/ocean_basal/__init__.py b/compass/landice/tests/ismip6_forcing/ocean_basal/__init__.py index 3a46b7eea9..594ff38950 100644 --- a/compass/landice/tests/ismip6_forcing/ocean_basal/__init__.py +++ b/compass/landice/tests/ismip6_forcing/ocean_basal/__init__.py @@ -1,6 +1,8 @@ from compass.testcase import TestCase from compass.landice.tests.ismip6_forcing.ocean_basal.process_basal_melt \ import ProcessBasalMelt +from compass.landice.tests.ismip6_forcing.configure import configure as \ + configure_testgroup class OceanBasal(TestCase): @@ -30,28 +32,4 @@ def configure(self): """ Configures test case """ - base_path_ismip6 = self.config.get(section="ismip6_ais", - option="base_path_ismip6") - base_path_mali = self.config.get(section="ismip6_ais", - option="base_path_mali") - mali_mesh_name = self.config.get(section="ismip6_ais", - option="mali_mesh_name") - mali_mesh_file = self.config.get(section="ismip6_ais", - option="mali_mesh_file") - - if base_path_ismip6 == "NotAvailable": - raise ValueError("You need to supply a user config file, which " - "should contain the ismip6_ais " - "section with the base_path_ismip6 option") - if base_path_mali == "NotAvailable": - raise ValueError("You need to supply a user config file, which " - "should contain the ismip6_ais " - "section with the base_path_mali option") - if mali_mesh_name == "NotAvailable": - raise ValueError("You need to supply a user config file, which " - "should contain the ismip6_ais " - "section with the mali_mesh_name option") - if mali_mesh_file == "NotAvailable": - raise ValueError("You need to supply a user config file, which " - "should contain the ismip6_ais " - "section with the mali_mesh_file option") + configure_testgroup(config=self.config, check_model_options=False) diff --git a/compass/landice/tests/ismip6_forcing/ocean_thermal/__init__.py b/compass/landice/tests/ismip6_forcing/ocean_thermal/__init__.py index 0e88257996..97b58717bc 100644 --- a/compass/landice/tests/ismip6_forcing/ocean_thermal/__init__.py +++ b/compass/landice/tests/ismip6_forcing/ocean_thermal/__init__.py @@ -1,6 +1,8 @@ from compass.testcase import TestCase from compass.landice.tests.ismip6_forcing.ocean_thermal.\ process_thermal_forcing import ProcessThermalForcing +from compass.landice.tests.ismip6_forcing.configure import configure as \ + configure_testgroup class OceanThermal(TestCase): @@ -30,28 +32,8 @@ def configure(self): """ Configures test case """ - base_path_ismip6 = self.config.get(section="ismip6_ais", - option="base_path_ismip6") - base_path_mali = self.config.get(section="ismip6_ais", - option="base_path_mali") - mali_mesh_name = self.config.get(section="ismip6_ais", - option="mali_mesh_name") - mali_mesh_file = self.config.get(section="ismip6_ais", - option="mali_mesh_file") - - if base_path_ismip6 == "NotAvailable": - raise ValueError("You need to supply a user config file, which " - "should contain the ismip6_ais " - "section with the base_path_ismip6 option") - if base_path_mali == "NotAvailable": - raise ValueError("You need to supply a user config file, which " - "should contain the ismip6_ais " - "section with the base_path_mali option") - if mali_mesh_name == "NotAvailable": - raise ValueError("You need to supply a user config file, which " - "should contain the ismip6_ais " - "section with the mali_mesh_name option") - if mali_mesh_file == "NotAvailable": - raise ValueError("You need to supply a user config file, which " - "should contain the ismip6_ais " - "section with the mali_mesh_file option") + process_obs_data = self.config.getboolean('ismip6_ais_ocean_thermal', + 'process_obs_data') + check_model_options = not process_obs_data + configure_testgroup(config=self.config, + check_model_options=check_model_options) diff --git a/compass/landice/tests/ismip6_forcing/ocean_thermal/process_thermal_forcing.py b/compass/landice/tests/ismip6_forcing/ocean_thermal/process_thermal_forcing.py index 0a8703acdd..76aac504a2 100644 --- a/compass/landice/tests/ismip6_forcing/ocean_thermal/process_thermal_forcing.py +++ b/compass/landice/tests/ismip6_forcing/ocean_thermal/process_thermal_forcing.py @@ -40,19 +40,6 @@ def setup(self): section = config['ismip6_ais_ocean_thermal'] process_obs_data = section.getboolean("process_obs_data") - if period_endyear == "NotAvailable": - raise ValueError("You need to supply a user config file, which " - "should contain the ismip6_ais " - "section with the period_endyear option") - if model == "NotAvailable": - raise ValueError("You need to supply a user config file, which " - "should contain the ismip6_ais " - "section with the model option") - if scenario == "NotAvailable": - raise ValueError("You need to supply a user config file, which " - "should contain the ismip6_ais " - "section with the scenario option") - self.add_input_file(filename=mali_mesh_file, target=os.path.join(base_path_mali, mali_mesh_file)) From e7e0f7071b34a501c1bf2ca63fa5753202b8026b Mon Sep 17 00:00:00 2001 From: hollyhan Date: Mon, 6 Jun 2022 11:24:21 -0600 Subject: [PATCH 28/58] Add parallel executable to ESMF_RegridWeightGen --- .../atmosphere/create_mapfile_smb.py | 34 ++++++++++++------- .../ismip6_forcing/atmosphere/process_smb.py | 6 ++-- .../tests/ismip6_forcing/create_mapfile.py | 32 +++++++++++------ .../ocean_basal/process_basal_melt.py | 6 ++-- .../ocean_thermal/process_thermal_forcing.py | 6 ++-- 5 files changed, 56 insertions(+), 28 deletions(-) diff --git a/compass/landice/tests/ismip6_forcing/atmosphere/create_mapfile_smb.py b/compass/landice/tests/ismip6_forcing/atmosphere/create_mapfile_smb.py index 5bbe62b200..f58bccc6e0 100644 --- a/compass/landice/tests/ismip6_forcing/atmosphere/create_mapfile_smb.py +++ b/compass/landice/tests/ismip6_forcing/atmosphere/create_mapfile_smb.py @@ -1,18 +1,25 @@ import os -import subprocess import netCDF4 import xarray as xr from mpas_tools.scrip.from_mpas import scrip_from_mpas +from mpas_tools.logging import check_call # function that creates a mapping file from ismip6 grid to mali mesh -def build_mapping_file(ismip6_grid_file, mapping_file, mali_mesh_file=None, +def build_mapping_file(config, cores, logger, ismip6_grid_file, + mapping_file, mali_mesh_file=None, method_remap=None): """ Build a mapping file if it does not exist. Parameters ---------- + config : compass.config.CompassConfigParser + Configuration options for a ismip6 forcing test case + cores : int + the number of cores for the ESMF_RegridWeightGen + logger : logging.Logger + A logger for output from the step ismip6_grid_file : str ismip6 grid file mapping_file : str @@ -51,16 +58,19 @@ def build_mapping_file(ismip6_grid_file, mapping_file, mali_mesh_file=None, "--method. Available options are 'bilinear'," "'neareststod', 'conserve'.") - args = ["ESMF_RegridWeightGen", - "-s", ismip6_scripfile, - "-d", mali_scripfile, - "-w", mapping_file, - "-m", method_remap, - "-i", "-64bit_offset", - "--dst_regional", "--src_regional"] - - # include flag and input and output file names - subprocess.check_call(args) + parallel_executable = config.get('parallel', 'parallel_executable') + # split the parallel executable into constituents in case it includes flags + args = parallel_executable.split(' ') + args.extend(["-n", f"{cores}", + "ESMF_RegridWeightGen", + "-s", ismip6_scripfile, + "-d", mali_scripfile, + "-w", mapping_file, + "-m", method_remap, + "-i", "-64bit_offset", + "--dst_regional", "--src_regional"]) + + check_call(args, logger) # remove the temporary scripfiles once the mapping file is generated print("Removing the temporary scripfiles...") diff --git a/compass/landice/tests/ismip6_forcing/atmosphere/process_smb.py b/compass/landice/tests/ismip6_forcing/atmosphere/process_smb.py index b75533ac1a..81d8b71e1a 100644 --- a/compass/landice/tests/ismip6_forcing/atmosphere/process_smb.py +++ b/compass/landice/tests/ismip6_forcing/atmosphere/process_smb.py @@ -25,7 +25,8 @@ def __init__(self, test_case, input_file=None): input_file : file name of ismip6 forcing data processed by this step """ self.input_file = input_file - super().__init__(test_case=test_case, name='process_smb') + super().__init__(test_case=test_case, name='process_smb', cores=4, + min_cores=1) def setup(self): """ @@ -151,7 +152,8 @@ def remap_ismip6smb_to_mali(self, input_file, output_file, mali_mesh_name, # build a mapping file if it doesn't already exist self.logger.info("Creating a mapping file. " "Mapping method used: {method_remap}") - build_mapping_file(input_file, mapping_file, mali_mesh_file, + build_mapping_file(self.config, self.cores, self.logger, + input_file, mapping_file, mali_mesh_file, method_remap) else: self.logger.info("Mapping file exists. " diff --git a/compass/landice/tests/ismip6_forcing/create_mapfile.py b/compass/landice/tests/ismip6_forcing/create_mapfile.py index 43caef9d5f..fcbd899a52 100644 --- a/compass/landice/tests/ismip6_forcing/create_mapfile.py +++ b/compass/landice/tests/ismip6_forcing/create_mapfile.py @@ -1,16 +1,22 @@ import os -import subprocess from mpas_tools.scrip.from_mpas import scrip_from_mpas +from mpas_tools.logging import check_call - -# function that creates a mapping file from ismip6 grid to mali mesh -def build_mapping_file(ismip6_grid_file, mapping_file, mali_mesh_file=None, +# function that creates a mapping file from ismip6 grid to mali +def build_mapping_file(config, cores, logger, ismip6_grid_file, + mapping_file, mali_mesh_file=None, method_remap=None): """ Build a mapping file if it does not exist. Parameters ---------- + config : compass.config.CompassConfigParser + Configuration options for a ismip6 forcing test case + cores : int + the number of cores for the ESMF_RegridWeightGen + logger : logging.Logger + A logger for output from the step ismip6_grid_file : str ismip6 grid file mapping_file : str @@ -40,13 +46,19 @@ def build_mapping_file(ismip6_grid_file, mapping_file, mali_mesh_file=None, "input/ouptut meshes") print("Creating temporary scripfiles for ismip6 grid and mali mesh...") - args = ["create_SCRIP_file_from_planar_rectangular_grid.py", - "--input", ismip6_grid_file, - "--scrip", ismip6_scripfile, - "--proj", ismip6_projection, - "--rank", "2"] + parallel_executable = config.get('parallel', 'parallel_executable') + # split the parallel executable into constituents in case it includes flags + args = parallel_executable.split(' ') + args.extend(["-n", f"{cores}", + "ESMF_RegridWeightGen", + "-s", ismip6_scripfile, + "-d", mali_scripfile, + "-w", mapping_file, + "-m", method_remap, + "-i", "-64bit_offset", + "--dst_regional", "--src_regional"]) - subprocess.check_call(args) + check_call(args, logger) # create a MALI mesh scripfile if mapping file does not exist scrip_from_mpas(mali_mesh_file, mali_scripfile) diff --git a/compass/landice/tests/ismip6_forcing/ocean_basal/process_basal_melt.py b/compass/landice/tests/ismip6_forcing/ocean_basal/process_basal_melt.py index 9e3a6cfef8..74465e9a38 100644 --- a/compass/landice/tests/ismip6_forcing/ocean_basal/process_basal_melt.py +++ b/compass/landice/tests/ismip6_forcing/ocean_basal/process_basal_melt.py @@ -23,7 +23,8 @@ def __init__(self, test_case): test_case : compass.landice.tests.ismip6_forcing.ocean_basal.OceanBasal The test case this step belongs to """ - super().__init__(test_case=test_case, name='process_basal_melt') + super().__init__(test_case=test_case, name='process_basal_melt', + cores=4, min_cores=1) def setup(self): """ @@ -165,7 +166,8 @@ def remap_ismip6BasalMelt_to_mali(self, input_file, output_file, if not os.path.exists(mapping_file): # build a mapping file if it doesn't already exist - build_mapping_file(input_file, mapping_file, mali_mesh_file, + build_mapping_file(self.config, self.cores, self.logger, + input_file, mapping_file, mali_mesh_file, method_remap) else: print("Mapping file exists. Remapping the input data...") diff --git a/compass/landice/tests/ismip6_forcing/ocean_thermal/process_thermal_forcing.py b/compass/landice/tests/ismip6_forcing/ocean_thermal/process_thermal_forcing.py index 76aac504a2..c42ef2b069 100644 --- a/compass/landice/tests/ismip6_forcing/ocean_thermal/process_thermal_forcing.py +++ b/compass/landice/tests/ismip6_forcing/ocean_thermal/process_thermal_forcing.py @@ -22,7 +22,8 @@ def __init__(self, test_case): OceanThermal The test case this step belongs to """ - super().__init__(test_case=test_case, name='process_thermal_forcing') + super().__init__(test_case=test_case, name='process_thermal_forcing', + cores=4, min_cores=1) def setup(self): """ @@ -145,7 +146,8 @@ def remap_ismip6thermalforcing_to_mali(self, input_file, if not os.path.exists(mapping_file): # build a mapping file if it doesn't already exist - build_mapping_file(input_file, mapping_file, mali_mesh_file, + build_mapping_file(self.config, self.cores, self.logger, + input_file, mapping_file, mali_mesh_file, method_remap) else: print("Mapping file exists. Remapping the input data...") From a11897340ba7e459e2c11e2e87b4d50e86f0c1ab Mon Sep 17 00:00:00 2001 From: hollyhan Date: Mon, 6 Jun 2022 12:57:52 -0600 Subject: [PATCH 29/58] Add a check to ensure mali mesh longitudinal range is [0 2pi] --- .../atmosphere/create_mapfile_smb.py | 23 +++++++- .../tests/ismip6_forcing/create_mapfile.py | 58 ++++++++++++------- 2 files changed, 57 insertions(+), 24 deletions(-) diff --git a/compass/landice/tests/ismip6_forcing/atmosphere/create_mapfile_smb.py b/compass/landice/tests/ismip6_forcing/atmosphere/create_mapfile_smb.py index f58bccc6e0..e4120ac9b8 100644 --- a/compass/landice/tests/ismip6_forcing/atmosphere/create_mapfile_smb.py +++ b/compass/landice/tests/ismip6_forcing/atmosphere/create_mapfile_smb.py @@ -1,4 +1,6 @@ import os +import shutil +import subprocess import netCDF4 import xarray as xr from mpas_tools.scrip.from_mpas import scrip_from_mpas @@ -37,6 +39,7 @@ def build_mapping_file(config, cores, logger, ismip6_grid_file, ismip6_scripfile = "temp_ismip6_8km_scrip.nc" mali_scripfile = "temp_mali_scrip.nc" + ismip6_projection = "ais-bedmap2" # create the ismip6 scripfile if mapping file does not exist # this is the projection of ismip6 data for Antarctica @@ -48,7 +51,21 @@ def build_mapping_file(config, cores, logger, ismip6_grid_file, create_atm_scrip(ismip6_grid_file, ismip6_scripfile) # create a MALI mesh scripfile if mapping file does not exist - scrip_from_mpas(mali_mesh_file, mali_scripfile) + # make sure the mali mesh file uses the longitude convention of [0 2pi] + # make changes on a duplicated file to avoid making changes to the + # original mesh file + + mali_mesh_copy = f"{mali_mesh_file}_copy" + shutil.copy(mali_mesh_file, f"{mali_mesh_file}_copy") + + args = ["set_lat_lon_fields_in_planar_grid.py", + "--file", mali_mesh_copy, + "--proj", ismip6_projection] + + subprocess.check_call(args) + + # create a MALI mesh scripfile if mapping file does not exist + scrip_from_mpas(mali_mesh_copy, mali_scripfile) # create a mapping file using ESMF weight gen print(f"Creating a mapping file. Mapping method used: {method_remap}") @@ -73,9 +90,11 @@ def build_mapping_file(config, cores, logger, ismip6_grid_file, check_call(args, logger) # remove the temporary scripfiles once the mapping file is generated - print("Removing the temporary scripfiles...") + print("Removing the temporary mesh and scripfiles...") os.remove(ismip6_scripfile) os.remove(mali_scripfile) + os.remove(mali_mesh_copy) + def create_atm_scrip(ismip6_grid_file, ismip6_scripfile): """ diff --git a/compass/landice/tests/ismip6_forcing/create_mapfile.py b/compass/landice/tests/ismip6_forcing/create_mapfile.py index fcbd899a52..21527273ab 100644 --- a/compass/landice/tests/ismip6_forcing/create_mapfile.py +++ b/compass/landice/tests/ismip6_forcing/create_mapfile.py @@ -1,7 +1,10 @@ import os +import shutil +import subprocess from mpas_tools.scrip.from_mpas import scrip_from_mpas from mpas_tools.logging import check_call + # function that creates a mapping file from ismip6 grid to mali def build_mapping_file(config, cores, logger, ismip6_grid_file, mapping_file, mali_mesh_file=None, @@ -46,21 +49,28 @@ def build_mapping_file(config, cores, logger, ismip6_grid_file, "input/ouptut meshes") print("Creating temporary scripfiles for ismip6 grid and mali mesh...") - parallel_executable = config.get('parallel', 'parallel_executable') - # split the parallel executable into constituents in case it includes flags - args = parallel_executable.split(' ') - args.extend(["-n", f"{cores}", - "ESMF_RegridWeightGen", - "-s", ismip6_scripfile, - "-d", mali_scripfile, - "-w", mapping_file, - "-m", method_remap, - "-i", "-64bit_offset", - "--dst_regional", "--src_regional"]) + args = ["create_SCRIP_file_from_planar_rectangular_grid.py", + "--input", ismip6_grid_file, + "--scrip", ismip6_scripfile, + "--proj", ismip6_projection, + "--rank", "2"] - check_call(args, logger) + subprocess.check_call(args) # create a MALI mesh scripfile if mapping file does not exist + # make sure the mali mesh file uses the longitude convention of [0 2pi] + # make changes on a duplicated file to avoid making changes to the + # original mesh file + + mali_mesh_copy = f"{mali_mesh_file}_copy" + shutil.copy(mali_mesh_file, f"{mali_mesh_file}_copy") + + args = ["set_lat_lon_fields_in_planar_grid.py", + "--file", mali_mesh_copy, + "--proj", ismip6_projection] + + subprocess.check_call(args) + scrip_from_mpas(mali_mesh_file, mali_scripfile) # create a mapping file using ESMF weight gen @@ -71,18 +81,22 @@ def build_mapping_file(config, cores, logger, ismip6_grid_file, "--method. Available options are 'bilinear'," "'neareststod', 'conserve'.") - args = ["ESMF_RegridWeightGen", - "-s", ismip6_scripfile, - "-d", mali_scripfile, - "-w", mapping_file, - "-m", method_remap, - "-i", "-64bit_offset", - "--dst_regional", "--src_regional"] + parallel_executable = config.get('parallel', 'parallel_executable') + # split the parallel executable into constituents in case it includes flags + args = parallel_executable.split(' ') + args.extend(["-n", f"{cores}", + "ESMF_RegridWeightGen", + "-s", ismip6_scripfile, + "-d", mali_scripfile, + "-w", mapping_file, + "-m", method_remap, + "-i", "-64bit_offset", + "--dst_regional", "--src_regional"]) - # include flag and input and output file names - subprocess.check_call(args) + check_call(args, logger) # remove the temporary scripfiles once the mapping file is generated - print("Removing the temporary scripfiles...") + print("Removing the temporary mesh and scripfiles...") os.remove(ismip6_scripfile) os.remove(mali_scripfile) + os.remove(mali_mesh_copy) From e9e0aa3a23b25ec64fe5d0d1612ba785639f10aa Mon Sep 17 00:00:00 2001 From: hollyhan Date: Sat, 11 Jun 2022 20:56:31 -0600 Subject: [PATCH 30/58] Add a step procesing modern climatology and applies climatology correction --- .../ismip6_forcing/atmosphere/__init__.py | 8 +- .../atmosphere/create_mapfile_smb.py | 121 ++++++--- .../ismip6_forcing/atmosphere/process_smb.py | 171 ++++++++---- .../atmosphere/process_smb_racmo.py | 256 ++++++++++++++++++ .../tests/ismip6_forcing/ismip6_forcing.cfg | 4 +- 5 files changed, 468 insertions(+), 92 deletions(-) create mode 100644 compass/landice/tests/ismip6_forcing/atmosphere/process_smb_racmo.py diff --git a/compass/landice/tests/ismip6_forcing/atmosphere/__init__.py b/compass/landice/tests/ismip6_forcing/atmosphere/__init__.py index c84431387e..c4dda63043 100644 --- a/compass/landice/tests/ismip6_forcing/atmosphere/__init__.py +++ b/compass/landice/tests/ismip6_forcing/atmosphere/__init__.py @@ -1,6 +1,8 @@ from compass.testcase import TestCase from compass.landice.tests.ismip6_forcing.atmosphere.process_smb \ import ProcessSMB +from compass.landice.tests.ismip6_forcing.atmosphere.process_smb_racmo \ + import ProcessSmbRacmo from compass.landice.tests.ismip6_forcing.configure import configure as \ configure_testgroup @@ -28,9 +30,13 @@ def __init__(self, test_group): step = ProcessSMB(test_case=self) self.add_step(step) + step = ProcessSmbRacmo(test_case=self) + self.add_step(step) def configure(self): """ Configures test case """ - configure_testgroup(config=self.config, check_model_options=True) + + configure_testgroup(config=self.config, + check_model_options=True) diff --git a/compass/landice/tests/ismip6_forcing/atmosphere/create_mapfile_smb.py b/compass/landice/tests/ismip6_forcing/atmosphere/create_mapfile_smb.py index e4120ac9b8..83b229c122 100644 --- a/compass/landice/tests/ismip6_forcing/atmosphere/create_mapfile_smb.py +++ b/compass/landice/tests/ismip6_forcing/atmosphere/create_mapfile_smb.py @@ -1,16 +1,17 @@ import os +import numpy as np import shutil import subprocess import netCDF4 import xarray as xr from mpas_tools.scrip.from_mpas import scrip_from_mpas from mpas_tools.logging import check_call +from pyremap.descriptor import interp_extrap_corners_2d # function that creates a mapping file from ismip6 grid to mali mesh def build_mapping_file(config, cores, logger, ismip6_grid_file, - mapping_file, mali_mesh_file=None, - method_remap=None): + mapping_file, mali_mesh_file=None, method_remap=None): """ Build a mapping file if it does not exist. @@ -27,30 +28,35 @@ def build_mapping_file(config, cores, logger, ismip6_grid_file, mapping_file : str weights for interpolation from ismip6_grid_file to mali_mesh_file mali_mesh_file : str, optional - The MALI mesh file if mapping file does not exist + The MALI mesh file is used if mapping file does not exist method_remap : str, optional Remapping method used in building a mapping file """ + if os.path.exists(mapping_file): + print("Mapping file exists. Not building a new one.") + return + + # create the scrip files if mapping file does not exist + print("Mapping file does not exist. Building one based on the " + "input/ouptut meshes") + print("Creating temporary scrip files for source and destination grids...") + if mali_mesh_file is None: raise ValueError("Mapping file does not exist. To build one, Mali " "mesh file with '-f' should be provided. " "Type --help for info") - ismip6_scripfile = "temp_ismip6_8km_scrip.nc" + # name temporary scrip files that will be used to build mapping file + source_grid_scripfile = "temp_source_scrip.nc" mali_scripfile = "temp_mali_scrip.nc" - ismip6_projection = "ais-bedmap2" - - # create the ismip6 scripfile if mapping file does not exist # this is the projection of ismip6 data for Antarctica - print("Mapping file does not exist. Building one based on the " - "input/ouptut meshes") - print("Creating temporary scripfiles for ismip6 grid and mali mesh...") + ismip6_projection = "ais-bedmap2" # create a scripfile for the atmosphere forcing data - create_atm_scrip(ismip6_grid_file, ismip6_scripfile) + create_atm_scrip(ismip6_grid_file, source_grid_scripfile) - # create a MALI mesh scripfile if mapping file does not exist + # create a MALI mesh scripfile # make sure the mali mesh file uses the longitude convention of [0 2pi] # make changes on a duplicated file to avoid making changes to the # original mesh file @@ -77,26 +83,36 @@ def build_mapping_file(config, cores, logger, ismip6_grid_file, parallel_executable = config.get('parallel', 'parallel_executable') # split the parallel executable into constituents in case it includes flags - args = parallel_executable.split(' ') - args.extend(["-n", f"{cores}", - "ESMF_RegridWeightGen", - "-s", ismip6_scripfile, - "-d", mali_scripfile, - "-w", mapping_file, - "-m", method_remap, - "-i", "-64bit_offset", - "--dst_regional", "--src_regional"]) - - check_call(args, logger) + # args = parallel_executable.split(' ') + # args.extend(["-n", f"{cores}", + # "ESMF_RegridWeightGen", + # "-s", source_grid_scripfile, + # "-d", mali_scripfile, + # "-w", mapping_file, + # "-m", method_remap, + # "-i", "-64bit_offset", + # "--dst_regional", "--src_regional"]) + # + # check_call(args, logger) + + args = ["ESMF_RegridWeightGen", + "-s", source_grid_scripfile, + "-d", mali_scripfile, + "-w", mapping_file, + "-m", method_remap, + "-i", "-64bit_offset", + "--dst_regional", "--src_regional"] + + subprocess.check_call(args) # remove the temporary scripfiles once the mapping file is generated print("Removing the temporary mesh and scripfiles...") - os.remove(ismip6_scripfile) + # os.remove(source_grid_scripfile) os.remove(mali_scripfile) os.remove(mali_mesh_copy) -def create_atm_scrip(ismip6_grid_file, ismip6_scripfile): +def create_atm_scrip(source_grid_file, source_grid_scripfile): """ create a scripfile for the ismip6 atmospheric forcing data. Note: the atmospheric forcing data do not have 'x' and 'y' coordinates and @@ -105,18 +121,22 @@ def create_atm_scrip(ismip6_grid_file, ismip6_scripfile): Parameters ---------- - ismip6_grid_file : str - input ismip6 grid file + source_grid_file : str + input smb grid file - ismip6_scripfile : str - outputput ismip6 scrip file + source_grid_scripfile : str + output scrip file of the input smb data """ - ds = xr.open_dataset(ismip6_grid_file) - out_file = netCDF4.Dataset(ismip6_scripfile, 'w') + ds = xr.open_dataset(source_grid_file) + out_file = netCDF4.Dataset(source_grid_scripfile, 'w') - nx = ds.sizes["x"] - ny = ds.sizes["y"] + if "rlon" in ds and "rlat" in ds: # this is for RACMO's rotated-pole grid + nx = ds.sizes["rlon"] + ny = ds.sizes["rlat"] + else: + nx = ds.sizes["x"] + ny = ds.sizes["y"] units = 'degrees' grid_size = nx * ny @@ -147,19 +167,38 @@ def create_atm_scrip(ismip6_grid_file, ismip6_scripfile): out_file.variables['grid_dims'][:] = [nx, ny] out_file.variables['grid_imask'][:] = 1 - lat_corner = ds.lat_bnds - if "time" in lat_corner.dims: - lat_corner = lat_corner.isel(time=0) + if 'lat_bnds' in ds and 'lon_bnds' in ds: + lat_corner = ds.lat_bnds + if "time" in lat_corner.dims: + lat_corner = lat_corner.isel(time=0) - grid_corner_lat = lat_corner.values.reshape((grid_size, 4)) + lon_corner = ds.lon_bnds + if "time" in lon_corner.dims: + lon_corner = lon_corner.isel(time=0) - lon_corner = ds.lon_bnds - if "time" in lon_corner.dims: - lon_corner = lon_corner.isel(time=0) + lat_corner = lat_corner.values + lon_corner = lon_corner.values + else: # this part is used for RACMO as it does not have lat_bnds & lon_bnds + lat_corner = _unwrap_corners(interp_extrap_corners_2d(ds.lat.values)) + lon_corner = _unwrap_corners(interp_extrap_corners_2d(ds.lon.values)) - grid_corner_lon = lon_corner.values.reshape((grid_size, 4)) + grid_corner_lat = lat_corner.reshape((grid_size, 4)) + grid_corner_lon = lon_corner.reshape((grid_size, 4)) out_file.variables['grid_corner_lat'][:] = grid_corner_lat out_file.variables['grid_corner_lon'][:] = grid_corner_lon out_file.close() + + +def _unwrap_corners(in_field): + """Turn a 2D array of corners into an array of rectangular mesh elements""" + out_field = np.zeros(((in_field.shape[0] - 1) * + (in_field.shape[1] - 1), 4)) + # corners are counterclockwise + out_field[:, 0] = in_field[0:-1, 0:-1].flat + out_field[:, 1] = in_field[0:-1, 1:].flat + out_field[:, 2] = in_field[1:, 1:].flat + out_field[:, 3] = in_field[1:, 0:-1].flat + + return out_field diff --git a/compass/landice/tests/ismip6_forcing/atmosphere/process_smb.py b/compass/landice/tests/ismip6_forcing/atmosphere/process_smb.py index 81d8b71e1a..f768c2864a 100644 --- a/compass/landice/tests/ismip6_forcing/atmosphere/process_smb.py +++ b/compass/landice/tests/ismip6_forcing/atmosphere/process_smb.py @@ -36,6 +36,7 @@ def setup(self): section = config['ismip6_ais'] base_path_ismip6 = section.get('base_path_ismip6') base_path_mali = section.get('base_path_mali') + output_base_path = section.get('output_base_path') mali_mesh_file = section.get('mali_mesh_file') period_endyear = section.get("period_endyear") model = section.get("model") @@ -47,13 +48,31 @@ def setup(self): input_file_list = self._files[period_endyear][model][scenario] for file in input_file_list: - print(base_path_ismip6) - print(os.path.join(base_path_ismip6, file)) self.add_input_file(filename=os.path.basename(file), - target=os.path.join(base_path_ismip6, file)) - - output_file = f"processed_SMB_{model}_{scenario}_{period_endyear}.nc" - self.add_output_file(filename=output_file) + target=os.path.join(base_path_ismip6, + file)) + + output_file_esm = f"processed_SMB_{model}_{scenario}_" \ + f"{period_endyear}.nc" + self.add_output_file(filename=output_file_esm) + + # add processed racmo data as input as it is needed for smb correction + racmo_clim_file = f"processed_RACMO2.3p2_ANT27" \ + f"_smb_climatology_1995-2017.nc" + racmo_path = f'{output_base_path}/atmosphere_forcing/' \ + f'RACMO_climatology_1995-2017' + + # check if the processed racmo input file exists + racmo_clim_file_final = os.path.join(racmo_path, racmo_clim_file) + if not os.path.exists(racmo_clim_file_final): + raise ValueError ("Processed RACMO data does not exist, " + "but it is required as an input file " + "to run this step. Please run `ProcessSmbRacmo` " + "tep prior to running this step.") + + self.add_input_file(filename=racmo_clim_file, + target=os.path.join(racmo_path, + racmo_clim_file)) def run(self): """ @@ -73,6 +92,22 @@ def run(self): section = config['ismip6_ais_atmosphere'] method_remap = section.get('method_remap') + # define file names needed + # input racmo climotology file + racmo_clim_file = f"processed_RACMO2.3p2_ANT27" \ + f"_smb_climatology_1995-2017.nc" + # temporary remapped climatology and anomaly files + clim_ismip6_temp = "clim_ismip6.nc" + remapped_clim_ismip6_temp = "remapped_clim.nc" + remapped_anomaly_ismip6_temp = "remapped_anomaly.nc" + # renamed remapped climatology and anomaly files (final outputs) + output_clim_ismip6 = f"processed_SMB_climatology_1995-2017_" \ + f"{model}_{scenario}.nc" + output_anomaly_ismip6 = f"processed_SMB_{model}_{scenario}_" \ + f"{period_endyear}.nc" + + # combine ismip6 forcing data covering different periods + # into a single file input_file_list = self._files[period_endyear][model][scenario] i = 0 for file in input_file_list: @@ -86,72 +121,100 @@ def run(self): combined_file_temp = "combined.nc" write_netcdf(input_file_combined, combined_file_temp) - # interpolate and rename the ismip6 thermal forcing data - remapped_file_temp = "remapped.nc" # temporary file name + # create smb climatology data over 1995-2017 + # take the time average over the period 1995-2017 + # note: make sure to have the correct time indexing for each + # smb anomaly files on which climatology is calculated. + logger.info(f"Calculating climatology for {model}_{scenario} forcing" + f"over 1995-2017") + args = [f"ncra", "-O", "-F", "-d", "time,1,23", + f"{combined_file_temp}", + f"{clim_ismip6_temp}"] + + subprocess.check_call(args) - # call the function that reads in, remap and rename the file. - logger.info("Calling a remapping function...") - self.remap_ismip6smb_to_mali(combined_file_temp, remapped_file_temp, - mali_mesh_name, mali_mesh_file, - method_remap) + # remap and rename the ismip6 smb climatology + logger.info("Remapping ismip6 climatology onto MALI mesh...") + self.remap_ismip6_smb_to_mali(clim_ismip6_temp, + remapped_clim_ismip6_temp, + mali_mesh_name, + mali_mesh_file, + method_remap) - # call the function that renames the ismip6 variables to MALI variables + # rename the ismip6 variables to MALI variables logger.info("Renaming the ismip6 variables to mali variable names...") - output_file = f"processed_SMB_{model}_{scenario}_{period_endyear}.nc" - self.rename_ismip6smb_to_mali_vars(remapped_file_temp, output_file) + self.rename_ismip6_smb_to_mali_vars(remapped_clim_ismip6_temp, + output_clim_ismip6) + + # remap and rename ismip6 smb anomaly + logger.info(f"Remapping the {model}_{scenario} SMB anomaly onto " + f"MALI mesh") + self.remap_ismip6_smb_to_mali(combined_file_temp, + remapped_anomaly_ismip6_temp, + mali_mesh_name, + mali_mesh_file, + method_remap) + + # rename the ismip6 variables to MALI variables + logger.info("Renaming the ismip6 variables to mali variable names...") + self.rename_ismip6_smb_to_mali_vars(remapped_anomaly_ismip6_temp, + output_anomaly_ismip6) # correct the SMB anomaly field with mali base SMB field - logger.info("Correcting the SMB anomaly field with MALI base SMB " - "from file {mali_mesh_file}...") - self.correct_SMB_anomaly_for_baseSMB(output_file, mali_mesh_file) + logger.info("Correcting the SMB anomaly field for the base SMB " + "climatology 1995-2017 ") + self.correct_smb_anomaly_for_climatology(racmo_clim_file, + output_clim_ismip6, + output_anomaly_ismip6) logger.info("Processing done successfully. " "Removing the temporary files 'remapped.nc' and " "'combined.nc'...") # remove the temporary remapped and combined files - os.remove(remapped_file_temp) + os.remove(remapped_clim_ismip6_temp) + os.remove(remapped_anomaly_ismip6_temp) os.remove(combined_file_temp) + os.remove(clim_ismip6_temp) + os.remove(output_clim_ismip6) # place the output file in appropriate directory - if output_base_path == "NotAvailable": - return - output_path = f'{output_base_path}/atmosphere_forcing/' \ f'{model}_{scenario}/1995-{period_endyear}' if not os.path.exists(output_path): print("Creating a new directory for the output data") os.makedirs(output_path) - src = os.path.join(os.getcwd(), output_file) - dst = os.path.join(output_path, output_file) + src = os.path.join(os.getcwd(), output_anomaly_ismip6) + dst = os.path.join(output_path, output_anomaly_ismip6) shutil.copy(src, dst) - def remap_ismip6smb_to_mali(self, input_file, output_file, mali_mesh_name, - mali_mesh_file, method_remap): + def remap_ismip6_smb_to_mali(self, input_file, output_file, mali_mesh_name, + mali_mesh_file, method_remap): """ - Remap the input ismip6 thermal forcing data onto mali mesh + Remap the input ismip6 smb forcing data onto mali mesh Parameters ---------- input_file: str - ismip6 smb data on its native polarstereo 8km grid + input smb data on its native polarstereo 8km grid output_file : str - ismip6 data remapped on mali mesh + smb data remapped on mali mesh mali_mesh_name : str name of the mali mesh used to name mapping files mali_mesh_file : str, optional The MALI mesh file if mapping file does not exist method_remap : str, optional Remapping method used in building a mapping file - """ - # check if a mapfile exists - mapping_file = f"map_ismip6_8km_to_{mali_mesh_name}_{method_remap}.nc" + """ + mapping_file = f"map_ismip6_8km_to_" \ + f"{mali_mesh_name}_{method_remap}.nc" + # check if mapfile exists if not os.path.exists(mapping_file): # build a mapping file if it doesn't already exist - self.logger.info("Creating a mapping file. " - "Mapping method used: {method_remap}") + self.logger.info(f"Creating a mapping file. " + f"Mapping method used: {method_remap}") build_mapping_file(self.config, self.cores, self.logger, input_file, mapping_file, mali_mesh_file, method_remap) @@ -168,9 +231,9 @@ def remap_ismip6smb_to_mali(self, input_file, output_file, mali_mesh_name, subprocess.check_call(args) - def rename_ismip6smb_to_mali_vars(self, remapped_file_temp, output_file): + def rename_ismip6_smb_to_mali_vars(self, remapped_file_temp, output_file): """ - Rename variables in the remapped ismip6 input data + Rename variables in the remapped source input data to the ones that MALI uses. Parameters @@ -214,27 +277,37 @@ def rename_ismip6smb_to_mali_vars(self, remapped_file_temp, output_file): write_netcdf(ds, output_file) ds.close() - def correct_SMB_anomaly_for_baseSMB(self, output_file, mali_mesh_file): + def correct_smb_anomaly_for_climatology(self, + racmo_clim_file, + output_clim_ismip6_file, + output_file_final): + """ Apply the MALI base SMB to the ismip6 SMB anomaly field Parameters ---------- - output_file : str - remapped ismip6 data renamed on mali mesh - mali_mesh_file : str - initialized MALI mesh file in which the base SMB field exists + racmo_clim_file : str + RACMO climatology file (1995-2017) + output_clim_ismip6_file : str + remapped and renamed ismip6 climatology file + output_file_final : str + climatology-corrected, final ismip6 smb anomaly file """ - ds = xr.open_dataset(output_file) - ds_base = xr.open_dataset(mali_mesh_file) - # get the first time index - ref_smb = ds_base["sfcMassBal"].isel(Time=0) - # correct for the reference smb - ds["sfcMassBal"] = ds["sfcMassBal"] + ref_smb + ds = xr.open_dataset(output_file_final) + ds_racmo_clim = xr.open_dataset(racmo_clim_file) + ds_ismip6_clim = xr.open_dataset(output_clim_ismip6_file) + + # calculate the climatology correction + corr_clim = ds_racmo_clim["sfcMassBal"].isel(Time=0) \ + - ds_ismip6_clim["sfcMassBal"].isel(Time=0) + + # correct the ismip6 smb anomaly + ds["sfcMassBal"] = ds["sfcMassBal"] + corr_clim # write to a new netCDF file - write_netcdf(ds, output_file) + write_netcdf(ds, output_file_final) ds.close() # create a nested dictionary for the ISMIP6 original forcing files including relative path diff --git a/compass/landice/tests/ismip6_forcing/atmosphere/process_smb_racmo.py b/compass/landice/tests/ismip6_forcing/atmosphere/process_smb_racmo.py new file mode 100644 index 0000000000..11714790be --- /dev/null +++ b/compass/landice/tests/ismip6_forcing/atmosphere/process_smb_racmo.py @@ -0,0 +1,256 @@ +import os +import shutil +import subprocess +import xarray as xr +from compass.landice.tests.ismip6_forcing.atmosphere.create_mapfile_smb \ + import build_mapping_file +from mpas_tools.io import write_netcdf +from compass.step import Step + + +class ProcessSmbRacmo(Step): + """ + A step for processing the regional RACMO surface mass balance data + """ + + def __init__(self, test_case, input_file=None): + """ + Create the step + + Parameters + ---------- + test_case : compass.landice.tests.ismip6_forcing.atmosphere.Atmosphere + The test case this step belongs to + + input_file : file name of ismip6 forcing data processed by this step + """ + self.input_file = input_file + super().__init__(test_case=test_case, name='process_smb_racmo', + cores=4, min_cores=1) + + def setup(self): + """ + Set up this step of the test case + """ + config = self.config + section = config['ismip6_ais'] + base_path_mali = section.get('base_path_mali') + mali_mesh_file = section.get('mali_mesh_file') + + section = config['ismip6_ais_atmosphere'] + process_smb_racmo = section.getboolean("process_smb_racmo") + + self.add_input_file(filename=mali_mesh_file, + target=os.path.join(base_path_mali, + mali_mesh_file)) + + if process_smb_racmo: + base_path_racmo = section.get("base_path_racmo") + input_file_list = self._files + self.add_input_file(filename=input_file_list[0], + target=os.path.join(base_path_racmo, + input_file_list[0])) + output_file = f"processed_RACMO2.3p2_ANT27" \ + f"_smb_climatology_1995-2017.nc" + + self.add_output_file(filename=output_file) + + def run(self): + """ + Run this step of the test case + """ + logger = self.logger + config = self.config + + section = config["ismip6_ais"] + mali_mesh_name = section.get("mali_mesh_name") + mali_mesh_file = section.get("mali_mesh_file") + output_base_path = section.get("output_base_path") + + section = config["ismip6_ais_atmosphere"] + method_remap = section.get("method_remap") + process_modern_smb = section.getboolean("process_smb_racmo") + + input_file_list = self._files + racmo_file_temp1 = "RACMO2.3p2_smb_climatology_1995_2017.nc" + racmo_file_temp2 = "RACMO2.3p2_smb_climatology_1995_2017_" \ + "correct_unit.nc" + output_file = f"processed_RACMO2.3p2_ANT27" \ + f"_smb_climatology_1995-2017.nc" + + output_file_final = os.path.join(output_base_path, output_file) + + if os.path.exists(output_file_final): + print("Processed RACMO SMB data already exists in the output path" + f"{output_base_path}. Not processing the source data...") + return + + input_file_list = self._files + # take the time average over the period 1995-2017 + args = ["ncra", "-O", "-F", "-d", "time,17,39", + f"{input_file_list[0]}", + f"{racmo_file_temp1}"] + + subprocess.check_call(args) + + # interpolate the racmo smb data + remapped_file_temp = "remapped.nc" # temporary file name + + # call the function that reads in, remap and rename the file. + logger.info("Calling a remapping function...") + self.remap_source_smb_to_mali(f"{racmo_file_temp1}", + remapped_file_temp, + mali_mesh_name, + mali_mesh_file, + method_remap) + + # perform algebraic operation on the source data in unit of kg/m^2 + # to be in unit of kg/m^2/s + args = ["ncap2", "-O", "-v", "-s", + "sfcMassBal=smb/(60*60*24*365)", + f"{remapped_file_temp}", + f"{racmo_file_temp2}"] + + subprocess.check_call(args) + + # change the unit attribute to kg/m^2/s + args = ["ncatted", "-O", "-a", + "units,sfcMassBal,m,c,'kg m-2 s-1'", + f"{racmo_file_temp2}"] + + subprocess.check_call(args) + + # call the function that renames the ismip6 variables to MALI variables + logger.info("Renaming source variables to mali variable names...") + + self.rename_source_smb_to_mali_vars(racmo_file_temp2, output_file) + + logger.info("Processing done successfully. " + "Removing the temporary files...") + # remove the temporary remapped and combined files + os.remove(remapped_file_temp) + # os.remove(racmo_file_temp1) + # os.remove(racmo_file_temp2) + + # place the output file in appropriate directory + output_path = f'{output_base_path}/atmosphere_forcing/' \ + f'RACMO_climatology_1995-2017' + if not os.path.exists(output_path): + print("Creating a new directory for the output data") + os.makedirs(output_path) + + src = os.path.join(os.getcwd(), output_file) + dst = os.path.join(output_path, output_file) + shutil.copy(src, dst) + + def remap_source_smb_to_mali(self, input_file, output_file, mali_mesh_name, + mali_mesh_file, method_remap): + """ + Remap the input ismip6 smb forcing data onto mali mesh + + Parameters + ---------- + input_file: str + input racmo smb data on its native rotaed pole grid + output_file : str + smb data remapped on mali mesh + mali_mesh_name : str + name of the mali mesh used to name mapping files + mali_mesh_file : str, optional + The MALI mesh file if mapping file does not exist + method_remap : str, optional + Remapping method used in building a mapping file + + """ + mapping_file = f"map_racmo_24km_to_" \ + f"{mali_mesh_name}_{method_remap}.nc" + + # check if a mapfile exists + if not os.path.exists(mapping_file): + # build a mapping file if it doesn't already exist + self.logger.info(f"Creating a mapping file. " + f"Mapping method used: {method_remap}") + build_mapping_file(self.config, self.cores, self.logger, + input_file, mapping_file, mali_mesh_file, + method_remap) + else: + self.logger.info("Mapping file exists. " + "Remapping the input data...") + + args = ["ncremap", + "-i", input_file, + "-o", output_file, + "-m", mapping_file, + "-v", "smb"] + + subprocess.check_call(args) + + def rename_source_smb_to_mali_vars(self, remapped_file_temp, output_file): + """ + Rename variables in the remapped source input data + to the ones that MALI uses. + + Parameters + ---------- + remapped_file_temp : str + temporary ismip6 data remapped on mali mesh + output_file : str + remapped ismip6 data renamed on mali mesh + """ + + # open dataset in 20 years chunk + ds = xr.open_dataset(remapped_file_temp, chunks=dict(time=20), + engine="netcdf4") + + # build dictionary for ismip6 variables that MALI takes in + ismip6_to_mali_dims = dict( + time="Time", + ncol="nCells") + ds = ds.rename(ismip6_to_mali_dims) + + # add xtime variable + xtime = [] + for t_index in range(ds.sizes["Time"]): + date = ds.Time[t_index] + date = date.dt.strftime("%Y-%m-%d_00:00:00") + date = str(date.values).ljust(64) + xtime.append(date) + + ds["xtime"] = ("Time", xtime) + ds["xtime"] = ds.xtime.astype('S') + + # drop unnecessary variables + ds = ds.drop_vars(["height"]) + + # squeeze unnecessary coordinate variable + ds["sfcMassBal"] = ds["sfcMassBal"].squeeze(dim="height") + + # write to a new netCDF file + write_netcdf(ds, output_file) + ds.close() + + def correct_SMB_anomaly_for_baseSMB(self, output_file, mali_mesh_file): + """ + Apply the MALI base SMB to the ismip6 SMB anomaly field + + Parameters + ---------- + output_file : str + remapped ismip6 data renamed on mali mesh + mali_mesh_file : str + initialized MALI mesh file in which the base SMB field exists + """ + + ds = xr.open_dataset(output_file) + ds_base = xr.open_dataset(mali_mesh_file) + # get the first time index + ref_smb = ds_base["sfcMassBal"].isel(Time=0) + # correct for the reference smb + ds["sfcMassBal"] = ds["sfcMassBal"] + ref_smb + + # write to a new netCDF file + write_netcdf(ds, output_file) + ds.close() + + # create a dictionary for the regional climate RACMO dataset and the + _files= ["RACMO2.3p2_ANT27_smb_yearly_1979_2018.nc"] diff --git a/compass/landice/tests/ismip6_forcing/ismip6_forcing.cfg b/compass/landice/tests/ismip6_forcing/ismip6_forcing.cfg index d33eafaacd..6dc430d41e 100644 --- a/compass/landice/tests/ismip6_forcing/ismip6_forcing.cfg +++ b/compass/landice/tests/ismip6_forcing/ismip6_forcing.cfg @@ -36,6 +36,8 @@ mali_mesh_file = NotAvailable # Remapping method used in building a mapping file. Options include: bilinear, neareststod, conserve method_remap = bilinear +# Full path to the RACMO regional climate model SMB data for modern climatology +path_smb_racmo = NotAvailable # config options for ismip6 ocean thermal forcing data test cases [ismip6_ais_ocean_thermal] @@ -43,7 +45,7 @@ method_remap = bilinear # Remapping method used in building a mapping file. Options include: bilinear, neareststod, conserve method_remap = bilinear -# Set to True if the want to process observational thermal forcing data. Set to False if want to process model thermal forcing data. +# Set to True if want to process observational thermal forcing data. Set to False if want to process model thermal forcing data. process_obs_data = False # config options for ismip6 ocean basal test case From 2bddefb8bc37310328be5cead6be971043610bf8 Mon Sep 17 00:00:00 2001 From: hollyhan Date: Sat, 11 Jun 2022 20:59:06 -0600 Subject: [PATCH 31/58] Make the output path required config option --- compass/landice/tests/ismip6_forcing/configure.py | 2 +- .../tests/ismip6_forcing/ocean_basal/process_basal_melt.py | 7 +------ .../ocean_thermal/process_thermal_forcing.py | 3 --- 3 files changed, 2 insertions(+), 10 deletions(-) diff --git a/compass/landice/tests/ismip6_forcing/configure.py b/compass/landice/tests/ismip6_forcing/configure.py index 842426419e..7a28ad36a4 100644 --- a/compass/landice/tests/ismip6_forcing/configure.py +++ b/compass/landice/tests/ismip6_forcing/configure.py @@ -15,7 +15,7 @@ def configure(config, check_model_options): section = 'ismip6_ais' options = ['base_path_ismip6', 'base_path_mali', 'mali_mesh_name', - 'mali_mesh_file'] + 'mali_mesh_file', 'output_base_path'] if check_model_options: options = options + ['model', 'scenario', 'period_endyear'] diff --git a/compass/landice/tests/ismip6_forcing/ocean_basal/process_basal_melt.py b/compass/landice/tests/ismip6_forcing/ocean_basal/process_basal_melt.py index 74465e9a38..1b265468fe 100644 --- a/compass/landice/tests/ismip6_forcing/ocean_basal/process_basal_melt.py +++ b/compass/landice/tests/ismip6_forcing/ocean_basal/process_basal_melt.py @@ -103,10 +103,7 @@ def run(self): os.remove(combined_file_temp) os.remove(remapped_file_temp) - # place the output file in appropriate directory - if output_base_path == "NotAvailable": - return - + # place the output file in appropriate director output_path = f'{output_base_path}/basal_melt/parametrizations/' if not os.path.exists(output_path): print("Creating a new directory for the output data") @@ -116,8 +113,6 @@ def run(self): dst = os.path.join(output_path, output_file) shutil.copy(src, dst) - print("") - def combine_ismip6_inputfiles(self, basin_file, coeff_gamma0_deltaT_file, combined_file_temp): """ diff --git a/compass/landice/tests/ismip6_forcing/ocean_thermal/process_thermal_forcing.py b/compass/landice/tests/ismip6_forcing/ocean_thermal/process_thermal_forcing.py index c42ef2b069..433fb0ca0f 100644 --- a/compass/landice/tests/ismip6_forcing/ocean_thermal/process_thermal_forcing.py +++ b/compass/landice/tests/ismip6_forcing/ocean_thermal/process_thermal_forcing.py @@ -110,9 +110,6 @@ def run(self): os.remove(remapped_file_temp) # place the output file in appropriate directory - if output_base_path == "NotAvailable": - return - if not os.path.exists(output_path): print("Creating a new directory for the output data") os.makedirs(output_path) From 99879478f72752a366ece83490a5f3ec30b71e5f Mon Sep 17 00:00:00 2001 From: hollyhan Date: Tue, 14 Jun 2022 09:06:31 -0600 Subject: [PATCH 32/58] Add a warning about the config option for processing the Racmo data --- .../ismip6_forcing/atmosphere/process_smb.py | 25 ++++++++++--------- .../atmosphere/process_smb_racmo.py | 12 +++++++-- 2 files changed, 23 insertions(+), 14 deletions(-) diff --git a/compass/landice/tests/ismip6_forcing/atmosphere/process_smb.py b/compass/landice/tests/ismip6_forcing/atmosphere/process_smb.py index f768c2864a..47c7fe3c2f 100644 --- a/compass/landice/tests/ismip6_forcing/atmosphere/process_smb.py +++ b/compass/landice/tests/ismip6_forcing/atmosphere/process_smb.py @@ -62,14 +62,6 @@ def setup(self): racmo_path = f'{output_base_path}/atmosphere_forcing/' \ f'RACMO_climatology_1995-2017' - # check if the processed racmo input file exists - racmo_clim_file_final = os.path.join(racmo_path, racmo_clim_file) - if not os.path.exists(racmo_clim_file_final): - raise ValueError ("Processed RACMO data does not exist, " - "but it is required as an input file " - "to run this step. Please run `ProcessSmbRacmo` " - "tep prior to running this step.") - self.add_input_file(filename=racmo_clim_file, target=os.path.join(racmo_path, racmo_clim_file)) @@ -96,10 +88,20 @@ def run(self): # input racmo climotology file racmo_clim_file = f"processed_RACMO2.3p2_ANT27" \ f"_smb_climatology_1995-2017.nc" + racmo_path = f'{output_base_path}/atmosphere_forcing/' \ + f'RACMO_climatology_1995-2017' + # check if the processed racmo input file exists + racmo_clim_file_final = os.path.join(racmo_path, racmo_clim_file) + if not os.path.exists(racmo_clim_file_final): + raise ValueError ("Processed RACMO data does not exist, " + "but it is required as an input file " + "to run this step. Please run `ProcessSmbRacmo` " + "tep prior to running this step.") + # temporary remapped climatology and anomaly files clim_ismip6_temp = "clim_ismip6.nc" - remapped_clim_ismip6_temp = "remapped_clim.nc" - remapped_anomaly_ismip6_temp = "remapped_anomaly.nc" + remapped_clim_ismip6_temp = "remapped_clim_ismip6.nc" + remapped_anomaly_ismip6_temp = "remapped_anomaly_ismip6.nc" # renamed remapped climatology and anomaly files (final outputs) output_clim_ismip6 = f"processed_SMB_climatology_1995-2017_" \ f"{model}_{scenario}.nc" @@ -168,8 +170,7 @@ def run(self): output_anomaly_ismip6) logger.info("Processing done successfully. " - "Removing the temporary files 'remapped.nc' and " - "'combined.nc'...") + "Removing the temporary files...") # remove the temporary remapped and combined files os.remove(remapped_clim_ismip6_temp) os.remove(remapped_anomaly_ismip6_temp) diff --git a/compass/landice/tests/ismip6_forcing/atmosphere/process_smb_racmo.py b/compass/landice/tests/ismip6_forcing/atmosphere/process_smb_racmo.py index 11714790be..165f9702dd 100644 --- a/compass/landice/tests/ismip6_forcing/atmosphere/process_smb_racmo.py +++ b/compass/landice/tests/ismip6_forcing/atmosphere/process_smb_racmo.py @@ -2,10 +2,13 @@ import shutil import subprocess import xarray as xr -from compass.landice.tests.ismip6_forcing.atmosphere.create_mapfile_smb \ - import build_mapping_file +import warnings from mpas_tools.io import write_netcdf from compass.step import Step +from compass.landice.tests.ismip6_forcing.atmosphere.create_mapfile_smb \ + import build_mapping_file + + class ProcessSmbRacmo(Step): @@ -54,6 +57,11 @@ def setup(self): f"_smb_climatology_1995-2017.nc" self.add_output_file(filename=output_file) + else: + warnings.warn(f"'process_smb_racmo' is set to 'False'. This step" + f" will not run unless set 'True' in the" + f" config file.") + def run(self): """ From e987f8c142f9ff5b00b0af498008f4310ab54a2b Mon Sep 17 00:00:00 2001 From: hollyhan Date: Tue, 14 Jun 2022 14:53:35 -0600 Subject: [PATCH 33/58] Add an xtime variable to SMB and OF climatology data --- .../ismip6_forcing/atmosphere/create_mapfile_smb.py | 2 +- .../ismip6_forcing/atmosphere/process_smb_racmo.py | 13 +++---------- .../ocean_thermal/process_thermal_forcing.py | 6 +++++- 3 files changed, 9 insertions(+), 12 deletions(-) diff --git a/compass/landice/tests/ismip6_forcing/atmosphere/create_mapfile_smb.py b/compass/landice/tests/ismip6_forcing/atmosphere/create_mapfile_smb.py index 83b229c122..e60bf24dc9 100644 --- a/compass/landice/tests/ismip6_forcing/atmosphere/create_mapfile_smb.py +++ b/compass/landice/tests/ismip6_forcing/atmosphere/create_mapfile_smb.py @@ -107,7 +107,7 @@ def build_mapping_file(config, cores, logger, ismip6_grid_file, # remove the temporary scripfiles once the mapping file is generated print("Removing the temporary mesh and scripfiles...") - # os.remove(source_grid_scripfile) + os.remove(source_grid_scripfile) os.remove(mali_scripfile) os.remove(mali_mesh_copy) diff --git a/compass/landice/tests/ismip6_forcing/atmosphere/process_smb_racmo.py b/compass/landice/tests/ismip6_forcing/atmosphere/process_smb_racmo.py index 165f9702dd..7cf5bebc48 100644 --- a/compass/landice/tests/ismip6_forcing/atmosphere/process_smb_racmo.py +++ b/compass/landice/tests/ismip6_forcing/atmosphere/process_smb_racmo.py @@ -137,8 +137,8 @@ def run(self): "Removing the temporary files...") # remove the temporary remapped and combined files os.remove(remapped_file_temp) - # os.remove(racmo_file_temp1) - # os.remove(racmo_file_temp2) + os.remove(racmo_file_temp1) + os.remove(racmo_file_temp2) # place the output file in appropriate directory output_path = f'{output_base_path}/atmosphere_forcing/' \ @@ -217,14 +217,7 @@ def rename_source_smb_to_mali_vars(self, remapped_file_temp, output_file): ds = ds.rename(ismip6_to_mali_dims) # add xtime variable - xtime = [] - for t_index in range(ds.sizes["Time"]): - date = ds.Time[t_index] - date = date.dt.strftime("%Y-%m-%d_00:00:00") - date = str(date.values).ljust(64) - xtime.append(date) - - ds["xtime"] = ("Time", xtime) + ds["xtime"] = ("Time", ["1995-01-01_00:00:00".ljust(64)]) ds["xtime"] = ds.xtime.astype('S') # drop unnecessary variables diff --git a/compass/landice/tests/ismip6_forcing/ocean_thermal/process_thermal_forcing.py b/compass/landice/tests/ismip6_forcing/ocean_thermal/process_thermal_forcing.py index 433fb0ca0f..d9a03aa7a5 100644 --- a/compass/landice/tests/ismip6_forcing/ocean_thermal/process_thermal_forcing.py +++ b/compass/landice/tests/ismip6_forcing/ocean_thermal/process_thermal_forcing.py @@ -173,7 +173,7 @@ def rename_ismip6thermalforcing_to_mali_vars(self, remapped_file_temp, config = self.config section = config['ismip6_ais_ocean_thermal'] - process_obs_data = section.get('process_obs_data') + process_obs_data = section.getboolean('process_obs_data') # open dataset in 20 years chunk ds = xr.open_dataset(remapped_file_temp, chunks=dict(time=20), @@ -187,6 +187,10 @@ def rename_ismip6thermalforcing_to_mali_vars(self, remapped_file_temp, ismip6_to_mali_dims = dict( z="nISMIP6OceanLayers", ncol="nCells") + ds["xtime"] = ("Time", ["1995-01-01_00:00:00".ljust(64)]) + ds["xtime"] = ds.xtime.astype('S') + ds["thermal_forcing"] = ds["thermal_forcing"].\ + expand_dims(dim="Time", axis=0) ds = ds.rename(ismip6_to_mali_dims) else: ismip6_to_mali_dims = dict( From 494ea33a3b451820e0ea4727e2bd7fb996ad8798 Mon Sep 17 00:00:00 2001 From: hollyhan Date: Tue, 21 Jun 2022 21:10:07 -0600 Subject: [PATCH 34/58] Add xtime variables to start January 1st of every year This is because every forcing file has different start date e.g., UKESM SMB forcing data has start date of 1995-01-16. We want all forcing files to start at 1995-01-01. --- .../tests/ismip6_forcing/atmosphere/process_smb.py | 10 ++++++++-- .../ismip6_forcing/atmosphere/process_smb_racmo.py | 2 +- .../ocean_thermal/process_thermal_forcing.py | 11 ++++++++--- 3 files changed, 17 insertions(+), 6 deletions(-) diff --git a/compass/landice/tests/ismip6_forcing/atmosphere/process_smb.py b/compass/landice/tests/ismip6_forcing/atmosphere/process_smb.py index 47c7fe3c2f..cffe0ebcde 100644 --- a/compass/landice/tests/ismip6_forcing/atmosphere/process_smb.py +++ b/compass/landice/tests/ismip6_forcing/atmosphere/process_smb.py @@ -1,4 +1,5 @@ import os +import numpy as np import shutil import subprocess import xarray as xr @@ -263,8 +264,13 @@ def rename_ismip6_smb_to_mali_vars(self, remapped_file_temp, output_file): xtime = [] for t_index in range(ds.sizes["Time"]): date = ds.Time[t_index] - date = date.dt.strftime("%Y-%m-%d_00:00:00") - date = str(date.values).ljust(64) + # forcing files do not all match even years, so round up the years + # pandas round function does not work for years, so do it manually + yr = date.dt.year.values + mo = date.dt.month.values + dy = date.dt.day.values + dec_yr = np.around(yr + (30 * (mo - 1) + dy) / 365.0) # approximate ok + date = f"{dec_yr.astype(int)}-01-01_00:00:00".ljust(64) xtime.append(date) ds["xtime"] = ("Time", xtime) diff --git a/compass/landice/tests/ismip6_forcing/atmosphere/process_smb_racmo.py b/compass/landice/tests/ismip6_forcing/atmosphere/process_smb_racmo.py index 7cf5bebc48..490ff7d1b6 100644 --- a/compass/landice/tests/ismip6_forcing/atmosphere/process_smb_racmo.py +++ b/compass/landice/tests/ismip6_forcing/atmosphere/process_smb_racmo.py @@ -217,7 +217,7 @@ def rename_source_smb_to_mali_vars(self, remapped_file_temp, output_file): ds = ds.rename(ismip6_to_mali_dims) # add xtime variable - ds["xtime"] = ("Time", ["1995-01-01_00:00:00".ljust(64)]) + ds["xtime"] = ("Time", ["2015-01-01_00:00:00".ljust(64)]) ds["xtime"] = ds.xtime.astype('S') # drop unnecessary variables diff --git a/compass/landice/tests/ismip6_forcing/ocean_thermal/process_thermal_forcing.py b/compass/landice/tests/ismip6_forcing/ocean_thermal/process_thermal_forcing.py index d9a03aa7a5..b1e1cb706b 100644 --- a/compass/landice/tests/ismip6_forcing/ocean_thermal/process_thermal_forcing.py +++ b/compass/landice/tests/ismip6_forcing/ocean_thermal/process_thermal_forcing.py @@ -187,7 +187,7 @@ def rename_ismip6thermalforcing_to_mali_vars(self, remapped_file_temp, ismip6_to_mali_dims = dict( z="nISMIP6OceanLayers", ncol="nCells") - ds["xtime"] = ("Time", ["1995-01-01_00:00:00".ljust(64)]) + ds["xtime"] = ("Time", ["2015-01-01_00:00:00".ljust(64)]) ds["xtime"] = ds.xtime.astype('S') ds["thermal_forcing"] = ds["thermal_forcing"].\ expand_dims(dim="Time", axis=0) @@ -202,8 +202,13 @@ def rename_ismip6thermalforcing_to_mali_vars(self, remapped_file_temp, xtime = [] for t_index in range(ds.sizes["Time"]): date = ds.Time[t_index] - date = date.dt.strftime("%Y-%m-%d_00:00:00") - date = str(date.values).ljust(64) + # forcing files do not all match even years, so round up the years + # pandas round function does not work for years, so do it manually + yr = date.dt.year.values + mo = date.dt.month.values + dy = date.dt.day.values + dec_yr = np.around(yr + (30 * (mo - 1) + dy) / 365.0) # approximate ok + date = f"{dec_yr.astype(int)}-01-01_00:00:00".ljust(64) xtime.append(date) ds["xtime"] = ("Time", xtime) From b97e491cfee8dc051c384ca9fe6bc61b579e770d Mon Sep 17 00:00:00 2001 From: hollyhan Date: Wed, 10 Aug 2022 09:55:48 -0600 Subject: [PATCH 35/58] Permute dimension of the thermal forcing variable Previously ismip6shelfMelt_3dThermalForcing field is written out with dimensions in the wrong order: ismip6shelfMelt_3dThermalForcing(Time, nISMIP6OceanLayers, nCells) instead of ismip6shelfMelt_3dThermalForcing(Time, nCells, nISMIP6OceanLayers) This commit fixes the wrong dimension. --- .../ocean_thermal/process_thermal_forcing.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/compass/landice/tests/ismip6_forcing/ocean_thermal/process_thermal_forcing.py b/compass/landice/tests/ismip6_forcing/ocean_thermal/process_thermal_forcing.py index b1e1cb706b..9965253238 100644 --- a/compass/landice/tests/ismip6_forcing/ocean_thermal/process_thermal_forcing.py +++ b/compass/landice/tests/ismip6_forcing/ocean_thermal/process_thermal_forcing.py @@ -1,4 +1,5 @@ import os +import numpy as np import shutil import subprocess import xarray as xr @@ -189,8 +190,8 @@ def rename_ismip6thermalforcing_to_mali_vars(self, remapped_file_temp, ncol="nCells") ds["xtime"] = ("Time", ["2015-01-01_00:00:00".ljust(64)]) ds["xtime"] = ds.xtime.astype('S') - ds["thermal_forcing"] = ds["thermal_forcing"].\ - expand_dims(dim="Time", axis=0) + ds["thermal_forcing"] = ds["thermal_forcing"].expand_dims( + dim="Time", axis=0) ds = ds.rename(ismip6_to_mali_dims) else: ismip6_to_mali_dims = dict( @@ -223,6 +224,10 @@ def rename_ismip6thermalforcing_to_mali_vars(self, remapped_file_temp, ds = ds.drop_vars(["z_bnds", "lat_vertices", "area", "lon_vertices", "lat", "lon"]) + # transpose dimension + ds["thermal_forcing"] = ds["thermal_forcing"].transpose( + "Time", "nCells", "nISMIP6OceanLayers") + # write to a new netCDF file write_netcdf(ds, output_file) ds.close() From 44fbcc18601db4ab1ce7a25f1e22151ae4883e0c Mon Sep 17 00:00:00 2001 From: hollyhan Date: Wed, 10 Aug 2022 11:41:32 -0600 Subject: [PATCH 36/58] Fix a directory name for basalmelt coefficient files 'parametrizations' -> 'parameterizations' --- .../ocean_basal/process_basal_melt.py | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/compass/landice/tests/ismip6_forcing/ocean_basal/process_basal_melt.py b/compass/landice/tests/ismip6_forcing/ocean_basal/process_basal_melt.py index 1b265468fe..b08dca74a4 100644 --- a/compass/landice/tests/ismip6_forcing/ocean_basal/process_basal_melt.py +++ b/compass/landice/tests/ismip6_forcing/ocean_basal/process_basal_melt.py @@ -104,7 +104,7 @@ def run(self): os.remove(remapped_file_temp) # place the output file in appropriate director - output_path = f'{output_base_path}/basal_melt/parametrizations/' + output_path = f'{output_base_path}/basal_melt/parameterizations/' if not os.path.exists(output_path): print("Creating a new directory for the output data") os.makedirs(output_path) @@ -213,15 +213,15 @@ def rename_ismip6BasalMelt_to_mali_vars(self, remapped_file_temp, # input files: input uniform melt rate coefficient (gamma0) # and temperature correction per basin _file_basin = "AIS/Ocean_Forcing/imbie2/imbie2_basin_numbers_8km.nc" - _files_coeff = {"AIS/Ocean_Forcing/parametrizations/coeff_gamma0_DeltaT_quadratic_local_5th_pct_PIGL_gamma_calibration.nc", - "AIS/Ocean_Forcing/parametrizations/coeff_gamma0_DeltaT_quadratic_local_5th_percentile.nc", - "AIS/Ocean_Forcing/parametrizations/coeff_gamma0_DeltaT_quadratic_local_95th_pct_PIGL_gamma_calibration.nc", - "AIS/Ocean_Forcing/parametrizations/coeff_gamma0_DeltaT_quadratic_local_95th_percentile.nc", - "AIS/Ocean_Forcing/parametrizations/coeff_gamma0_DeltaT_quadratic_local_median_PIGL_gamma_calibration.nc", - "AIS/Ocean_Forcing/parametrizations/coeff_gamma0_DeltaT_quadratic_local_median.nc", - "AIS/Ocean_Forcing/parametrizations/coeff_gamma0_DeltaT_quadratic_non_local_5th_pct_PIGL_gamma_calibration.nc", - "AIS/Ocean_Forcing/parametrizations/coeff_gamma0_DeltaT_quadratic_non_local_5th_percentile.nc", - "AIS/Ocean_Forcing/parametrizations/coeff_gamma0_DeltaT_quadratic_non_local_95th_pct_PIGL_gamma_calibration.nc", - "AIS/Ocean_Forcing/parametrizations/coeff_gamma0_DeltaT_quadratic_non_local_95th_percentile.nc", - "AIS/Ocean_Forcing/parametrizations/coeff_gamma0_DeltaT_quadratic_non_local_median_PIGL_gamma_calibration.nc", - "AIS/Ocean_Forcing/parametrizations/coeff_gamma0_DeltaT_quadratic_non_local_median.nc"} + _files_coeff = {"AIS/Ocean_Forcing/parameterizations/coeff_gamma0_DeltaT_quadratic_local_5th_pct_PIGL_gamma_calibration.nc", + "AIS/Ocean_Forcing/parameterizations/coeff_gamma0_DeltaT_quadratic_local_5th_percentile.nc", + "AIS/Ocean_Forcing/parameterizations/coeff_gamma0_DeltaT_quadratic_local_95th_pct_PIGL_gamma_calibration.nc", + "AIS/Ocean_Forcing/parameterizations/coeff_gamma0_DeltaT_quadratic_local_95th_percentile.nc", + "AIS/Ocean_Forcing/parameterizations/coeff_gamma0_DeltaT_quadratic_local_median_PIGL_gamma_calibration.nc", + "AIS/Ocean_Forcing/parameterizations/coeff_gamma0_DeltaT_quadratic_local_median.nc", + "AIS/Ocean_Forcing/parameterizations/coeff_gamma0_DeltaT_quadratic_non_local_5th_pct_PIGL_gamma_calibration.nc", + "AIS/Ocean_Forcing/parameterizations/coeff_gamma0_DeltaT_quadratic_non_local_5th_percentile.nc", + "AIS/Ocean_Forcing/parameterizations/coeff_gamma0_DeltaT_quadratic_non_local_95th_pct_PIGL_gamma_calibration.nc", + "AIS/Ocean_Forcing/parameterizations/coeff_gamma0_DeltaT_quadratic_non_local_95th_percentile.nc", + "AIS/Ocean_Forcing/parameterizations/coeff_gamma0_DeltaT_quadratic_non_local_median_PIGL_gamma_calibration.nc", + "AIS/Ocean_Forcing/parameterizations/coeff_gamma0_DeltaT_quadratic_non_local_median.nc"} From 14768462092737f90886710a1ae0b87a4bba14f4 Mon Sep 17 00:00:00 2001 From: hollyhan <33441196+hollyhan@users.noreply.github.com> Date: Wed, 10 Aug 2022 11:53:28 -0600 Subject: [PATCH 37/58] Update ismip6_forcing.cfg --- compass/landice/tests/ismip6_forcing/ismip6_forcing.cfg | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/compass/landice/tests/ismip6_forcing/ismip6_forcing.cfg b/compass/landice/tests/ismip6_forcing/ismip6_forcing.cfg index 6dc430d41e..4b07f076d0 100644 --- a/compass/landice/tests/ismip6_forcing/ismip6_forcing.cfg +++ b/compass/landice/tests/ismip6_forcing/ismip6_forcing.cfg @@ -29,15 +29,17 @@ mali_mesh_name = NotAvailable # MALI mesh file to be used to build mapping file (e.g.Antarctic_8to80km_20220407.nc). User has to supply. mali_mesh_file = NotAvailable - # config options for ismip6 antarctic ice sheet SMB forcing data test cases [ismip6_ais_atmosphere] # Remapping method used in building a mapping file. Options include: bilinear, neareststod, conserve method_remap = bilinear +# Set True to process RACMO modern climatology +process_smb_racmo = True + # Full path to the RACMO regional climate model SMB data for modern climatology -path_smb_racmo = NotAvailable +base_path_racmo = NotAvailable # config options for ismip6 ocean thermal forcing data test cases [ismip6_ais_ocean_thermal] From 9b63d8a265b0837d18dfd555d9a45efcf69bd8d1 Mon Sep 17 00:00:00 2001 From: hollyhan Date: Wed, 10 Aug 2022 19:48:29 -0600 Subject: [PATCH 38/58] Remove adding 'xtime' variable to RACMO processed data --- .../tests/ismip6_forcing/atmosphere/process_smb_racmo.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/compass/landice/tests/ismip6_forcing/atmosphere/process_smb_racmo.py b/compass/landice/tests/ismip6_forcing/atmosphere/process_smb_racmo.py index 490ff7d1b6..edcc1ed1b4 100644 --- a/compass/landice/tests/ismip6_forcing/atmosphere/process_smb_racmo.py +++ b/compass/landice/tests/ismip6_forcing/atmosphere/process_smb_racmo.py @@ -216,10 +216,6 @@ def rename_source_smb_to_mali_vars(self, remapped_file_temp, output_file): ncol="nCells") ds = ds.rename(ismip6_to_mali_dims) - # add xtime variable - ds["xtime"] = ("Time", ["2015-01-01_00:00:00".ljust(64)]) - ds["xtime"] = ds.xtime.astype('S') - # drop unnecessary variables ds = ds.drop_vars(["height"]) From 4b47fd60e91c62009c6a8bb3bf8a28acb8bff449 Mon Sep 17 00:00:00 2001 From: hollyhan Date: Thu, 8 Sep 2022 11:19:02 -0600 Subject: [PATCH 39/58] Add a counter to the name of the temporary files For an unknown reason, ncremapping of gamma0 values from the source files yields wrong gamma0 values in the processed output files when the name of the temporary files are kept as "combined.nc" and "remapped.nc" even when the temporary files are removed at the end of each file processing, requiring the temporary file names to be unique. This commit adds a counter to the temporary file created inside the loop. --- .../ocean_basal/process_basal_melt.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/compass/landice/tests/ismip6_forcing/ocean_basal/process_basal_melt.py b/compass/landice/tests/ismip6_forcing/ocean_basal/process_basal_melt.py index b08dca74a4..07a17aaef4 100644 --- a/compass/landice/tests/ismip6_forcing/ocean_basal/process_basal_melt.py +++ b/compass/landice/tests/ismip6_forcing/ocean_basal/process_basal_melt.py @@ -67,16 +67,19 @@ def run(self): # combine, interpolate and rename the basin file and deltaT0_gamma0 # ismip6 input files - combined_file_temp = "combined.nc" # temporary file names - remapped_file_temp = "remapped.nc" # call the function that combines data # logger.info = ('calling combine_ismip6_inputfiles') input_file_list = self._files_coeff - i = 0 - for file in input_file_list: + + for i, file in enumerate(input_file_list): print(f"processing the input file {os.path.basename(file)}") - i += 1 + # temporary file names. Note: The counter 'i' seems to be necessary + # for 'ncremap' to correctly interpolate the gamma0 values for an + # unknown reason. + combined_file_temp = f"combined_{i}.nc" + remapped_file_temp = f"remapped_{i}.nc" + self.combine_ismip6_inputfiles(os.path.basename(self._file_basin), os.path.basename(file), combined_file_temp) @@ -213,7 +216,7 @@ def rename_ismip6BasalMelt_to_mali_vars(self, remapped_file_temp, # input files: input uniform melt rate coefficient (gamma0) # and temperature correction per basin _file_basin = "AIS/Ocean_Forcing/imbie2/imbie2_basin_numbers_8km.nc" - _files_coeff = {"AIS/Ocean_Forcing/parameterizations/coeff_gamma0_DeltaT_quadratic_local_5th_pct_PIGL_gamma_calibration.nc", + _files_coeff = ["AIS/Ocean_Forcing/parameterizations/coeff_gamma0_DeltaT_quadratic_local_5th_pct_PIGL_gamma_calibration.nc", "AIS/Ocean_Forcing/parameterizations/coeff_gamma0_DeltaT_quadratic_local_5th_percentile.nc", "AIS/Ocean_Forcing/parameterizations/coeff_gamma0_DeltaT_quadratic_local_95th_pct_PIGL_gamma_calibration.nc", "AIS/Ocean_Forcing/parameterizations/coeff_gamma0_DeltaT_quadratic_local_95th_percentile.nc", @@ -224,4 +227,4 @@ def rename_ismip6BasalMelt_to_mali_vars(self, remapped_file_temp, "AIS/Ocean_Forcing/parameterizations/coeff_gamma0_DeltaT_quadratic_non_local_95th_pct_PIGL_gamma_calibration.nc", "AIS/Ocean_Forcing/parameterizations/coeff_gamma0_DeltaT_quadratic_non_local_95th_percentile.nc", "AIS/Ocean_Forcing/parameterizations/coeff_gamma0_DeltaT_quadratic_non_local_median_PIGL_gamma_calibration.nc", - "AIS/Ocean_Forcing/parameterizations/coeff_gamma0_DeltaT_quadratic_non_local_median.nc"} + "AIS/Ocean_Forcing/parameterizations/coeff_gamma0_DeltaT_quadratic_non_local_median.nc"] From d02f18781e2db88ff2a6772970d7e5f3881d1667 Mon Sep 17 00:00:00 2001 From: hollyhan Date: Thu, 8 Sep 2022 15:16:13 -0600 Subject: [PATCH 40/58] Change 'cores' to 'ntasks' etc. --- .../tests/ismip6_forcing/atmosphere/process_smb.py | 6 +++--- .../tests/ismip6_forcing/atmosphere/process_smb_racmo.py | 9 +++------ .../ismip6_forcing/ocean_basal/process_basal_melt.py | 4 ++-- .../ocean_thermal/process_thermal_forcing.py | 4 ++-- 4 files changed, 10 insertions(+), 13 deletions(-) diff --git a/compass/landice/tests/ismip6_forcing/atmosphere/process_smb.py b/compass/landice/tests/ismip6_forcing/atmosphere/process_smb.py index cffe0ebcde..6aee515ba0 100644 --- a/compass/landice/tests/ismip6_forcing/atmosphere/process_smb.py +++ b/compass/landice/tests/ismip6_forcing/atmosphere/process_smb.py @@ -26,8 +26,8 @@ def __init__(self, test_case, input_file=None): input_file : file name of ismip6 forcing data processed by this step """ self.input_file = input_file - super().__init__(test_case=test_case, name='process_smb', cores=4, - min_cores=1) + super().__init__(test_case=test_case, name='process_smb', ntasks=4, + min_tasks=1) def setup(self): """ @@ -217,7 +217,7 @@ def remap_ismip6_smb_to_mali(self, input_file, output_file, mali_mesh_name, # build a mapping file if it doesn't already exist self.logger.info(f"Creating a mapping file. " f"Mapping method used: {method_remap}") - build_mapping_file(self.config, self.cores, self.logger, + build_mapping_file(self.config, self.ntasks, self.logger, input_file, mapping_file, mali_mesh_file, method_remap) else: diff --git a/compass/landice/tests/ismip6_forcing/atmosphere/process_smb_racmo.py b/compass/landice/tests/ismip6_forcing/atmosphere/process_smb_racmo.py index edcc1ed1b4..b74aae3aa7 100644 --- a/compass/landice/tests/ismip6_forcing/atmosphere/process_smb_racmo.py +++ b/compass/landice/tests/ismip6_forcing/atmosphere/process_smb_racmo.py @@ -9,8 +9,6 @@ import build_mapping_file - - class ProcessSmbRacmo(Step): """ A step for processing the regional RACMO surface mass balance data @@ -29,7 +27,7 @@ def __init__(self, test_case, input_file=None): """ self.input_file = input_file super().__init__(test_case=test_case, name='process_smb_racmo', - cores=4, min_cores=1) + ntasks=4, min_tasks=1) def setup(self): """ @@ -62,7 +60,6 @@ def setup(self): f" will not run unless set 'True' in the" f" config file.") - def run(self): """ Run this step of the test case @@ -178,7 +175,7 @@ def remap_source_smb_to_mali(self, input_file, output_file, mali_mesh_name, # build a mapping file if it doesn't already exist self.logger.info(f"Creating a mapping file. " f"Mapping method used: {method_remap}") - build_mapping_file(self.config, self.cores, self.logger, + build_mapping_file(self.config, self.ntasks, self.logger, input_file, mapping_file, mali_mesh_file, method_remap) else: @@ -250,4 +247,4 @@ def correct_SMB_anomaly_for_baseSMB(self, output_file, mali_mesh_file): ds.close() # create a dictionary for the regional climate RACMO dataset and the - _files= ["RACMO2.3p2_ANT27_smb_yearly_1979_2018.nc"] + _files = ["RACMO2.3p2_ANT27_smb_yearly_1979_2018.nc"] diff --git a/compass/landice/tests/ismip6_forcing/ocean_basal/process_basal_melt.py b/compass/landice/tests/ismip6_forcing/ocean_basal/process_basal_melt.py index 07a17aaef4..16d95521b2 100644 --- a/compass/landice/tests/ismip6_forcing/ocean_basal/process_basal_melt.py +++ b/compass/landice/tests/ismip6_forcing/ocean_basal/process_basal_melt.py @@ -24,7 +24,7 @@ def __init__(self, test_case): The test case this step belongs to """ super().__init__(test_case=test_case, name='process_basal_melt', - cores=4, min_cores=1) + ntasks=4, min_tasks=1) def setup(self): """ @@ -164,7 +164,7 @@ def remap_ismip6BasalMelt_to_mali(self, input_file, output_file, if not os.path.exists(mapping_file): # build a mapping file if it doesn't already exist - build_mapping_file(self.config, self.cores, self.logger, + build_mapping_file(self.config, self.ntasks, self.logger, input_file, mapping_file, mali_mesh_file, method_remap) else: diff --git a/compass/landice/tests/ismip6_forcing/ocean_thermal/process_thermal_forcing.py b/compass/landice/tests/ismip6_forcing/ocean_thermal/process_thermal_forcing.py index 9965253238..315ce3886d 100644 --- a/compass/landice/tests/ismip6_forcing/ocean_thermal/process_thermal_forcing.py +++ b/compass/landice/tests/ismip6_forcing/ocean_thermal/process_thermal_forcing.py @@ -24,7 +24,7 @@ def __init__(self, test_case): The test case this step belongs to """ super().__init__(test_case=test_case, name='process_thermal_forcing', - cores=4, min_cores=1) + ntasks=4, min_tasks=1) def setup(self): """ @@ -144,7 +144,7 @@ def remap_ismip6thermalforcing_to_mali(self, input_file, if not os.path.exists(mapping_file): # build a mapping file if it doesn't already exist - build_mapping_file(self.config, self.cores, self.logger, + build_mapping_file(self.config, self.ntasks, self.logger, input_file, mapping_file, mali_mesh_file, method_remap) else: From e21d2fe1f0643605dfd775c60bec5fc94bef5722 Mon Sep 17 00:00:00 2001 From: hollyhan Date: Fri, 9 Sep 2022 10:09:36 -0600 Subject: [PATCH 41/58] Edit doc strings and revert back to using a parallel call parallel call to ESMF_RegridWeightGen --- .../atmosphere/create_mapfile_smb.py | 35 +++++++------------ .../tests/ismip6_forcing/create_mapfile.py | 3 +- 2 files changed, 15 insertions(+), 23 deletions(-) diff --git a/compass/landice/tests/ismip6_forcing/atmosphere/create_mapfile_smb.py b/compass/landice/tests/ismip6_forcing/atmosphere/create_mapfile_smb.py index e60bf24dc9..3d9383282a 100644 --- a/compass/landice/tests/ismip6_forcing/atmosphere/create_mapfile_smb.py +++ b/compass/landice/tests/ismip6_forcing/atmosphere/create_mapfile_smb.py @@ -9,11 +9,12 @@ from pyremap.descriptor import interp_extrap_corners_2d -# function that creates a mapping file from ismip6 grid to mali mesh def build_mapping_file(config, cores, logger, ismip6_grid_file, mapping_file, mali_mesh_file=None, method_remap=None): """ Build a mapping file if it does not exist. + Mapping file is then used to remap the ismip6 source file in polarstero + coordinate to unstructured mali mesh Parameters ---------- @@ -83,27 +84,17 @@ def build_mapping_file(config, cores, logger, ismip6_grid_file, parallel_executable = config.get('parallel', 'parallel_executable') # split the parallel executable into constituents in case it includes flags - # args = parallel_executable.split(' ') - # args.extend(["-n", f"{cores}", - # "ESMF_RegridWeightGen", - # "-s", source_grid_scripfile, - # "-d", mali_scripfile, - # "-w", mapping_file, - # "-m", method_remap, - # "-i", "-64bit_offset", - # "--dst_regional", "--src_regional"]) - # - # check_call(args, logger) - - args = ["ESMF_RegridWeightGen", - "-s", source_grid_scripfile, - "-d", mali_scripfile, - "-w", mapping_file, - "-m", method_remap, - "-i", "-64bit_offset", - "--dst_regional", "--src_regional"] - - subprocess.check_call(args) + args = parallel_executable.split(' ') + args.extend(["-n", f"{cores}", + "ESMF_RegridWeightGen", + "-s", source_grid_scripfile, + "-d", mali_scripfile, + "-w", mapping_file, + "-m", method_remap, + "-i", "-64bit_offset", + "--dst_regional", "--src_regional"]) + + check_call(args, logger) # remove the temporary scripfiles once the mapping file is generated print("Removing the temporary mesh and scripfiles...") diff --git a/compass/landice/tests/ismip6_forcing/create_mapfile.py b/compass/landice/tests/ismip6_forcing/create_mapfile.py index 21527273ab..b0bbab4e0b 100644 --- a/compass/landice/tests/ismip6_forcing/create_mapfile.py +++ b/compass/landice/tests/ismip6_forcing/create_mapfile.py @@ -5,12 +5,13 @@ from mpas_tools.logging import check_call -# function that creates a mapping file from ismip6 grid to mali def build_mapping_file(config, cores, logger, ismip6_grid_file, mapping_file, mali_mesh_file=None, method_remap=None): """ Build a mapping file if it does not exist. + Mapping file is then used to remap the ismip6 source file in polarstero + coordinate to unstructured mali mesh Parameters ---------- From 1d7d9b1162f0ad7a22e75369665460c805a9b5ad Mon Sep 17 00:00:00 2001 From: hollyhan Date: Fri, 9 Sep 2022 12:46:20 -0600 Subject: [PATCH 42/58] Change single quotes to double quotes for consistency Also fix some PEP8 issues and wrong indentations --- .../landice/tests/ismip6_forcing/__init__.py | 9 +- .../ismip6_forcing/atmosphere/__init__.py | 2 +- .../atmosphere/create_mapfile_smb.py | 45 +++++---- .../ismip6_forcing/atmosphere/process_smb.py | 54 +++++------ .../atmosphere/process_smb_racmo.py | 16 ++-- .../landice/tests/ismip6_forcing/configure.py | 8 +- .../tests/ismip6_forcing/create_mapfile.py | 2 +- .../ismip6_forcing/ocean_basal/__init__.py | 2 +- .../ocean_basal/process_basal_melt.py | 52 +++++----- .../ismip6_forcing/ocean_thermal/__init__.py | 6 +- .../ocean_thermal/process_thermal_forcing.py | 95 +++++++++++-------- 11 files changed, 153 insertions(+), 138 deletions(-) diff --git a/compass/landice/tests/ismip6_forcing/__init__.py b/compass/landice/tests/ismip6_forcing/__init__.py index 3272adee26..8cc1734dc6 100644 --- a/compass/landice/tests/ismip6_forcing/__init__.py +++ b/compass/landice/tests/ismip6_forcing/__init__.py @@ -6,16 +6,19 @@ class Ismip6Forcing(TestGroup): """ - A test group for processing the ISMIP6 - ocean and atmosphere forcing data + A test group for processing the ISMIP6 ocean and atmosphere forcing data """ def __init__(self, mpas_core): """ + Create the test group + + Parameters + ---------- mpas_core : compass.landice.Landice the MPAS core that this test group belongs to """ - super().__init__(mpas_core=mpas_core, name='ismip6_forcing') + super().__init__(mpas_core=mpas_core, name="ismip6_forcing") self.add_test_case(Atmosphere(test_group=self)) self.add_test_case(OceanBasal(test_group=self)) diff --git a/compass/landice/tests/ismip6_forcing/atmosphere/__init__.py b/compass/landice/tests/ismip6_forcing/atmosphere/__init__.py index c4dda63043..7a380ce649 100644 --- a/compass/landice/tests/ismip6_forcing/atmosphere/__init__.py +++ b/compass/landice/tests/ismip6_forcing/atmosphere/__init__.py @@ -24,7 +24,7 @@ def __init__(self, test_group): test_group : compass.landice.tests.ismip6_forcing.Ismip6Forcing The test group that this test case belongs to """ - name = 'atmosphere' + name = "atmosphere" subdir = name super().__init__(test_group=test_group, name=name, subdir=subdir) diff --git a/compass/landice/tests/ismip6_forcing/atmosphere/create_mapfile_smb.py b/compass/landice/tests/ismip6_forcing/atmosphere/create_mapfile_smb.py index 3d9383282a..50b24c21f6 100644 --- a/compass/landice/tests/ismip6_forcing/atmosphere/create_mapfile_smb.py +++ b/compass/landice/tests/ismip6_forcing/atmosphere/create_mapfile_smb.py @@ -128,7 +128,7 @@ def create_atm_scrip(source_grid_file, source_grid_scripfile): else: nx = ds.sizes["x"] ny = ds.sizes["y"] - units = 'degrees' + units = "degrees" grid_size = nx * ny @@ -137,28 +137,28 @@ def create_atm_scrip(source_grid_file, source_grid_scripfile): out_file.createDimension("grid_rank", 2) # Variables - grid_center_lat = out_file.createVariable('grid_center_lat', 'f8', - ('grid_size',)) + grid_center_lat = out_file.createVariable("grid_center_lat", "f8", + ("grid_size",)) grid_center_lat.units = units - grid_center_lon = out_file.createVariable('grid_center_lon', 'f8', - ('grid_size',)) + grid_center_lon = out_file.createVariable("grid_center_lon", "f8", + ("grid_size",)) grid_center_lon.units = units - grid_corner_lat = out_file.createVariable('grid_corner_lat', 'f8', - ('grid_size', 'grid_corners')) + grid_corner_lat = out_file.createVariable("grid_corner_lat", "f8", + ("grid_size", "grid_corners")) grid_corner_lat.units = units - grid_corner_lon = out_file.createVariable('grid_corner_lon', 'f8', - ('grid_size', 'grid_corners')) + grid_corner_lon = out_file.createVariable("grid_corner_lon", "f8", + ("grid_size", "grid_corners")) grid_corner_lon.units = units - grid_imask = out_file.createVariable('grid_imask', 'i4', ('grid_size',)) - grid_imask.units = 'unitless' - out_file.createVariable('grid_dims', 'i4', ('grid_rank',)) + grid_imask = out_file.createVariable("grid_imask", "i4", ("grid_size",)) + grid_imask.units = "unitless" + out_file.createVariable("grid_dims", "i4", ("grid_rank",)) - out_file.variables['grid_center_lat'][:] = ds.lat.values.flat - out_file.variables['grid_center_lon'][:] = ds.lon.values.flat - out_file.variables['grid_dims'][:] = [nx, ny] - out_file.variables['grid_imask'][:] = 1 + out_file.variables["grid_center_lat"][:] = ds.lat.values.flat + out_file.variables["grid_center_lon"][:] = ds.lon.values.flat + out_file.variables["grid_dims"][:] = [nx, ny] + out_file.variables["grid_imask"][:] = 1 - if 'lat_bnds' in ds and 'lon_bnds' in ds: + if "lat_bnds" in ds and "lon_bnds" in ds: lat_corner = ds.lat_bnds if "time" in lat_corner.dims: lat_corner = lat_corner.isel(time=0) @@ -169,21 +169,24 @@ def create_atm_scrip(source_grid_file, source_grid_scripfile): lat_corner = lat_corner.values lon_corner = lon_corner.values - else: # this part is used for RACMO as it does not have lat_bnds & lon_bnds + else: + # this part is used for RACMO as it does not have lat_bnds & lon_bnds lat_corner = _unwrap_corners(interp_extrap_corners_2d(ds.lat.values)) lon_corner = _unwrap_corners(interp_extrap_corners_2d(ds.lon.values)) grid_corner_lat = lat_corner.reshape((grid_size, 4)) grid_corner_lon = lon_corner.reshape((grid_size, 4)) - out_file.variables['grid_corner_lat'][:] = grid_corner_lat - out_file.variables['grid_corner_lon'][:] = grid_corner_lon + out_file.variables["grid_corner_lat"][:] = grid_corner_lat + out_file.variables["grid_corner_lon"][:] = grid_corner_lon out_file.close() def _unwrap_corners(in_field): - """Turn a 2D array of corners into an array of rectangular mesh elements""" + """ + Turn a 2D array of corners into an array of rectangular mesh elements + """ out_field = np.zeros(((in_field.shape[0] - 1) * (in_field.shape[1] - 1), 4)) # corners are counterclockwise diff --git a/compass/landice/tests/ismip6_forcing/atmosphere/process_smb.py b/compass/landice/tests/ismip6_forcing/atmosphere/process_smb.py index 6aee515ba0..1cd96ee2f8 100644 --- a/compass/landice/tests/ismip6_forcing/atmosphere/process_smb.py +++ b/compass/landice/tests/ismip6_forcing/atmosphere/process_smb.py @@ -26,7 +26,7 @@ def __init__(self, test_case, input_file=None): input_file : file name of ismip6 forcing data processed by this step """ self.input_file = input_file - super().__init__(test_case=test_case, name='process_smb', ntasks=4, + super().__init__(test_case=test_case, name="process_smb", ntasks=4, min_tasks=1) def setup(self): @@ -34,11 +34,11 @@ def setup(self): Set up this step of the test case """ config = self.config - section = config['ismip6_ais'] - base_path_ismip6 = section.get('base_path_ismip6') - base_path_mali = section.get('base_path_mali') - output_base_path = section.get('output_base_path') - mali_mesh_file = section.get('mali_mesh_file') + section = config["ismip6_ais"] + base_path_ismip6 = section.get("base_path_ismip6") + base_path_mali = section.get("base_path_mali") + output_base_path = section.get("output_base_path") + mali_mesh_file = section.get("mali_mesh_file") period_endyear = section.get("period_endyear") model = section.get("model") scenario = section.get("scenario") @@ -59,9 +59,9 @@ def setup(self): # add processed racmo data as input as it is needed for smb correction racmo_clim_file = f"processed_RACMO2.3p2_ANT27" \ - f"_smb_climatology_1995-2017.nc" - racmo_path = f'{output_base_path}/atmosphere_forcing/' \ - f'RACMO_climatology_1995-2017' + f"_smb_climatology_1995-2017.nc" + racmo_path = f"{output_base_path}/atmosphere_forcing/" \ + f"RACMO_climatology_1995-2017" self.add_input_file(filename=racmo_clim_file, target=os.path.join(racmo_path, @@ -74,30 +74,30 @@ def run(self): logger = self.logger config = self.config - section = config['ismip6_ais'] - mali_mesh_name = section.get('mali_mesh_name') - mali_mesh_file = section.get('mali_mesh_file') + section = config["ismip6_ais"] + mali_mesh_name = section.get("mali_mesh_name") + mali_mesh_file = section.get("mali_mesh_file") period_endyear = section.get("period_endyear") model = section.get("model") scenario = section.get("scenario") - output_base_path = section.get('output_base_path') + output_base_path = section.get("output_base_path") - section = config['ismip6_ais_atmosphere'] - method_remap = section.get('method_remap') + section = config["ismip6_ais_atmosphere"] + method_remap = section.get("method_remap") # define file names needed # input racmo climotology file racmo_clim_file = f"processed_RACMO2.3p2_ANT27" \ - f"_smb_climatology_1995-2017.nc" - racmo_path = f'{output_base_path}/atmosphere_forcing/' \ - f'RACMO_climatology_1995-2017' + f"_smb_climatology_1995-2017.nc" + racmo_path = f"{output_base_path}/atmosphere_forcing/" \ + f"RACMO_climatology_1995-2017" # check if the processed racmo input file exists racmo_clim_file_final = os.path.join(racmo_path, racmo_clim_file) if not os.path.exists(racmo_clim_file_final): - raise ValueError ("Processed RACMO data does not exist, " - "but it is required as an input file " - "to run this step. Please run `ProcessSmbRacmo` " - "tep prior to running this step.") + raise ValueError("Processed RACMO data does not exist, " + "but it is required as an input file " + "to run this step. Please run `ProcessSmbRacmo` " + "step prior to running this step.") # temporary remapped climatology and anomaly files clim_ismip6_temp = "clim_ismip6.nc" @@ -165,7 +165,7 @@ def run(self): # correct the SMB anomaly field with mali base SMB field logger.info("Correcting the SMB anomaly field for the base SMB " - "climatology 1995-2017 ") + "climatology 1995-2017...") self.correct_smb_anomaly_for_climatology(racmo_clim_file, output_clim_ismip6, output_anomaly_ismip6) @@ -180,10 +180,10 @@ def run(self): os.remove(output_clim_ismip6) # place the output file in appropriate directory - output_path = f'{output_base_path}/atmosphere_forcing/' \ - f'{model}_{scenario}/1995-{period_endyear}' + output_path = f"{output_base_path}/atmosphere_forcing/" \ + f"{model}_{scenario}/1995-{period_endyear}" if not os.path.exists(output_path): - print("Creating a new directory for the output data") + print("Creating a new directory for the output data...") os.makedirs(output_path) src = os.path.join(os.getcwd(), output_anomaly_ismip6) @@ -269,7 +269,7 @@ def rename_ismip6_smb_to_mali_vars(self, remapped_file_temp, output_file): yr = date.dt.year.values mo = date.dt.month.values dy = date.dt.day.values - dec_yr = np.around(yr + (30 * (mo - 1) + dy) / 365.0) # approximate ok + dec_yr = np.around(yr + (30 * (mo - 1) + dy) / 365.0) date = f"{dec_yr.astype(int)}-01-01_00:00:00".ljust(64) xtime.append(date) diff --git a/compass/landice/tests/ismip6_forcing/atmosphere/process_smb_racmo.py b/compass/landice/tests/ismip6_forcing/atmosphere/process_smb_racmo.py index b74aae3aa7..3d1159c634 100644 --- a/compass/landice/tests/ismip6_forcing/atmosphere/process_smb_racmo.py +++ b/compass/landice/tests/ismip6_forcing/atmosphere/process_smb_racmo.py @@ -34,11 +34,11 @@ def setup(self): Set up this step of the test case """ config = self.config - section = config['ismip6_ais'] - base_path_mali = section.get('base_path_mali') - mali_mesh_file = section.get('mali_mesh_file') + section = config["ismip6_ais"] + base_path_mali = section.get("base_path_mali") + mali_mesh_file = section.get("mali_mesh_file") - section = config['ismip6_ais_atmosphere'] + section = config["ismip6_ais_atmosphere"] process_smb_racmo = section.getboolean("process_smb_racmo") self.add_input_file(filename=mali_mesh_file, @@ -74,9 +74,7 @@ def run(self): section = config["ismip6_ais_atmosphere"] method_remap = section.get("method_remap") - process_modern_smb = section.getboolean("process_smb_racmo") - input_file_list = self._files racmo_file_temp1 = "RACMO2.3p2_smb_climatology_1995_2017.nc" racmo_file_temp2 = "RACMO2.3p2_smb_climatology_1995_2017_" \ "correct_unit.nc" @@ -138,8 +136,8 @@ def run(self): os.remove(racmo_file_temp2) # place the output file in appropriate directory - output_path = f'{output_base_path}/atmosphere_forcing/' \ - f'RACMO_climatology_1995-2017' + output_path = f"{output_base_path}/atmosphere_forcing/" \ + f"RACMO_climatology_1995-2017" if not os.path.exists(output_path): print("Creating a new directory for the output data") os.makedirs(output_path) @@ -223,7 +221,7 @@ def rename_source_smb_to_mali_vars(self, remapped_file_temp, output_file): write_netcdf(ds, output_file) ds.close() - def correct_SMB_anomaly_for_baseSMB(self, output_file, mali_mesh_file): + def correct_smb_anomaly_for_base_smb(self, output_file, mali_mesh_file): """ Apply the MALI base SMB to the ismip6 SMB anomaly field diff --git a/compass/landice/tests/ismip6_forcing/configure.py b/compass/landice/tests/ismip6_forcing/configure.py index 7a28ad36a4..e5d3b23f42 100644 --- a/compass/landice/tests/ismip6_forcing/configure.py +++ b/compass/landice/tests/ismip6_forcing/configure.py @@ -13,11 +13,11 @@ def configure(config, check_model_options): """ - section = 'ismip6_ais' - options = ['base_path_ismip6', 'base_path_mali', 'mali_mesh_name', - 'mali_mesh_file', 'output_base_path'] + section = "ismip6_ais" + options = ["base_path_ismip6", "base_path_mali", "mali_mesh_name", + "mali_mesh_file", "output_base_path"] if check_model_options: - options = options + ['model', 'scenario', 'period_endyear'] + options = options + ["model", "scenario", "period_endyear"] for option in options: value = config.get(section=section, option=option) diff --git a/compass/landice/tests/ismip6_forcing/create_mapfile.py b/compass/landice/tests/ismip6_forcing/create_mapfile.py index b0bbab4e0b..2b862985d4 100644 --- a/compass/landice/tests/ismip6_forcing/create_mapfile.py +++ b/compass/landice/tests/ismip6_forcing/create_mapfile.py @@ -82,7 +82,7 @@ def build_mapping_file(config, cores, logger, ismip6_grid_file, "--method. Available options are 'bilinear'," "'neareststod', 'conserve'.") - parallel_executable = config.get('parallel', 'parallel_executable') + parallel_executable = config.get("parallel", "parallel_executable") # split the parallel executable into constituents in case it includes flags args = parallel_executable.split(' ') args.extend(["-n", f"{cores}", diff --git a/compass/landice/tests/ismip6_forcing/ocean_basal/__init__.py b/compass/landice/tests/ismip6_forcing/ocean_basal/__init__.py index 594ff38950..f4732ba263 100644 --- a/compass/landice/tests/ismip6_forcing/ocean_basal/__init__.py +++ b/compass/landice/tests/ismip6_forcing/ocean_basal/__init__.py @@ -22,7 +22,7 @@ def __init__(self, test_group): test_group : compass.landice.tests.ismip6_forcing.Ismip6Forcing The test group that this test case belongs to """ - name = 'ocean_basal' + name = "ocean_basal" super().__init__(test_group=test_group, name=name) step = ProcessBasalMelt(test_case=self) diff --git a/compass/landice/tests/ismip6_forcing/ocean_basal/process_basal_melt.py b/compass/landice/tests/ismip6_forcing/ocean_basal/process_basal_melt.py index 16d95521b2..d09bedd250 100644 --- a/compass/landice/tests/ismip6_forcing/ocean_basal/process_basal_melt.py +++ b/compass/landice/tests/ismip6_forcing/ocean_basal/process_basal_melt.py @@ -23,7 +23,7 @@ def __init__(self, test_case): test_case : compass.landice.tests.ismip6_forcing.ocean_basal.OceanBasal The test case this step belongs to """ - super().__init__(test_case=test_case, name='process_basal_melt', + super().__init__(test_case=test_case, name="process_basal_melt", ntasks=4, min_tasks=1) def setup(self): @@ -31,12 +31,12 @@ def setup(self): Set up this step of the test case """ config = self.config - section = config['ismip6_ais'] - base_path_ismip6 = section.get('base_path_ismip6') - base_path_mali = section.get('base_path_mali') - mali_mesh_file = section.get('mali_mesh_file') + section = config["ismip6_ais"] + base_path_ismip6 = section.get("base_path_ismip6") + base_path_mali = section.get("base_path_mali") + mali_mesh_file = section.get("mali_mesh_file") - section = config['ismip6_ais_ocean_basal'] + section = config["ismip6_ais_ocean_basal"] self.add_input_file(filename=mali_mesh_file, target=os.path.join(base_path_mali, mali_mesh_file)) @@ -57,13 +57,13 @@ def run(self): """ # logger = self.logger config = self.config - section = config['ismip6_ais'] - mali_mesh_name = section.get('mali_mesh_name') - mali_mesh_file = section.get('mali_mesh_file') - output_base_path = section.get('output_base_path') + section = config["ismip6_ais"] + mali_mesh_name = section.get("mali_mesh_name") + mali_mesh_file = section.get("mali_mesh_file") + output_base_path = section.get("output_base_path") - section = config['ismip6_ais_ocean_basal'] - method_remap = section.get('method_remap') + section = config["ismip6_ais_ocean_basal"] + method_remap = section.get("method_remap") # combine, interpolate and rename the basin file and deltaT0_gamma0 # ismip6 input files @@ -86,28 +86,28 @@ def run(self): # remap the input forcing file. print("Calling the remapping function...") - self.remap_ismip6BasalMelt_to_mali(combined_file_temp, - remapped_file_temp, - mali_mesh_name, - mali_mesh_file, method_remap) + self.remap_ismip6_basal_melt_to_mali_vars(combined_file_temp, + remapped_file_temp, + mali_mesh_name, + mali_mesh_file, + method_remap) output_file = f"processed_basin_and_{os.path.basename(file)}" # rename the ismip6 variables to MALI variables print("Renaming the ismip6 variables to mali variable names...") - self.rename_ismip6BasalMelt_to_mali_vars(remapped_file_temp, - output_file) + self.rename_ismip6_basal_melt_to_mali_vars(remapped_file_temp, + output_file) print("Remapping and renamping process done successfully. " - "Removing the temporary files 'combined.nc' " - "and 'remapped.nc'") + "Removing the temporary files...") # remove the temporary combined file os.remove(combined_file_temp) os.remove(remapped_file_temp) # place the output file in appropriate director - output_path = f'{output_base_path}/basal_melt/parameterizations/' + output_path = f"{output_base_path}/basal_melt/parameterizations/" if not os.path.exists(output_path): print("Creating a new directory for the output data") os.makedirs(output_path) @@ -138,9 +138,9 @@ def combine_ismip6_inputfiles(self, basin_file, coeff_gamma0_deltaT_file, ds["ismip6shelfMelt_basin"] = ds_basin.basinNumber write_netcdf(ds, combined_file_temp) - def remap_ismip6BasalMelt_to_mali(self, input_file, output_file, - mali_mesh_name, mali_mesh_file, - method_remap): + def remap_ismip6_basal_melt_to_mali_vars(self, input_file, output_file, + mali_mesh_name, mali_mesh_file, + method_remap): """ Remap the input ismip6 basal melt data onto mali mesh @@ -178,8 +178,8 @@ def remap_ismip6BasalMelt_to_mali(self, input_file, output_file, subprocess.check_call(args) - def rename_ismip6BasalMelt_to_mali_vars(self, remapped_file_temp, - output_file): + def rename_ismip6_basal_melt_to_mali_vars(self, remapped_file_temp, + output_file): """ Rename variables in the remapped ismip6 input data to the ones that MALI uses. diff --git a/compass/landice/tests/ismip6_forcing/ocean_thermal/__init__.py b/compass/landice/tests/ismip6_forcing/ocean_thermal/__init__.py index 97b58717bc..958b5fd4c0 100644 --- a/compass/landice/tests/ismip6_forcing/ocean_thermal/__init__.py +++ b/compass/landice/tests/ismip6_forcing/ocean_thermal/__init__.py @@ -22,7 +22,7 @@ def __init__(self, test_group): test_group : compass.landice.tests.ismip6_forcing.Ismip6Forcing The test group that this test case belongs to """ - name = 'ocean_thermal' + name = "ocean_thermal" super().__init__(test_group=test_group, name=name) step = ProcessThermalForcing(test_case=self) @@ -32,8 +32,8 @@ def configure(self): """ Configures test case """ - process_obs_data = self.config.getboolean('ismip6_ais_ocean_thermal', - 'process_obs_data') + process_obs_data = self.config.getboolean("ismip6_ais_ocean_thermal", + "process_obs_data") check_model_options = not process_obs_data configure_testgroup(config=self.config, check_model_options=check_model_options) diff --git a/compass/landice/tests/ismip6_forcing/ocean_thermal/process_thermal_forcing.py b/compass/landice/tests/ismip6_forcing/ocean_thermal/process_thermal_forcing.py index 315ce3886d..33c5854bab 100644 --- a/compass/landice/tests/ismip6_forcing/ocean_thermal/process_thermal_forcing.py +++ b/compass/landice/tests/ismip6_forcing/ocean_thermal/process_thermal_forcing.py @@ -23,7 +23,7 @@ def __init__(self, test_case): OceanThermal The test case this step belongs to """ - super().__init__(test_case=test_case, name='process_thermal_forcing', + super().__init__(test_case=test_case, name="process_thermal_forcing", ntasks=4, min_tasks=1) def setup(self): @@ -31,15 +31,15 @@ def setup(self): Set up this step of the test case """ config = self.config - section = config['ismip6_ais'] - base_path_ismip6 = section.get('base_path_ismip6') - base_path_mali = section.get('base_path_mali') - mali_mesh_file = section.get('mali_mesh_file') + section = config["ismip6_ais"] + base_path_ismip6 = section.get("base_path_ismip6") + base_path_mali = section.get("base_path_mali") + mali_mesh_file = section.get("mali_mesh_file") period_endyear = section.get("period_endyear") model = section.get("model") scenario = section.get("scenario") - section = config['ismip6_ais_ocean_thermal'] + section = config["ismip6_ais_ocean_thermal"] process_obs_data = section.getboolean("process_obs_data") self.add_input_file(filename=mali_mesh_file, @@ -51,10 +51,12 @@ def setup(self): output_file = f"processed_obs_TF_1995-2017_8km_x_60m.nc" else: input_file = self._files[period_endyear][model][scenario] - output_file = f"processed_TF_{model}_{scenario}_{period_endyear}.nc" + output_file = f"processed_TF_{model}_{scenario}_" \ + f"{period_endyear}.nc" self.add_input_file(filename=os.path.basename(input_file[0]), - target=os.path.join(base_path_ismip6, input_file[0])) + target=os.path.join(base_path_ismip6, + input_file[0])) self.add_output_file(filename=output_file) def run(self): @@ -64,28 +66,29 @@ def run(self): logger = self.logger config = self.config - section = config['ismip6_ais'] - mali_mesh_name = section.get('mali_mesh_name') - mali_mesh_file = section.get('mali_mesh_file') + section = config["ismip6_ais"] + mali_mesh_name = section.get("mali_mesh_name") + mali_mesh_file = section.get("mali_mesh_file") period_endyear = section.get("period_endyear") model = section.get("model") scenario = section.get("scenario") - output_base_path = section.get('output_base_path') + output_base_path = section.get("output_base_path") - section = config['ismip6_ais_ocean_thermal'] - method_remap = section.get('method_remap') - process_obs_data = section.getboolean('process_obs_data') + section = config["ismip6_ais_ocean_thermal"] + method_remap = section.get("method_remap") + process_obs_data = section.getboolean("process_obs_data") if process_obs_data: input_file_list = self._file_obs - output_file = f'processed_obs_TF_1995-2017_8km_x_60m.nc' - output_path = f'{output_base_path}/ocean_thermal_forcing/'\ - f'obs' + output_file = f"processed_obs_TF_1995-2017_8km_x_60m.nc" + output_path = f"{output_base_path}/ocean_thermal_forcing/"\ + f"obs" else: input_file_list = self._files[period_endyear][model][scenario] - output_file = f'processed_TF_{model}_{scenario}_{period_endyear}.nc' - output_path = f'{output_base_path}/ocean_thermal_forcing/' \ - f'{model}_{scenario}/1995-{period_endyear}' + output_file = f"processed_TF_" \ + f"{model}_{scenario}_{period_endyear}.nc" + output_path = f"{output_base_path}/ocean_thermal_forcing/" \ + f"{model}_{scenario}/1995-{period_endyear}" input_file = os.path.basename(input_file_list[0]) @@ -94,34 +97,38 @@ def run(self): # call the function that reads in, remap and rename the file. print("Calling a remapping function...") - self.remap_ismip6thermalforcing_to_mali(input_file, - remapped_file_temp, - mali_mesh_name, - mali_mesh_file, method_remap) + self.remap_ismip6_thermal_forcing_to_mali_vars(input_file, + remapped_file_temp, + mali_mesh_name, + mali_mesh_file, + method_remap) # call the function that renames the ismip6 variables to MALI variables print("Renaming the ismip6 variables to mali variable names...") - self.rename_ismip6thermalforcing_to_mali_vars(remapped_file_temp, - output_file) + self.rename_ismip6_thermal_forcing_to_mali_vars(remapped_file_temp, + output_file) print("Remapping and renamping process done successfully. " - "Removing the temporary file 'remapped.nc'") + "Removing the temporary file 'remapped.nc'...") # remove the temporary combined file os.remove(remapped_file_temp) # place the output file in appropriate directory if not os.path.exists(output_path): - print("Creating a new directory for the output data") + print("Creating a new directory for the output data...") os.makedirs(output_path) src = os.path.join(os.getcwd(), output_file) dst = os.path.join(output_path, output_file) shutil.copy(src, dst) - def remap_ismip6thermalforcing_to_mali(self, input_file, - output_file, mali_mesh_name, - mali_mesh_file, method_remap): + def remap_ismip6_thermal_forcing_to_mali_vars(self, + input_file, + output_file, + mali_mesh_name, + mali_mesh_file, + method_remap): """ Remap the input ismip6 thermal forcing data onto mali mesh @@ -158,8 +165,9 @@ def remap_ismip6thermalforcing_to_mali(self, input_file, subprocess.check_call(args) - def rename_ismip6thermalforcing_to_mali_vars(self, remapped_file_temp, - output_file): + def rename_ismip6_thermal_forcing_to_mali_vars(self, + remapped_file_temp, + output_file): """ Rename variables in the remapped ismip6 input data to the ones that MALI uses. @@ -173,15 +181,15 @@ def rename_ismip6thermalforcing_to_mali_vars(self, remapped_file_temp, """ config = self.config - section = config['ismip6_ais_ocean_thermal'] - process_obs_data = section.getboolean('process_obs_data') + section = config["ismip6_ais_ocean_thermal"] + process_obs_data = section.getboolean("process_obs_data") # open dataset in 20 years chunk ds = xr.open_dataset(remapped_file_temp, chunks=dict(time=20), engine="netcdf4") ds["ismip6shelfMelt_zOcean"] = ds.z - ds = ds.drop_vars('z') # dropping 'z' while it's still called 'z' + ds = ds.drop_vars("z") # dropping 'z' while it's still called 'z' # build dictionary for ismip6 variables that MALI takes in if process_obs_data: @@ -189,7 +197,7 @@ def rename_ismip6thermalforcing_to_mali_vars(self, remapped_file_temp, z="nISMIP6OceanLayers", ncol="nCells") ds["xtime"] = ("Time", ["2015-01-01_00:00:00".ljust(64)]) - ds["xtime"] = ds.xtime.astype('S') + ds["xtime"] = ds.xtime.astype("S") ds["thermal_forcing"] = ds["thermal_forcing"].expand_dims( dim="Time", axis=0) ds = ds.rename(ismip6_to_mali_dims) @@ -203,12 +211,14 @@ def rename_ismip6thermalforcing_to_mali_vars(self, remapped_file_temp, xtime = [] for t_index in range(ds.sizes["Time"]): date = ds.Time[t_index] - # forcing files do not all match even years, so round up the years - # pandas round function does not work for years, so do it manually + # forcing files do not all match even years, + # so round up the years + # pandas round function does not work for years, + # so do it manually yr = date.dt.year.values mo = date.dt.month.values dy = date.dt.day.values - dec_yr = np.around(yr + (30 * (mo - 1) + dy) / 365.0) # approximate ok + dec_yr = np.around(yr + (30 * (mo - 1) + dy) / 365.0) date = f"{dec_yr.astype(int)}-01-01_00:00:00".ljust(64) xtime.append(date) @@ -225,7 +235,8 @@ def rename_ismip6thermalforcing_to_mali_vars(self, remapped_file_temp, "lon_vertices", "lat", "lon"]) # transpose dimension - ds["thermal_forcing"] = ds["thermal_forcing"].transpose( + ds["ismip6shelfMelt_3dThermalForcing"] = \ + ds["ismip6shelfMelt_3dThermalForcing"].transpose( "Time", "nCells", "nISMIP6OceanLayers") # write to a new netCDF file From a18b4a98f76d1df0aba33f150b9a4c1e10bd810b Mon Sep 17 00:00:00 2001 From: hollyhan Date: Fri, 9 Sep 2022 14:18:20 -0600 Subject: [PATCH 43/58] Use 'check_call' from mpas tool and also pass 'logger' --- .../atmosphere/create_mapfile_smb.py | 16 +++++----- .../ismip6_forcing/atmosphere/process_smb.py | 8 +++-- .../atmosphere/process_smb_racmo.py | 21 +++++++------ .../tests/ismip6_forcing/create_mapfile.py | 18 ++++++----- .../ocean_basal/process_basal_melt.py | 30 ++++++++++++------- .../ocean_thermal/process_thermal_forcing.py | 18 ++++++----- 6 files changed, 65 insertions(+), 46 deletions(-) diff --git a/compass/landice/tests/ismip6_forcing/atmosphere/create_mapfile_smb.py b/compass/landice/tests/ismip6_forcing/atmosphere/create_mapfile_smb.py index 50b24c21f6..ead5184d14 100644 --- a/compass/landice/tests/ismip6_forcing/atmosphere/create_mapfile_smb.py +++ b/compass/landice/tests/ismip6_forcing/atmosphere/create_mapfile_smb.py @@ -1,7 +1,6 @@ import os import numpy as np import shutil -import subprocess import netCDF4 import xarray as xr from mpas_tools.scrip.from_mpas import scrip_from_mpas @@ -35,13 +34,14 @@ def build_mapping_file(config, cores, logger, ismip6_grid_file, """ if os.path.exists(mapping_file): - print("Mapping file exists. Not building a new one.") + logger.info(f"Mapping file exists. Not building a new one.") return # create the scrip files if mapping file does not exist - print("Mapping file does not exist. Building one based on the " - "input/ouptut meshes") - print("Creating temporary scrip files for source and destination grids...") + logger.info(f"Mapping file does not exist. Building one based on the" + f" input/ouptut meshes") + logger.info(f"Creating temporary scrip files for source and " + f"destination grids...") if mali_mesh_file is None: raise ValueError("Mapping file does not exist. To build one, Mali " @@ -69,7 +69,7 @@ def build_mapping_file(config, cores, logger, ismip6_grid_file, "--file", mali_mesh_copy, "--proj", ismip6_projection] - subprocess.check_call(args) + check_call(args, logger=logger) # create a MALI mesh scripfile if mapping file does not exist scrip_from_mpas(mali_mesh_copy, mali_scripfile) @@ -94,10 +94,10 @@ def build_mapping_file(config, cores, logger, ismip6_grid_file, "-i", "-64bit_offset", "--dst_regional", "--src_regional"]) - check_call(args, logger) + check_call(args, logger=logger) # remove the temporary scripfiles once the mapping file is generated - print("Removing the temporary mesh and scripfiles...") + logger.info(f"Removing the temporary mesh and scripfiles...") os.remove(source_grid_scripfile) os.remove(mali_scripfile) os.remove(mali_mesh_copy) diff --git a/compass/landice/tests/ismip6_forcing/atmosphere/process_smb.py b/compass/landice/tests/ismip6_forcing/atmosphere/process_smb.py index 1cd96ee2f8..0cc919b3e9 100644 --- a/compass/landice/tests/ismip6_forcing/atmosphere/process_smb.py +++ b/compass/landice/tests/ismip6_forcing/atmosphere/process_smb.py @@ -1,11 +1,11 @@ import os import numpy as np import shutil -import subprocess import xarray as xr from compass.landice.tests.ismip6_forcing.atmosphere.create_mapfile_smb \ import build_mapping_file from mpas_tools.io import write_netcdf +from mpas_tools.logging import check_call from compass.step import Step @@ -134,7 +134,7 @@ def run(self): f"{combined_file_temp}", f"{clim_ismip6_temp}"] - subprocess.check_call(args) + check_call(args, logger=logger) # remap and rename the ismip6 smb climatology logger.info("Remapping ismip6 climatology onto MALI mesh...") @@ -190,6 +190,8 @@ def run(self): dst = os.path.join(output_path, output_anomaly_ismip6) shutil.copy(src, dst) + logger.info(f"!---Done processing the file---!") + def remap_ismip6_smb_to_mali(self, input_file, output_file, mali_mesh_name, mali_mesh_file, method_remap): """ @@ -231,7 +233,7 @@ def remap_ismip6_smb_to_mali(self, input_file, output_file, mali_mesh_name, "-m", mapping_file, "-v", "smb_anomaly"] - subprocess.check_call(args) + check_call(args, logger=self.logger) def rename_ismip6_smb_to_mali_vars(self, remapped_file_temp, output_file): """ diff --git a/compass/landice/tests/ismip6_forcing/atmosphere/process_smb_racmo.py b/compass/landice/tests/ismip6_forcing/atmosphere/process_smb_racmo.py index 3d1159c634..fde71533d1 100644 --- a/compass/landice/tests/ismip6_forcing/atmosphere/process_smb_racmo.py +++ b/compass/landice/tests/ismip6_forcing/atmosphere/process_smb_racmo.py @@ -1,9 +1,9 @@ import os import shutil -import subprocess import xarray as xr import warnings from mpas_tools.io import write_netcdf +from mpas_tools.logging import check_call from compass.step import Step from compass.landice.tests.ismip6_forcing.atmosphere.create_mapfile_smb \ import build_mapping_file @@ -84,8 +84,9 @@ def run(self): output_file_final = os.path.join(output_base_path, output_file) if os.path.exists(output_file_final): - print("Processed RACMO SMB data already exists in the output path" - f"{output_base_path}. Not processing the source data...") + logger.info("Processed RACMO SMB data already exists in the " + "output path {output_base_path}. " + "Not processing the source data...") return input_file_list = self._files @@ -94,13 +95,13 @@ def run(self): f"{input_file_list[0]}", f"{racmo_file_temp1}"] - subprocess.check_call(args) + check_call(args, logger=logger) # interpolate the racmo smb data remapped_file_temp = "remapped.nc" # temporary file name # call the function that reads in, remap and rename the file. - logger.info("Calling a remapping function...") + logger.info("Calling the remapping function...") self.remap_source_smb_to_mali(f"{racmo_file_temp1}", remapped_file_temp, mali_mesh_name, @@ -114,14 +115,14 @@ def run(self): f"{remapped_file_temp}", f"{racmo_file_temp2}"] - subprocess.check_call(args) + check_call(args, logger=logger) # change the unit attribute to kg/m^2/s args = ["ncatted", "-O", "-a", "units,sfcMassBal,m,c,'kg m-2 s-1'", f"{racmo_file_temp2}"] - subprocess.check_call(args) + check_call(args, logger=logger) # call the function that renames the ismip6 variables to MALI variables logger.info("Renaming source variables to mali variable names...") @@ -139,13 +140,15 @@ def run(self): output_path = f"{output_base_path}/atmosphere_forcing/" \ f"RACMO_climatology_1995-2017" if not os.path.exists(output_path): - print("Creating a new directory for the output data") + logger.info("Creating a new directory for the output data") os.makedirs(output_path) src = os.path.join(os.getcwd(), output_file) dst = os.path.join(output_path, output_file) shutil.copy(src, dst) + logger.info(f"!---Done processing the file---!") + def remap_source_smb_to_mali(self, input_file, output_file, mali_mesh_name, mali_mesh_file, method_remap): """ @@ -186,7 +189,7 @@ def remap_source_smb_to_mali(self, input_file, output_file, mali_mesh_name, "-m", mapping_file, "-v", "smb"] - subprocess.check_call(args) + check_call(args, logger=self.logger) def rename_source_smb_to_mali_vars(self, remapped_file_temp, output_file): """ diff --git a/compass/landice/tests/ismip6_forcing/create_mapfile.py b/compass/landice/tests/ismip6_forcing/create_mapfile.py index 2b862985d4..81ef70e67c 100644 --- a/compass/landice/tests/ismip6_forcing/create_mapfile.py +++ b/compass/landice/tests/ismip6_forcing/create_mapfile.py @@ -32,7 +32,7 @@ def build_mapping_file(config, cores, logger, ismip6_grid_file, """ if os.path.exists(mapping_file): - print("Mapping file exists. Not building a new one.") + logger.info(f"Mapping file exists. Not building a new one.") return if mali_mesh_file is None: @@ -46,9 +46,10 @@ def build_mapping_file(config, cores, logger, ismip6_grid_file, # create the ismip6 scripfile if mapping file does not exist # this is the projection of ismip6 data for Antarctica - print("Mapping file does not exist. Building one based on the " - "input/ouptut meshes") - print("Creating temporary scripfiles for ismip6 grid and mali mesh...") + logger.info(f"Mapping file does not exist. Building one based on " + f"the input/ouptut meshes") + logger.info(f"Creating temporary scripfiles " + f"for ismip6 grid and mali mesh...") args = ["create_SCRIP_file_from_planar_rectangular_grid.py", "--input", ismip6_grid_file, @@ -56,7 +57,7 @@ def build_mapping_file(config, cores, logger, ismip6_grid_file, "--proj", ismip6_projection, "--rank", "2"] - subprocess.check_call(args) + check_call(args, logger=logger) # create a MALI mesh scripfile if mapping file does not exist # make sure the mali mesh file uses the longitude convention of [0 2pi] @@ -70,12 +71,13 @@ def build_mapping_file(config, cores, logger, ismip6_grid_file, "--file", mali_mesh_copy, "--proj", ismip6_projection] - subprocess.check_call(args) + check_call(args, logger=logger) scrip_from_mpas(mali_mesh_file, mali_scripfile) # create a mapping file using ESMF weight gen - print(f"Creating a mapping file. Mapping method used: {method_remap}") + logger.info(f"Creating a mapping file... " + f"Mapping method used: {method_remap}") if method_remap is None: raise ValueError("Desired remapping option should be provided with " @@ -97,7 +99,7 @@ def build_mapping_file(config, cores, logger, ismip6_grid_file, check_call(args, logger) # remove the temporary scripfiles once the mapping file is generated - print("Removing the temporary mesh and scripfiles...") + logger.info(f"Removing the temporary mesh and scripfiles...") os.remove(ismip6_scripfile) os.remove(mali_scripfile) os.remove(mali_mesh_copy) diff --git a/compass/landice/tests/ismip6_forcing/ocean_basal/process_basal_melt.py b/compass/landice/tests/ismip6_forcing/ocean_basal/process_basal_melt.py index d09bedd250..ac07002899 100644 --- a/compass/landice/tests/ismip6_forcing/ocean_basal/process_basal_melt.py +++ b/compass/landice/tests/ismip6_forcing/ocean_basal/process_basal_melt.py @@ -1,10 +1,10 @@ import os import shutil -import subprocess import xarray as xr from compass.landice.tests.ismip6_forcing.create_mapfile \ import build_mapping_file from mpas_tools.io import write_netcdf +from mpas_tools.logging import check_call from compass.step import Step @@ -55,7 +55,7 @@ def run(self): """ Run this step of the test case """ - # logger = self.logger + logger = self.logger config = self.config section = config["ismip6_ais"] mali_mesh_name = section.get("mali_mesh_name") @@ -69,11 +69,13 @@ def run(self): # ismip6 input files # call the function that combines data - # logger.info = ('calling combine_ismip6_inputfiles') + logger.info(f"Combining the ismip6 input files") input_file_list = self._files_coeff for i, file in enumerate(input_file_list): - print(f"processing the input file {os.path.basename(file)}") + logger.info(f"!---Start processing the current file---!") + logger.info(f"processing the input file " + f"'{os.path.basename(file)}'") # temporary file names. Note: The counter 'i' seems to be necessary # for 'ncremap' to correctly interpolate the gamma0 values for an # unknown reason. @@ -85,7 +87,7 @@ def run(self): combined_file_temp) # remap the input forcing file. - print("Calling the remapping function...") + logger.info(f"Calling the remapping function...") self.remap_ismip6_basal_melt_to_mali_vars(combined_file_temp, remapped_file_temp, mali_mesh_name, @@ -95,12 +97,13 @@ def run(self): output_file = f"processed_basin_and_{os.path.basename(file)}" # rename the ismip6 variables to MALI variables - print("Renaming the ismip6 variables to mali variable names...") + logger.info(f"Renaming the ismip6 variables to " + f"mali variable names...") self.rename_ismip6_basal_melt_to_mali_vars(remapped_file_temp, output_file) - print("Remapping and renamping process done successfully. " - "Removing the temporary files...") + logger.info(f"Remapping and renamping process done successfully. " + f"Removing the temporary files...") # remove the temporary combined file os.remove(combined_file_temp) @@ -109,13 +112,17 @@ def run(self): # place the output file in appropriate director output_path = f"{output_base_path}/basal_melt/parameterizations/" if not os.path.exists(output_path): - print("Creating a new directory for the output data") + logger.info("Creating a new directory for the output data...") os.makedirs(output_path) src = os.path.join(os.getcwd(), output_file) dst = os.path.join(output_path, output_file) shutil.copy(src, dst) + logger.info(f"!---Done processing the current file---!") + logger.info(f"") + logger.info(f"") + def combine_ismip6_inputfiles(self, basin_file, coeff_gamma0_deltaT_file, combined_file_temp): """ @@ -168,7 +175,8 @@ def remap_ismip6_basal_melt_to_mali_vars(self, input_file, output_file, input_file, mapping_file, mali_mesh_file, method_remap) else: - print("Mapping file exists. Remapping the input data...") + self.logger.info(f"Mapping file exists. " + f"Remapping the input data...") # remap the input data args = ["ncremap", @@ -176,7 +184,7 @@ def remap_ismip6_basal_melt_to_mali_vars(self, input_file, output_file, "-o", output_file, "-m", mapping_file] - subprocess.check_call(args) + check_call(args, logger=self.logger) def rename_ismip6_basal_melt_to_mali_vars(self, remapped_file_temp, output_file): diff --git a/compass/landice/tests/ismip6_forcing/ocean_thermal/process_thermal_forcing.py b/compass/landice/tests/ismip6_forcing/ocean_thermal/process_thermal_forcing.py index 33c5854bab..d7683cbeec 100644 --- a/compass/landice/tests/ismip6_forcing/ocean_thermal/process_thermal_forcing.py +++ b/compass/landice/tests/ismip6_forcing/ocean_thermal/process_thermal_forcing.py @@ -6,6 +6,7 @@ from compass.landice.tests.ismip6_forcing.create_mapfile \ import build_mapping_file from mpas_tools.io import write_netcdf +from mpas_tools.logging import check_call from compass.step import Step @@ -96,7 +97,7 @@ def run(self): remapped_file_temp = "remapped.nc" # temporary file name # call the function that reads in, remap and rename the file. - print("Calling a remapping function...") + logger.info("Calling the remapping function...") self.remap_ismip6_thermal_forcing_to_mali_vars(input_file, remapped_file_temp, mali_mesh_name, @@ -104,25 +105,27 @@ def run(self): method_remap) # call the function that renames the ismip6 variables to MALI variables - print("Renaming the ismip6 variables to mali variable names...") + logger.info(f"Renaming the ismip6 variables to mali variable names...") self.rename_ismip6_thermal_forcing_to_mali_vars(remapped_file_temp, output_file) - print("Remapping and renamping process done successfully. " - "Removing the temporary file 'remapped.nc'...") + logger.info(f"Remapping and renamping process done successfully. " + f"Removing the temporary file 'remapped.nc'...") # remove the temporary combined file os.remove(remapped_file_temp) # place the output file in appropriate directory if not os.path.exists(output_path): - print("Creating a new directory for the output data...") + logger.info(f"Creating a new directory for the output data...") os.makedirs(output_path) src = os.path.join(os.getcwd(), output_file) dst = os.path.join(output_path, output_file) shutil.copy(src, dst) + logger.info(f"!---Done processing the file---!") + def remap_ismip6_thermal_forcing_to_mali_vars(self, input_file, output_file, @@ -155,7 +158,8 @@ def remap_ismip6_thermal_forcing_to_mali_vars(self, input_file, mapping_file, mali_mesh_file, method_remap) else: - print("Mapping file exists. Remapping the input data...") + self.logger.info(f"Mapping file exists. " + f"Remapping the input data...") # remap the input data args = ["ncremap", @@ -163,7 +167,7 @@ def remap_ismip6_thermal_forcing_to_mali_vars(self, "-o", output_file, "-m", mapping_file] - subprocess.check_call(args) + check_call(args, logger=self.logger) def rename_ismip6_thermal_forcing_to_mali_vars(self, remapped_file_temp, From 0423b39ac26261b6da353b861c8bde669e726bc8 Mon Sep 17 00:00:00 2001 From: hollyhan Date: Fri, 9 Sep 2022 16:50:38 -0600 Subject: [PATCH 44/58] Clean up doc strings squash --- .../ismip6_forcing/atmosphere/create_mapfile_smb.py | 6 ++++++ .../tests/ismip6_forcing/atmosphere/process_smb.py | 12 +++++++++--- .../ismip6_forcing/atmosphere/process_smb_racmo.py | 9 +++++++-- compass/landice/tests/ismip6_forcing/configure.py | 1 - .../landice/tests/ismip6_forcing/create_mapfile.py | 6 ++++++ .../ocean_basal/process_basal_melt.py | 13 ++++++++++--- .../ocean_thermal/process_thermal_forcing.py | 5 +++++ 7 files changed, 43 insertions(+), 9 deletions(-) diff --git a/compass/landice/tests/ismip6_forcing/atmosphere/create_mapfile_smb.py b/compass/landice/tests/ismip6_forcing/atmosphere/create_mapfile_smb.py index ead5184d14..a8d3a2e6a9 100644 --- a/compass/landice/tests/ismip6_forcing/atmosphere/create_mapfile_smb.py +++ b/compass/landice/tests/ismip6_forcing/atmosphere/create_mapfile_smb.py @@ -19,16 +19,22 @@ def build_mapping_file(config, cores, logger, ismip6_grid_file, ---------- config : compass.config.CompassConfigParser Configuration options for a ismip6 forcing test case + cores : int the number of cores for the ESMF_RegridWeightGen + logger : logging.Logger A logger for output from the step + ismip6_grid_file : str ismip6 grid file + mapping_file : str weights for interpolation from ismip6_grid_file to mali_mesh_file + mali_mesh_file : str, optional The MALI mesh file is used if mapping file does not exist + method_remap : str, optional Remapping method used in building a mapping file """ diff --git a/compass/landice/tests/ismip6_forcing/atmosphere/process_smb.py b/compass/landice/tests/ismip6_forcing/atmosphere/process_smb.py index 0cc919b3e9..8bd42c28fa 100644 --- a/compass/landice/tests/ismip6_forcing/atmosphere/process_smb.py +++ b/compass/landice/tests/ismip6_forcing/atmosphere/process_smb.py @@ -201,15 +201,18 @@ def remap_ismip6_smb_to_mali(self, input_file, output_file, mali_mesh_name, ---------- input_file: str input smb data on its native polarstereo 8km grid + output_file : str smb data remapped on mali mesh + mali_mesh_name : str name of the mali mesh used to name mapping files + mali_mesh_file : str, optional The MALI mesh file if mapping file does not exist + method_remap : str, optional Remapping method used in building a mapping file - """ mapping_file = f"map_ismip6_8km_to_" \ f"{mali_mesh_name}_{method_remap}.nc" @@ -244,6 +247,7 @@ def rename_ismip6_smb_to_mali_vars(self, remapped_file_temp, output_file): ---------- remapped_file_temp : str temporary ismip6 data remapped on mali mesh + output_file : str remapped ismip6 data renamed on mali mesh """ @@ -298,8 +302,10 @@ def correct_smb_anomaly_for_climatology(self, ---------- racmo_clim_file : str RACMO climatology file (1995-2017) + output_clim_ismip6_file : str remapped and renamed ismip6 climatology file + output_file_final : str climatology-corrected, final ismip6 smb anomaly file """ @@ -309,8 +315,8 @@ def correct_smb_anomaly_for_climatology(self, ds_ismip6_clim = xr.open_dataset(output_clim_ismip6_file) # calculate the climatology correction - corr_clim = ds_racmo_clim["sfcMassBal"].isel(Time=0) \ - - ds_ismip6_clim["sfcMassBal"].isel(Time=0) + corr_clim = (ds_racmo_clim["sfcMassBal"].isel(Time=0) - + ds_ismip6_clim["sfcMassBal"].isel(Time=0)) # correct the ismip6 smb anomaly ds["sfcMassBal"] = ds["sfcMassBal"] + corr_clim diff --git a/compass/landice/tests/ismip6_forcing/atmosphere/process_smb_racmo.py b/compass/landice/tests/ismip6_forcing/atmosphere/process_smb_racmo.py index fde71533d1..5f4d175ada 100644 --- a/compass/landice/tests/ismip6_forcing/atmosphere/process_smb_racmo.py +++ b/compass/landice/tests/ismip6_forcing/atmosphere/process_smb_racmo.py @@ -157,16 +157,19 @@ def remap_source_smb_to_mali(self, input_file, output_file, mali_mesh_name, Parameters ---------- input_file: str - input racmo smb data on its native rotaed pole grid + input racmo smb data on its native rotated pole grid + output_file : str smb data remapped on mali mesh + mali_mesh_name : str name of the mali mesh used to name mapping files + mali_mesh_file : str, optional The MALI mesh file if mapping file does not exist + method_remap : str, optional Remapping method used in building a mapping file - """ mapping_file = f"map_racmo_24km_to_" \ f"{mali_mesh_name}_{method_remap}.nc" @@ -200,6 +203,7 @@ def rename_source_smb_to_mali_vars(self, remapped_file_temp, output_file): ---------- remapped_file_temp : str temporary ismip6 data remapped on mali mesh + output_file : str remapped ismip6 data renamed on mali mesh """ @@ -232,6 +236,7 @@ def correct_smb_anomaly_for_base_smb(self, output_file, mali_mesh_file): ---------- output_file : str remapped ismip6 data renamed on mali mesh + mali_mesh_file : str initialized MALI mesh file in which the base SMB field exists """ diff --git a/compass/landice/tests/ismip6_forcing/configure.py b/compass/landice/tests/ismip6_forcing/configure.py index e5d3b23f42..1f9ec16570 100644 --- a/compass/landice/tests/ismip6_forcing/configure.py +++ b/compass/landice/tests/ismip6_forcing/configure.py @@ -10,7 +10,6 @@ def configure(config, check_model_options): check_model_options : bool Whether we check ``model``, ``scenario``, ``period_endyear`` - """ section = "ismip6_ais" diff --git a/compass/landice/tests/ismip6_forcing/create_mapfile.py b/compass/landice/tests/ismip6_forcing/create_mapfile.py index 81ef70e67c..954f5a719b 100644 --- a/compass/landice/tests/ismip6_forcing/create_mapfile.py +++ b/compass/landice/tests/ismip6_forcing/create_mapfile.py @@ -17,16 +17,22 @@ def build_mapping_file(config, cores, logger, ismip6_grid_file, ---------- config : compass.config.CompassConfigParser Configuration options for a ismip6 forcing test case + cores : int the number of cores for the ESMF_RegridWeightGen + logger : logging.Logger A logger for output from the step + ismip6_grid_file : str ismip6 grid file + mapping_file : str weights for interpolation from ismip6_grid_file to mali_mesh_file + mali_mesh_file : str, optional The MALI mesh file if mapping file does not exist + method_remap : str, optional Remapping method used in building a mapping file """ diff --git a/compass/landice/tests/ismip6_forcing/ocean_basal/process_basal_melt.py b/compass/landice/tests/ismip6_forcing/ocean_basal/process_basal_melt.py index ac07002899..3b3e1a4df1 100644 --- a/compass/landice/tests/ismip6_forcing/ocean_basal/process_basal_melt.py +++ b/compass/landice/tests/ismip6_forcing/ocean_basal/process_basal_melt.py @@ -123,7 +123,7 @@ def run(self): logger.info(f"") logger.info(f"") - def combine_ismip6_inputfiles(self, basin_file, coeff_gamma0_deltaT_file, + def combine_ismip6_inputfiles(self, basin_file, coeff_gamma0_deltat_file, combined_file_temp): """ Combine ismip6 input files before regridding onto mali mesh @@ -132,15 +132,17 @@ def combine_ismip6_inputfiles(self, basin_file, coeff_gamma0_deltaT_file, ---------- basin_file : str imbie2 basin numbers in ismip6 grid - coeff_gamma0_deltaT_file : str + + coeff_gamma0_deltat_file : str uniform melt rate coefficient (gamma0) and temperature correction per basin + combined_file_temp : str temporary output file that has all the variables combined """ ds_basin = xr.open_dataset(basin_file, engine="netcdf4") - ds = xr.open_dataset(coeff_gamma0_deltaT_file, engine="netcdf4") + ds = xr.open_dataset(coeff_gamma0_deltat_file, engine="netcdf4") ds["ismip6shelfMelt_basin"] = ds_basin.basinNumber write_netcdf(ds, combined_file_temp) @@ -156,12 +158,16 @@ def remap_ismip6_basal_melt_to_mali_vars(self, input_file, output_file, input_file: str temporary output file that has all the variables combined combined_file_temp generated in the above function + output_file : str ismip6 data remapped on mali mesh + mali_mesh_name : str name of the mali mesh used to name mapping files + mali_mesh_file : str, optional The MALI mesh file if mapping file does not exist + method_remap : str, optional Remapping method used in building a mapping file """ @@ -196,6 +202,7 @@ def rename_ismip6_basal_melt_to_mali_vars(self, remapped_file_temp, ---------- remapped_file_temp : str temporary ismip6 data remapped on mali mesh + output_file : str remapped ismip6 data renamed on mali mesh """ diff --git a/compass/landice/tests/ismip6_forcing/ocean_thermal/process_thermal_forcing.py b/compass/landice/tests/ismip6_forcing/ocean_thermal/process_thermal_forcing.py index d7683cbeec..7397088909 100644 --- a/compass/landice/tests/ismip6_forcing/ocean_thermal/process_thermal_forcing.py +++ b/compass/landice/tests/ismip6_forcing/ocean_thermal/process_thermal_forcing.py @@ -139,12 +139,16 @@ def remap_ismip6_thermal_forcing_to_mali_vars(self, ---------- input_file: str ismip6 thermal forcing data on its native polarstereo 8km grid + output_file : str ismip6 data remapped on mali mesh + mali_mesh_name : str name of the mali mesh used to name mapping files + mali_mesh_file : str, optional The MALI mesh file if mapping file does not exist + method_remap : str, optional Remapping method used in building a mapping file """ @@ -180,6 +184,7 @@ def rename_ismip6_thermal_forcing_to_mali_vars(self, ---------- remapped_file_temp : str temporary ismip6 data remapped on mali mesh + output_file : str remapped ismip6 data renamed on mali mesh """ From d02b608f4542312c21fa91a73e446f75c2f13c21 Mon Sep 17 00:00:00 2001 From: hollyhan Date: Sat, 10 Sep 2022 11:37:41 -0600 Subject: [PATCH 45/58] Edit users guide and developers guide --- docs/developers_guide/landice/api.rst | 36 ++- .../landice/test_groups/ismip6_forcing.rst | 22 +- .../landice/test_groups/ismip6_forcing.rst | 289 +++++++++++++++--- 3 files changed, 292 insertions(+), 55 deletions(-) diff --git a/docs/developers_guide/landice/api.rst b/docs/developers_guide/landice/api.rst index 9d4a7fcaf9..e5b6ee9977 100644 --- a/docs/developers_guide/landice/api.rst +++ b/docs/developers_guide/landice/api.rst @@ -240,12 +240,44 @@ ismip6_forcing :toctree: generated/ Ismip6Forcing + configure.configure + create_mapfile.build_mapping_file atmosphere.Atmosphere - atmosphere.Atmosphere.run + atmosphere.Atmosphere.configure + atmosphere.create_mapfile_smb + atmosphere.create_mapfile_smb.build_mapping_file + atmosphere.create_mapfile_smb.create_atm_scrip + atmosphere.process_smb.ProcessSMB + atmosphere.process_smb.ProcessSMB.setup + atmosphere.process_smb.ProcessSMB.run + atmosphere.process_smb.ProcessSMB.remap_ismip6_smb_to_mali + atmosphere.process_smb.ProcessSMB.rename_ismip6_smb_to_mali_vars + atmosphere.process_smb.ProcessSMB.correct_smb_anomaly_for_climatology + atmosphere.process_smb_racmo.ProcessSmbRacmo + atmosphere.process_smb_racmo.ProcessSmbRacmo.setup + atmosphere.process_smb_racmo.ProcessSmbRacmo.run + atmosphere.process_smb_racmo.ProcessSmbRacmo.remap_source_smb_to_mali + atmosphere.process_smb_racmo.ProcessSmbRacmo.rename_source_smb_to_mali_vars + atmosphere.process_smb_racmo.ProcessSmbRacmo.correct_smb_anomaly_for_base_smb + + ocean_basal.OceanBasal + ocean_basal.OceanBasal.configure + ocean_basal.process_basal_melt.ProcessBasalMelt + ocean_basal.process_basal_melt.ProcessBasalMelt.setup + ocean_basal.process_basal_melt.ProcessBasalMelt.run + ocean_basal.process_basal_melt.ProcessBasalMelt.combine_ismip6_inputfiles + ocean_basal.process_basal_melt.ProcessBasalMelt.remap_ismip6_basal_melt_to_mali_vars + ocean_basal.process_basal_melt.ProcessBasalMelt.rename_ismip6_basal_melt_to_mali_vars ocean_thermal.OceanThermal - ocean_thermal.OceanThermal.run + ocean_thermal.OceanThermal.configure + ocean_thermal.process_thermal_forcing.ProcessThermalForcing + ocean_thermal.process_thermal_forcing.ProcessThermalForcing.setup + ocean_thermal.process_thermal_forcing.ProcessThermalForcing.run + ocean_thermal.process_thermal_forcing.ProcessThermalForcing.remap_ismip6_thermal_forcing_to_mali_vars + ocean_thermal.process_thermal_forcing.ProcessThermalForcing.rename_ismip6_thermal_forcing_to_mali_vars + kangerlussuaq ~~~~~~~~~~~~~ diff --git a/docs/developers_guide/landice/test_groups/ismip6_forcing.rst b/docs/developers_guide/landice/test_groups/ismip6_forcing.rst index 2f9b58d9cc..8f853d8ec1 100644 --- a/docs/developers_guide/landice/test_groups/ismip6_forcing.rst +++ b/docs/developers_guide/landice/test_groups/ismip6_forcing.rst @@ -4,9 +4,21 @@ ismip6_forcing ============== The ``ismip6_forcing`` test group (:py:class:`compass.landice.tests.ismip6_ -forcing.Ismip6Forcing`) processes the atmospheric and oceaninc forcing data of -the Ice Sheet Model Intercomparison for CMIP6 (ISMIP6) protocol. Here, -we describe the shared framework for this test group and the 3 test cases. +forcing.Ismip6Forcing`) processes (i.e., remapping and renaming) the +atmospheric and oceanic forcing data of the Ice Sheet Model +Intercomparison for CMIP6 (ISMIP6) protocol from its native polarstereo grid to +the unstructure MALI mesh. The test group includes three test cases, +``atmosphere``, ``ocean_basal`` and ``ocean_thermal``. The ``atmosphere`` +test case has two steps: ``process_smb`` and ``process_smb_racmo``; +the ``ocean_basal`` test case has one step, ``process_basal_melt``; +the ``ocean_thermal`` has one step, ``process_thermal_forcing``. +Each step has the local methods (functions) of remapping and +renaming the original ismip6 data to the format that MALI can incorporate in +its forward simulations. In remapping the data, all test cases import the +method ``build_mapping_file`` to create or use scrip files +of the source (ISMIP6) and destination (MALI) mesh depending on the existence +of a mapping file. Below, we describe the shared framework for this +test group and the 3 test cases. .. _dev_landice_ismip6_forcing_framework: @@ -44,7 +56,7 @@ ocean_basal ------------ The :py:class:`compass.landice.tests.ismip6_forcing.ocean_basal.OceanBasal` -performs processing of the coefficients for the basal melt parametrization +performs processing of the coefficients for the basal melt parameterization utilized by the ISMIP6 protocol. Processing data includes combining the -IMBIE2 basin number file and parametrization coefficients and remapping onto +IMBIE2 basin number file and parameterization coefficients and remapping onto the MALI mesh. diff --git a/docs/users_guide/landice/test_groups/ismip6_forcing.rst b/docs/users_guide/landice/test_groups/ismip6_forcing.rst index deff500ea5..503efc896d 100644 --- a/docs/users_guide/landice/test_groups/ismip6_forcing.rst +++ b/docs/users_guide/landice/test_groups/ismip6_forcing.rst @@ -1,30 +1,150 @@ .. _landice_ismip6_forcing: ismip6_forcing -============ - -The ``landice/ismip6_forcing`` test group implements processing of atmospheric -and ocean forcing data of the Ice Sheet Model Intercomparison for CMIP6 -(ISMIP6) protocol. The ISMIP6 source data can be obtained by directly -contacting the authors. Reference: https://www.climate-cryosphere.org/wiki/index.php?title=ISMIP6-Projections2300-Antarctica#A2.2_Retrieving_datasets_and_uploading_your_model_output - -The test group includes 3 test cases, ``atmosphere``, ``ocean_thermal`` and -``ocean_basal``. All test cases are made up of a single main step, -``ProcessSMB``,``ProcessThermalForcing`` and ``ProcessBasalMelt``, -respectively. Each step has the local methods (functions) of remapping and -renaming the original ismip6 data to the format that MALI can incorporate in -its forward simulations. In remapping the data, all test cases import the -method ``build_mapping_file`` to create or use scrip files -of the source (ISMIP6) and destination (MALI) mesh depending on the existence -of a mapping file. Approximated time for processing a single forcing file -on Cori (single core) is 2, 1, and 5 minutes for the atmosphere, ocean basal, +============== + +The ``landice/ismip6_forcing`` test group processes (i.e., remaps and renames) +the atmospheric and ocean forcing data of the Ice Sheet Model Intercomparison for CMIP6 +(ISMIP6) protocol. The processed data is used to force MALI in its simulations +under a relevant ISMIP6 (either the 2100 or 2300) experimental protocol. +The test group includes three test cases, ``atmosphere``, ``ocean_basal`` and +``ocean_thermal``; the ``atmosphere`` test case has two steps: +``process_smb`` and ``process_smb_racmo``. The ``ocean_basal`` and the +``ocean_thermal`` test case each has one step, ``process_basal_melt``, and +``process_thermal_forcing``, respectively. (For more details on the steps of +each test case, see :ref:`landice_ismip6_forcing_atmosphere`, +:ref:`landice_ismip6_forcing_ocean_basal` and +:ref:`landice_ismip6_forcing_ocean_thermal`.) +Approximated time for processing a single forcing file +on Cori (single core) is 2, 7, and 1 minutes for the atmosphere, ocean basal, and ocean thermal testcase, respectively. +Before providing the details of necessary source data of the ISMIP6 protocols, +we provide a summary of instructions of an overall process of this test group: +To start off, users need to provide a MALI mesh onto which the source data +will be remapped and renamed. More information about obtaining these meshes will +be provided as soon as they are publicly available. Then, for a given MALI mesh, + +1. run :ref:`landice_ismip6_forcing_ocean_basal` once, independent of the +model, scenario and end year. + +2. run :ref:`landice_ismip6_forcing_ocean_thermal` once with +``process_obs_data = True`` in the config file (:ref:`landice_ismip6_forcing_config`) +to process the thermal forcing from the observational climatology (used for +control runs). + +3. for each model, scenario and end year, run :ref:`landice_ismip6_forcing_ocean_thermal`, +and run :ref:`landice_ismip6_forcing_ocean_atmosphere` with +``process_racmo_smb = True`` once with any model, and run with subsequent +models with ``process_racmo_smb = False``, but it is harmless to run it again +with ``process_racmo_smb = True`` as it does nothing if data is already +available, and the processing is very quick (less than a minute). + +There are six different of input data sets other than the MALI mesh that +users need in order to use this package: (#1) atmospheric surface mass balance (SMB) +anomaly forcing and (#2) ocean thermal forcing for projection runs, modern +climatology files for (#3) atmospheric and (#4) ocean forcing, (#5) ISMIP6 basin +number file, and (#6) the ocean basal-melt parameter files. + +Except for the file #3, The ISMIP6 source data (files #1, 2, 4-6) can be obtained by contacting the ISMIP6 steering +committee as described `here. `_ +Once the users get access to the data on `Globus `_, +and within the base path directory that the user specifies in the config file +(i.e., the config option ``base_path_ismip6``; +see :ref:`landice_ismip6_forcing_config`), they +need to manually download the files following the directory structure +and names provided in the GHub endpoints (named ``GHub-ISMIP6-Forcing`` +for the 2100 CE projection protocol and ``ISMIP6-Projections-Forcing-2300`` +for the 2300 CE projection protocol). That is, the directory paths and the file +names must exactly match (even the letter case) those that are provided by the +GHub endpoints. For example, if a user wants to process the atmosphere (SMB) +forcing representing the ``SSP585`` scenario from the ``UKESM1-0-LL`` model +provided by the ISMIP6-2100 protocol (i.e., from the ``GHub-ISMIP6-Forcing`` +Globus endpoint), they must create the directory path +``AIS/Atmosphere_Forcing/UKESM1-0-LL/Regridded_8km/`` in their local system +where they will download the file ``UKESM1-0-LL_anomaly_ssp585_1995-2100_8km.nc``. +Note that the users only need to download SMB anomaly files (not the climatology +file) that cover the period from 1995 to ``period_endyear`` +(either 2100 or 2300, defined in the config file; +see :ref:`landice_ismip6_forcing_config`). + +Equivalently, for the +ocean forcing, the user should create the directory path +``AIS/Ocean_Forcing/ukesm1-0-ll_ssp585/1995-2100/`` and download the file +``UKESM1-0-LL_ssp585_thermal_forcing_8km_x_60m.nc`` from the same endpoint. +Users do not need to download the thermal +forcing files for the years previous to 1995 as only the files downloaded from +``1995-{period_endyear}`` will be processed. +Also note to be ware that unlike those in the ``GHub-ISMIP6-Forcing`` endpoint, +the directory names in the ``ISMIP6-Projections-Forcing-2300`` endpoint have a +lower case "f" for the ``AIS/Atmospheric_forcing/`` and ``AIS/Ocean_forcing/``. + +In addition to atmospheric and ocean thermal forcing files that +correspond to specific climate model (e.g., UKESM1-0-LL, CCSM4) and scenarios +(e.g., SSP585, RCP85, RCP26-repeat), modern +climatology files are needed. For the ``atmosphere`` testcase, users +need to download the RACMO modern climatology data +``RACMO2.3p2_ANT27_smb_yearly_1979_2018.nc`` from +`here. `_ +The RACMO file is used to correct the ISMIP6 the surface mass balance (SMB) +data with the modern climatology. For the ``ocean_thermal`` case, users need to +download the modern ocean thermal forcing climatology file named +``obs_thermal_forcing_1995-2017_8km_x_60m.nc`` in the directory +``AIS/Ocean_F{f}orcing/climatology_from_obs_1995-2017/`` +(the salinity and temperature files do not have to be downloaded). + + +For the ``ocean_basal`` testcase, users need to additionally download +the basin number file ``imbie2_basin_numbers_8km.nc`` in the directory +``AIS/Ocean_Forcing/imbie2/`` (or ``AIS/Ocean_forcing/imbie2/``, if from the +``ISMIP6-Projections-Forcing-2300`` endpoint); all of the files that +start their name with ``coeff_gamma0_DeltaT_quadratic_local`` in the directory +''AIS/Ocean_F{f}orcing/parameterizations/'', which contain parameter values needed +for calculating the basal melt underneath the ice shelves in MALI simulations. + +Note that both the RACMO SMB data and ocean basal-melt parameters not +associated with any climate models and scenarios and thus can be processed only +once and can be applied to MALI with any set of processed climate forcing data. + + +In the next section (ref:`landice_ismip6_forcing_config`), we provide +instructions and examples of how users can configure necessary options including +paths to necessary source files and the output path of the processed data +within which the subdirectories called ``atmosphere_forcing/``, ``basal_melt/`` +and ``ocean_thermal_forcing/`` (and further subdirectories that match the source +file directory structure) are created if the directories do not already exist) +and where processed files will be saved. + +.. _landice_ismip6_forcing_config: + config options -------------- All three test cases share some set of default config options under the section -``[ismip6_ais]`` and separate config options under the corresponding section: +``[ismip6_ais]`` and have separate config options for each test case: +``[ismip6_ais_atmosphere]``, ``[ismip6_ais_ocean_thermal]``, and +``[ismip6_ais_ocean_basal]``. In the general config section +``[ismip6_ais]``, users need to supply base paths to input files and MALI mesh +file, and MALI mesh name, as well as the model name, climate forcing scenario +and the projection end year of the ISMIP6 forcing data, which can be chosen +from the available options as given in the config file (see the example file +below.) In the ``ismip6_ais_atmosphere`` section, users need to indicate +``True`` or ``False`` on whether to process the RACMO modern climatology +(``True`` is required to run the ``process_smb_racmo`` step, which needs to be +run before the ``process_smb`` step). In the ``ismip6_ais_ocean_thermal`` +section, users need to indicate ``True`` or ``False`` and on whether to process +the observational thermal forcing data, which represents the modern ocean +climatology between 1995-2017. When ``process_obs_data`` is set to ``True``, +ocean forcing anomaly files from climate models are not processed (these files +will only be processed when ``process_obs_data`` is set to ``False``) + +For most the ``[ismip6_ais_atmosphere]`` and ``[ismip6_ais_ocean_thermal]`` +config sections users may choose the interpolation scheme among +``bilinear``, ``neareststod`` and ``conserve`` methods. The exception is that +the ``ocean basal`` test case should always use the ``neareststod`` method +because the source files have a single valued data per basin. + +Below is the default config options: .. code-block:: cfg @@ -32,30 +152,34 @@ All three test cases share some set of default config options under the section [ismip6_ais] # Base path to the input ismip6 ocean and smb forcing files. User has to supply. - base_path_ismip6 = /Users/hollyhan/Desktop/Data/ISMIP6-Projections-Forcing-2300/ + base_path_ismip6 = /Users/hollyhan/Desktop/ISMIP6_2300_Protocol/ISMIP6-Projections-Forcing-2300/ # Base path to the the MALI mesh. User has to supply. - base_path_mali = /Users/hollyhan/Desktop/Data/MALI-mesh/ + base_path_mali = /Users/hollyhan/Desktop/RESEARCH/MALI/mesh_files/ # Forcing end year of the ISMIP6 data. User has to supply. # Available end years are 2100 and 2300. period_endyear = 2300 + # Base path to which output forcing files are saved. + output_base_path = /Users/hollyhan/Desktop/ISMIP6_2300_Protocol/Process_Forcing_Testcase/ + # Name of climate model name used to generate ISMIP6 forcing data. User has to supply. - # Available model names are the following: CCSM4, CESM2-WACCM, CSIRO-Mk3-6-0, HadGEM2-ES, NorESM1-M, UKESM1-0-LL + # Available model names for the 2100 projection are the following: CCSM4, CESM2, CNRM_CM6, CNRM_ESM2, CSIRO-Mk3-6-0, HadGEM2-ES, IPSL-CM5A-MR, MIROC-ESM-CHEM, NorESM1-M, UKESM1-0-LL + # Available model names for the 2300 projection are the following: CCSM4, CESM2-WACCM, CSIRO-Mk3-6-0, HadGEM2-ES, NorESM1-M, UKESM1-0-LL model = NorESM1-M # Scenarios used by climate model. User has to supply. - # Available scenarios are the following: RCP26, RCP26-repeat, RCP85, SSP126, SSP585 + # Available scenarios for the 2100 projection are the following: RCP26, RCP26-repeat, RCP85, SSP126, SSP585 (SSP585v1 and SSP585v2 for the CESM2 model) + # Available scenarios for the 2300 projection are the following: RCP26, RCP26-repeat, RCP85, RCP85-repeat, SSP126, SSP585, SSP585-repeat scenario = RCP26-repeat # name of the mali mesh. User has to supply. Note: It is used to name mapping files # (e,g. 'map_ismip6_8km_to_{mali_mesh_name}_{method_remap}.nc'). - mali_mesh_name = Antarctica_8to80km - - # MALI mesh file to be used to build mapping file (netCDF format). User has to supply. - mali_mesh_file = Antarctica_8to80km_20220407.nc + mali_mesh_name = Antarctica_8to30km + # MALI mesh file to be used to build mapping file (e.g.Antarctic_8to80km_20220407.nc). User has to supply. + mali_mesh_file = AIS_8to30km_r01_20220607.nc # config options for ismip6 antarctic ice sheet SMB forcing data test cases [ismip6_ais_atmosphere] @@ -63,47 +187,116 @@ All three test cases share some set of default config options under the section # Remapping method used in building a mapping file. Options include: bilinear, neareststod, conserve method_remap = bilinear + # Set True to process RACMO modern climatology + process_smb_racmo = True + + # Full path to the RACMO regional climate model SMB data for modern climatology + base_path_racmo = /Users/hollyhan/Desktop/ISMIP6_2300_Protocol/ISMIP6-Projections-Forcing-2300/RACMO2.3p2_ANT27_SMB_yearly_1979_2018/ + # config options for ismip6 ocean thermal forcing data test cases [ismip6_ais_ocean_thermal] # Remapping method used in building a mapping file. Options include: bilinear, neareststod, conserve method_remap = bilinear - # Set to True if the want to process observational thermal forcing data. Set to False if want to process model thermal forcing data. + # Set to True if want to process observational thermal forcing data. Set to False if want to process model thermal forcing data. # Note: when set True, the ['ismip6_ais'] config options 'period_endyear', 'model' and 'scenario' will be ignored. - process_obs_data = False + process_obs_data = True + +Below is the example config options that users might create in running +the test group. This example is for processing the NorESM1-M RCP2.6 repeat +forcing to the year 2300 onto the 8-80km Antarctic Ice Sheet MALI mesh. +The example is configured to perform the `atmosphere\process_smb_racmo` step to +process the RACMO modern SMB climatology but not the modern thermal forcing. + +.. code-block:: cfg + + # config options for ismip6 antarctic ice sheet data set + [ismip6_ais] + # Base path to the input ismip6 ocean and smb forcing files. User has to supply. + base_path_ismip6 = NotAvailable + + # Base path to the the MALI mesh. User has to supply. + base_path_mali = NotAvailable - # config options for ismip6 ocean basal test case - [ismip6_ais_ocean_basal] + # Forcing end year of the ISMIP6 data. User has to supply. + # Available end years are 2100 and 2300. + period_endyear = NotAvailable - # Remapping method used in building a mapping file. Ocean basal testcase will always want to - # use neareststod method. - method_remap = neareststod + # Base path to which output forcing files are saved. + output_base_path = NotAvailable + + # Name of climate model name used to generate ISMIP6 forcing data. User has to supply. + # Available model names for the 2100 projection are the following: CCSM4, CESM2, CNRM_CM6, CNRM_ESM2, CSIRO-Mk3-6-0, HadGEM2-ES, IPSL-CM5A-MR, MIROC-ESM-CHEM, NorESM1-M, UKESM1-0-LL + # Available model names for the 2300 projection are the following: CCSM4, CESM2-WACCM, CSIRO-Mk3-6-0, HadGEM2-ES, NorESM1-M, UKESM1-0-LL + model = NotAvailable + + # Scenarios used by climate model. User has to supply. + # Available scenarios for the 2100 projection are the following: RCP26, RCP26-repeat, RCP85, SSP126, SSP585 (SSP585v1 and SSP585v2 for the CESM2 model) + # Available scenarios for the 2300 projection are the following: RCP26, RCP26-repeat, RCP85, RCP85-repeat, SSP126, SSP585, SSP585-repeat + scenario = NotAvailable + + # name of the mali mesh. User has to supply. Note: It is used to name mapping files + # (e,g. 'map_ismip6_8km_to_{mali_mesh_name}_{method_remap}.nc'). + mali_mesh_name = NotAvailable + + # MALI mesh file to be used to build mapping file (e.g.Antarctic_8to80km_20220407.nc). User has to supply. + mali_mesh_file = NotAvailable + + # config options for ismip6 antarctic ice sheet SMB forcing data test cases + [ismip6_ais_atmosphere] + + # Remapping method used in building a mapping file. Options include: bilinear, neareststod, conserve + method_remap = bilinear + + # Set True to process RACMO modern climatology + process_smb_racmo = True + + # Full path to the RACMO regional climate model SMB data for modern climatology + base_path_racmo = NotAvailable + + # config options for ismip6 ocean thermal forcing data test cases + [ismip6_ais_ocean_thermal] + + # Remapping method used in building a mapping file. Options include: bilinear, neareststod, conserve + method_remap = bilinear + + # Set to True if want to process observational thermal forcing data. Set to False if want to process model thermal forcing data. + # Note: when set True, the ['ismip6_ais'] config options 'period_endyear', 'model' and 'scenario' will be ignored. + process_obs_data = True + + +.. _landice_ismip6_forcing_atmosphere: atmosphere ---------- -The ``landice/ismip6_forcing/atmosphere`` -performs processing of the surface mass balance (SMB) forcing. -Processing data includes regridding the original ISMIP6 SMB data from its -native polarstereo grid to MALI's unstructured grid, renaming variables and -correcting the SMB anomaly field for the MALI base SMB. +The ``landice/ismip6_forcing/atmosphere`` test case +performs processing of the surface mass balance (SMB) forcing data provided by +the ISMIP6 and RACMO. Processing data includes regridding the SMB forcing data +SMB data from the native grid (polarstereo grid for the ISMIP6 files and +rotated pole grid for the RACMO file) to MALI's unstructured grid, renaming +variables, and correcting the ISMIP6 SMB anomaly field for the base SMB +(modern climatology) provided by RACMO. -ocean_thermal -------------- - -The ``landice/ismip6_forcing/ocean_thermal`` -performs the processing of ocean thermal forcing. Processing data includes -regridding the original ISMIP6 thermal forcing data from its native -polarstereo grid to MALI's unstructured grid and renaming variables. +.. _landice_ismip6_forcing_ocean_basal: ocean_basal ------------ -The ``landice.tests.ismip6_forcing.ocean_basal`` -performs processing of the coefficients for the basal melt parametrization +The ``landice/tests/ismip6_forcing/ocean_basal`` test case +performs processing of the coefficients for the basal melt parameterization utilized by the ISMIP6 protocol. Processing data includes combining the -IMBIE2 basin number file and parametrization coefficients and remapping onto +IMBIE2 basin numbers file and parameterization coefficients and remapping onto the MALI mesh. +.. _landice_ismip6_forcing_ocean_thermal: + +ocean_thermal +------------- + +The ``landice/ismip6_forcing/ocean_thermal`` test case +performs the processing of ocean thermal forcing. Processing data includes +regridding the original ISMIP6 thermal forcing data from its native +polarstereo grid to MALI's unstructured grid and renaming variables. From 4bd9ea3c3d6818e599ec4982da6f58a9564a63aa Mon Sep 17 00:00:00 2001 From: hollyhan Date: Thu, 15 Sep 2022 15:01:15 -0600 Subject: [PATCH 46/58] Edit the order of atmosphere step to Racmo step comes first before the SMB step --- compass/landice/tests/ismip6_forcing/atmosphere/__init__.py | 4 ++-- .../landice/tests/ismip6_forcing/atmosphere/process_smb.py | 5 +++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/compass/landice/tests/ismip6_forcing/atmosphere/__init__.py b/compass/landice/tests/ismip6_forcing/atmosphere/__init__.py index 7a380ce649..f8de7e3dad 100644 --- a/compass/landice/tests/ismip6_forcing/atmosphere/__init__.py +++ b/compass/landice/tests/ismip6_forcing/atmosphere/__init__.py @@ -28,10 +28,10 @@ def __init__(self, test_group): subdir = name super().__init__(test_group=test_group, name=name, subdir=subdir) - step = ProcessSMB(test_case=self) - self.add_step(step) step = ProcessSmbRacmo(test_case=self) self.add_step(step) + step = ProcessSMB(test_case=self) + self.add_step(step) def configure(self): """ diff --git a/compass/landice/tests/ismip6_forcing/atmosphere/process_smb.py b/compass/landice/tests/ismip6_forcing/atmosphere/process_smb.py index 8bd42c28fa..495844519c 100644 --- a/compass/landice/tests/ismip6_forcing/atmosphere/process_smb.py +++ b/compass/landice/tests/ismip6_forcing/atmosphere/process_smb.py @@ -96,8 +96,9 @@ def run(self): if not os.path.exists(racmo_clim_file_final): raise ValueError("Processed RACMO data does not exist, " "but it is required as an input file " - "to run this step. Please run `ProcessSmbRacmo` " - "step prior to running this step.") + "to run this step. Please run `process_smb_racmo`" + "step prior to running this step by setting" + "the config option 'process_smb_racmo' to 'True.") # temporary remapped climatology and anomaly files clim_ismip6_temp = "clim_ismip6.nc" From 59b59178db9656dea88422389f458f6f5fac07b6 Mon Sep 17 00:00:00 2001 From: hollyhan Date: Mon, 14 Nov 2022 10:32:58 -0700 Subject: [PATCH 47/58] Change 'warning' to 'print' for the 'process_smb_ramo' message --- .../tests/ismip6_forcing/atmosphere/process_smb_racmo.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/compass/landice/tests/ismip6_forcing/atmosphere/process_smb_racmo.py b/compass/landice/tests/ismip6_forcing/atmosphere/process_smb_racmo.py index 5f4d175ada..54c5badb5c 100644 --- a/compass/landice/tests/ismip6_forcing/atmosphere/process_smb_racmo.py +++ b/compass/landice/tests/ismip6_forcing/atmosphere/process_smb_racmo.py @@ -1,7 +1,6 @@ import os import shutil import xarray as xr -import warnings from mpas_tools.io import write_netcdf from mpas_tools.logging import check_call from compass.step import Step @@ -56,9 +55,9 @@ def setup(self): self.add_output_file(filename=output_file) else: - warnings.warn(f"'process_smb_racmo' is set to 'False'. This step" - f" will not run unless set 'True' in the" - f" config file.") + print(f"\n'Warning: process_smb_racmo' is set to 'False'." + f" This step will not run unless set 'True' in the" + f" config file.\n") def run(self): """ From f1d41185a36367635976229d26dc839bf8f09257 Mon Sep 17 00:00:00 2001 From: hollyhan Date: Thu, 15 Sep 2022 15:21:04 -0600 Subject: [PATCH 48/58] Assign a default remapping method for processs_ocean_basal --- .../tests/ismip6_forcing/ocean_basal/process_basal_melt.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/compass/landice/tests/ismip6_forcing/ocean_basal/process_basal_melt.py b/compass/landice/tests/ismip6_forcing/ocean_basal/process_basal_melt.py index 3b3e1a4df1..28ad8aad0e 100644 --- a/compass/landice/tests/ismip6_forcing/ocean_basal/process_basal_melt.py +++ b/compass/landice/tests/ismip6_forcing/ocean_basal/process_basal_melt.py @@ -61,9 +61,9 @@ def run(self): mali_mesh_name = section.get("mali_mesh_name") mali_mesh_file = section.get("mali_mesh_file") output_base_path = section.get("output_base_path") - - section = config["ismip6_ais_ocean_basal"] - method_remap = section.get("method_remap") + # we always want neareststod for the remapping method because we want + # a single value per basin + method_remap = "neareststod" # combine, interpolate and rename the basin file and deltaT0_gamma0 # ismip6 input files From 5f52278533aff000bfbc17cd202ed261aa660f18 Mon Sep 17 00:00:00 2001 From: hollyhan Date: Mon, 3 Oct 2022 10:42:26 -0600 Subject: [PATCH 49/58] Fix dictionary keys for the forcing files So that the keys (file path and names) match the updated original ISMIP6 data repo --- .../ismip6_forcing/atmosphere/process_smb.py | 30 +++++++++---------- .../ocean_thermal/process_thermal_forcing.py | 20 ++++++------- 2 files changed, 25 insertions(+), 25 deletions(-) diff --git a/compass/landice/tests/ismip6_forcing/atmosphere/process_smb.py b/compass/landice/tests/ismip6_forcing/atmosphere/process_smb.py index 495844519c..23ffd145d3 100644 --- a/compass/landice/tests/ismip6_forcing/atmosphere/process_smb.py +++ b/compass/landice/tests/ismip6_forcing/atmosphere/process_smb.py @@ -383,38 +383,38 @@ def correct_smb_anomaly_for_climatology(self, "2300": { "CCSM4": { "RCP85": [ - "AIS/Atmospheric_Forcing/CCSM4_RCP85/Regridded_08km/CCSM4_8km_anomaly_1995-2100.nc", - "AIS/Atmospheric_Forcing/CCSM4_RCP85/Regridded_08km/CCSM4_8km_anomaly_2101-2300.nc"] + "AIS/Atmospheric_forcing/CCSM4_RCP85/Regridded_08km/CCSM4_8km_anomaly_1995-2100.nc", + "AIS/Atmospheric_forcing/CCSM4_RCP85/Regridded_08km/CCSM4_8km_anomaly_2101-2300.nc"] }, "CESM2-WACCM": { "SSP585": [ - "AIS/Atmospheric_Forcing/CESM2-WACCM_ssp585/Regridded_8km/CESM2-WACCM_8km_anomaly_ssp585_1995-2100.nc", - "AIS/Atmospheric_Forcing/CESM2-WACCM_ssp585/Regridded_8km/CESM2-WACCM_8km_anomaly_ssp585_2101-2299.nc"], + "AIS/Atmospheric_forcing/CESM2-WACCM_ssp585/Regridded_8km/CESM2-WACCM_8km_anomaly_ssp585_1995-2100.nc", + "AIS/Atmospheric_forcing/CESM2-WACCM_ssp585/Regridded_8km/CESM2-WACCM_8km_anomaly_ssp585_2101-2299.nc"], "SSP585-repeat": [ - "AIS/Atmospheric_Forcing/CESM2-WACCM_ssp585-repeat/Regridded_8km/CESM2-WACCM_8km_anomaly_ssp585_1995-2300.nc"] + "AIS/Atmospheric_forcing/CESM2-WACCM_ssp585-repeat/Regridded_8km/CESM2-WACCM_8km_anomaly_ssp585_1995-2300_v2.nc"] }, "HadGEM2-ES": { "RCP85": [ - "AIS/Atmospheric_Forcing/HadGEM2-ES_RCP85/Regridded_8km/HadGEM2-ES_8km_anomaly_rcp85_1995-2100.nc", - "AIS/Atmospheric_Forcing/HadGEM2-ES_RCP85/Regridded_8km/HadGEM2-ES_8km_anomaly_rcp85_2101-2299.nc"], + "AIS/Atmospheric_forcing/HadGEM2-ES_RCP85/Regridded_8km/HadGEM2-ES_8km_anomaly_rcp85_1995-2100.nc", + "AIS/Atmospheric_forcing/HadGEM2-ES_RCP85/Regridded_8km/HadGEM2-ES_8km_anomaly_rcp85_2101-2299.nc"], "RCP85-repeat": [ - "AIS/Atmospheric_Forcing/HadGEM2-ES-RCP85-repeat/Regridded_8km/HadGEM2-ES_8km_anomaly_rcp85_1995-2300.nc"] + "AIS/Atmospheric_forcing/HadGEM2-ES-RCP85-repeat/Regridded_8km/HadGEM2-ES_8km_anomaly_rcp85_1995-2300_v2.nc"] }, "NorESM1-M": { "RCP26-repeat": [ - "AIS/Atmospheric_Forcing/NorESM1-M_RCP26-repeat/Regridded_8km/NorESM1-M_8km_anomaly_rcp26_1995-2300.nc"], + "AIS/Atmospheric_forcing/NorESM1-M_RCP26-repeat/Regridded_8km/NorESM1-M_8km_anomaly_rcp26_1995-2300_v2.nc"], "RCP85-repeat": [ - "AIS/Atmospheric_Forcing/NorESM1-M_RCP85-repeat/Regridded_8km/NorESM1-M_anomaly_1995-2300.nc"] + "AIS/Atmospheric_forcing/NorESM1-M_RCP85-repeat/Regridded_8km/NorESM1-M_8km_anomaly_1995-2300_v2.nc"] }, "UKESM1-0-LL": { "SSP126": [ - "AIS/Atmospheric_Forcing/UKESM1-0-LL_ssp126/Regridded_8km/UKESM1-0-LL_8km_anomaly_ssp126_1995-2100.nc" - "AIS/Atmospheric_Forcing/UKESM1-0-LL_ssp126/Regridded_8km/UKESM1-0-LL_8km_anomaly_ssp126_2101-2300.nc"], + "AIS/Atmospheric_forcing/UKESM1-0-LL_ssp126/Regridded_8km/UKESM1-0-LL_8km_anomaly_ssp126_1995-2100.nc", + "AIS/Atmospheric_forcing/UKESM1-0-LL_ssp126/Regridded_8km/UKESM1-0-LL_8km_anomaly_ssp126_2101-2300.nc"], "SSP585": [ - "AIS/Atmospheric_Forcing/UKESM1-0-LL_ssp585/Regridding_8km/UKESM1-0-LL_8km_anomaly_ssp585_1995-2100.nc", - "AIS/Atmospheric_Forcing/UKESM1-0-LL_ssp585/Regridding_8km/UKESM1-0-LL_8km_anomaly_ssp585_2101-2300.nc"], + "AIS/Atmospheric_forcing/UKESM1-0-LL_ssp585/Regridding_8km/UKESM1-0-LL_8km_anomaly_ssp585_1995-2100.nc", + "AIS/Atmospheric_forcing/UKESM1-0-LL_ssp585/Regridding_8km/UKESM1-0-LL_8km_anomaly_ssp585_2101-2300.nc"], "SSP585-repeat": [ - "AIS/Atmospheric_Forcing/UKESM1-0-LL_ssp585-repeat/Regridding_8km/UKESM1-0-LL_8km_anomaly_ssp585_1995-2300.nc"] + "AIS/Atmospheric_forcing/UKESM1-0-LL_ssp585-repeat/Regridding_8km/UKESM1-0-LL_8km_anomaly_ssp585_1995-2300_v2.nc"] } } } diff --git a/compass/landice/tests/ismip6_forcing/ocean_thermal/process_thermal_forcing.py b/compass/landice/tests/ismip6_forcing/ocean_thermal/process_thermal_forcing.py index 7397088909..e027bc93da 100644 --- a/compass/landice/tests/ismip6_forcing/ocean_thermal/process_thermal_forcing.py +++ b/compass/landice/tests/ismip6_forcing/ocean_thermal/process_thermal_forcing.py @@ -307,33 +307,33 @@ def rename_ismip6_thermal_forcing_to_mali_vars(self, "2300": { "CCSM4": { "RCP85": [ - "AIS/Ocean_Forcing/ccsm4_RCP85/1995-2300/CCSM4_RCP85_thermal_forcing_8km_x_60m.nc"] + "AIS/Ocean_forcing/ccsm4_RCP85/1995-2300/CCSM4_RCP85_thermal_forcing_8km_x_60m.nc"] }, "CESM2-WACCM": { "SSP585": [ - "AIS/Ocean_Forcing/cesm2-waccm_ssp585/1995-2299/CESM2-WACCM_SSP585_thermal_forcing_8km_x_60m.nc"], + "AIS/Ocean_forcing/cesm2-waccm_ssp585/1995-2299/CESM2-WACCM_SSP585_thermal_forcing_8km_x_60m.nc"], "SSP585-repeat": [ - "AIS/Ocean_Forcing/cesm2-waccm_ssp585-repeat/1995-2300/CESM2-WACCM_ssp585_thermal_forcing_8km_x_60m.nc"] + "AIS/Ocean_forcing/cesm2-waccm_ssp585-repeat/1995-2300/CESM2-WACCM_ssp585_thermal_forcing_8km_x_60m.nc"] }, "HadGEM2-ES": { "RCP85": [ - "AIS/Ocean_Forcing/hadgem2-es_RCP85/1995-2299/HadGEM2-ES_RCP85_thermal_forcing_8km_x_60m.nc"], + "AIS/Ocean_forcing/hadgem2-es_RCP85/1995-2299/HadGEM2-ES_RCP85_thermal_forcing_8km_x_60m.nc"], "RCP85-repeat": [ - "AIS/Ocean_Forcing/hadgem2-es_RCP85-repeat/1995-2300/HadGEM2-ES_rcp85_thermal_forcing_8km_x_60m.nc"] + "AIS/Ocean_forcing/hadgem2-es_RCP85-repeat/1995-2300/HadGEM2-ES_rcp85_thermal_forcing_8km_x_60m.nc"] }, "NorESM1-M": { "RCP26-repeat": [ - "AIS/Ocean_Forcing/noresm1-m_RCP26-repeat/1995-2300/NorESM1-M_RCP26_thermal_forcing_8km_x_60m.nc"], + "AIS/Ocean_forcing/noresm1-m_RCP26-repeat/1995-2300/NorESM1-M_RCP26_thermal_forcing_8km_x_60m.nc"], "RCP85-repeat": [ - "AIS/Ocean_Forcing/noresm1-m_RCP85-repeat/1995-2300/NorESM1-M_thermal_forcing_8km_x_60m.nc"] + "AIS/Ocean_forcing/noresm1-m_RCP85-repeat/1995-2300/NorESM1-M_thermal_forcing_8km_x_60m.nc"] }, "UKESM1-0-LL": { "SSP126": [ - "AIS/Ocean_Forcing/ukesm1-0-ll_ssp126/1995-2300/UKESM1-0-LL_thermal_forcing_8km_x_60m.nc"], + "AIS/Ocean_forcing/ukesm1-0-ll_ssp126/1995-2300/UKESM1-0-LL_thermal_forcing_8km_x_60m_v2.nc"], "SSP585": [ - "AIS/Ocean_Forcing/ukesm1-0-ll_ssp585/1995-2300/UKESM1-0-LL_SSP585_thermal_forcing_8km_x_60m.nc"], + "AIS/Ocean_forcing/ukesm1-0-ll_ssp585/1995-2300/UKESM1-0-LL_SSP585_thermal_forcing_8km_x_60m.nc"], "SSP585-repeat": [ - "AIS/Ocean_Forcing/ukesm1-0-ll_ssp585-repeat/1995-2300/UKESM1-0-LL_ssp585_thermal_forcing_8km_x_60m.nc"] + "AIS/Ocean_forcing/ukesm1-0-ll_ssp585-repeat/1995-2300/UKESM1-0-LL_ssp585_thermal_forcing_8km_x_60m.nc"] } } } From f92aa7244a05d44f7f3d7b9338dd2b33ab87977d Mon Sep 17 00:00:00 2001 From: hollyhan Date: Mon, 3 Oct 2022 14:18:55 -0600 Subject: [PATCH 50/58] Create a nested dictionary for basalmelt parameter files --- .../ocean_basal/process_basal_melt.py | 62 +++++++++++++------ 1 file changed, 42 insertions(+), 20 deletions(-) diff --git a/compass/landice/tests/ismip6_forcing/ocean_basal/process_basal_melt.py b/compass/landice/tests/ismip6_forcing/ocean_basal/process_basal_melt.py index 28ad8aad0e..84c995f74e 100644 --- a/compass/landice/tests/ismip6_forcing/ocean_basal/process_basal_melt.py +++ b/compass/landice/tests/ismip6_forcing/ocean_basal/process_basal_melt.py @@ -35,17 +35,18 @@ def setup(self): base_path_ismip6 = section.get("base_path_ismip6") base_path_mali = section.get("base_path_mali") mali_mesh_file = section.get("mali_mesh_file") + period_endyear = section.get("period_endyear") - section = config["ismip6_ais_ocean_basal"] + input_file_basin = self._file_basin[period_endyear] + input_files_coeff = self._files_coeff[period_endyear] self.add_input_file(filename=mali_mesh_file, target=os.path.join(base_path_mali, mali_mesh_file)) - self.add_input_file(filename=os.path.basename(self._file_basin), + self.add_input_file(filename=os.path.basename(input_file_basin), target=os.path.join(base_path_ismip6, - self._file_basin)) + input_file_basin)) - input_file_list = self._files_coeff - for file in input_file_list: + for file in input_files_coeff: self.add_input_file(filename=os.path.basename(file), target=os.path.join(base_path_ismip6, file)) self.add_output_file(filename=f"processed_basin_and_" @@ -60,6 +61,7 @@ def run(self): section = config["ismip6_ais"] mali_mesh_name = section.get("mali_mesh_name") mali_mesh_file = section.get("mali_mesh_file") + period_endyear = section.get("period_endyear") output_base_path = section.get("output_base_path") # we always want neareststod for the remapping method because we want # a single value per basin @@ -70,7 +72,8 @@ def run(self): # call the function that combines data logger.info(f"Combining the ismip6 input files") - input_file_list = self._files_coeff + input_file_basin = self._file_basin[period_endyear] + input_file_list = self._files_coeff[period_endyear] for i, file in enumerate(input_file_list): logger.info(f"!---Start processing the current file---!") @@ -82,7 +85,7 @@ def run(self): combined_file_temp = f"combined_{i}.nc" remapped_file_temp = f"remapped_{i}.nc" - self.combine_ismip6_inputfiles(os.path.basename(self._file_basin), + self.combine_ismip6_inputfiles(os.path.basename(input_file_basin), os.path.basename(file), combined_file_temp) @@ -230,16 +233,35 @@ def rename_ismip6_basal_melt_to_mali_vars(self, remapped_file_temp, # input files: input uniform melt rate coefficient (gamma0) # and temperature correction per basin - _file_basin = "AIS/Ocean_Forcing/imbie2/imbie2_basin_numbers_8km.nc" - _files_coeff = ["AIS/Ocean_Forcing/parameterizations/coeff_gamma0_DeltaT_quadratic_local_5th_pct_PIGL_gamma_calibration.nc", - "AIS/Ocean_Forcing/parameterizations/coeff_gamma0_DeltaT_quadratic_local_5th_percentile.nc", - "AIS/Ocean_Forcing/parameterizations/coeff_gamma0_DeltaT_quadratic_local_95th_pct_PIGL_gamma_calibration.nc", - "AIS/Ocean_Forcing/parameterizations/coeff_gamma0_DeltaT_quadratic_local_95th_percentile.nc", - "AIS/Ocean_Forcing/parameterizations/coeff_gamma0_DeltaT_quadratic_local_median_PIGL_gamma_calibration.nc", - "AIS/Ocean_Forcing/parameterizations/coeff_gamma0_DeltaT_quadratic_local_median.nc", - "AIS/Ocean_Forcing/parameterizations/coeff_gamma0_DeltaT_quadratic_non_local_5th_pct_PIGL_gamma_calibration.nc", - "AIS/Ocean_Forcing/parameterizations/coeff_gamma0_DeltaT_quadratic_non_local_5th_percentile.nc", - "AIS/Ocean_Forcing/parameterizations/coeff_gamma0_DeltaT_quadratic_non_local_95th_pct_PIGL_gamma_calibration.nc", - "AIS/Ocean_Forcing/parameterizations/coeff_gamma0_DeltaT_quadratic_non_local_95th_percentile.nc", - "AIS/Ocean_Forcing/parameterizations/coeff_gamma0_DeltaT_quadratic_non_local_median_PIGL_gamma_calibration.nc", - "AIS/Ocean_Forcing/parameterizations/coeff_gamma0_DeltaT_quadratic_non_local_median.nc"] + _file_basin = { + "2100": "AIS/Ocean_Forcing/imbie2/imbie2_basin_numbers_8km.nc", + "2300": "AIS/Ocean_forcing/imbie2/imbie2_basin_numbers_8km.nc" + } + + _files_coeff = { + "2100": ["AIS/Ocean_Forcing/parameterizations/coeff_gamma0_DeltaT_quadratic_local_5th_pct_PIGL_gamma_calibration.nc", + "AIS/Ocean_Forcing/parameterizations/coeff_gamma0_DeltaT_quadratic_local_5th_percentile.nc", + "AIS/Ocean_Forcing/parameterizations/coeff_gamma0_DeltaT_quadratic_local_95th_pct_PIGL_gamma_calibration.nc", + "AIS/Ocean_Forcing/parameterizations/coeff_gamma0_DeltaT_quadratic_local_95th_percentile.nc", + "AIS/Ocean_Forcing/parameterizations/coeff_gamma0_DeltaT_quadratic_local_median_PIGL_gamma_calibration.nc", + "AIS/Ocean_Forcing/parameterizations/coeff_gamma0_DeltaT_quadratic_local_median.nc", + "AIS/Ocean_Forcing/parameterizations/coeff_gamma0_DeltaT_quadratic_non_local_5th_pct_PIGL_gamma_calibration.nc", + "AIS/Ocean_Forcing/parameterizations/coeff_gamma0_DeltaT_quadratic_non_local_5th_percentile.nc", + "AIS/Ocean_Forcing/parameterizations/coeff_gamma0_DeltaT_quadratic_non_local_95th_pct_PIGL_gamma_calibration.nc", + "AIS/Ocean_Forcing/parameterizations/coeff_gamma0_DeltaT_quadratic_non_local_95th_percentile.nc", + "AIS/Ocean_Forcing/parameterizations/coeff_gamma0_DeltaT_quadratic_non_local_median_PIGL_gamma_calibration.nc", + "AIS/Ocean_Forcing/parameterizations/coeff_gamma0_DeltaT_quadratic_non_local_median.nc"], + + "2300": ["AIS/Ocean_forcing/parameterizations/coeff_gamma0_DeltaT_quadratic_local_5th_pct_PIGL_gamma_calibration.nc", + "AIS/Ocean_forcing/parameterizations/coeff_gamma0_DeltaT_quadratic_local_5th_percentile.nc", + "AIS/Ocean_forcing/parameterizations/coeff_gamma0_DeltaT_quadratic_local_95th_pct_PIGL_gamma_calibration.nc", + "AIS/Ocean_forcing/parameterizations/coeff_gamma0_DeltaT_quadratic_local_95th_percentile.nc", + "AIS/Ocean_forcing/parameterizations/coeff_gamma0_DeltaT_quadratic_local_median_PIGL_gamma_calibration.nc", + "AIS/Ocean_forcing/parameterizations/coeff_gamma0_DeltaT_quadratic_local_median.nc", + "AIS/Ocean_forcing/parameterizations/coeff_gamma0_DeltaT_quadratic_non_local_5th_pct_PIGL_gamma_calibration.nc", + "AIS/Ocean_forcing/parameterizations/coeff_gamma0_DeltaT_quadratic_non_local_5th_percentile.nc", + "AIS/Ocean_forcing/parameterizations/coeff_gamma0_DeltaT_quadratic_non_local_95th_pct_PIGL_gamma_calibration.nc", + "AIS/Ocean_forcing/parameterizations/coeff_gamma0_DeltaT_quadratic_non_local_95th_percentile.nc", + "AIS/Ocean_forcing/parameterizations/coeff_gamma0_DeltaT_quadratic_non_local_median_PIGL_gamma_calibration.nc", + "AIS/Ocean_forcing/parameterizations/coeff_gamma0_DeltaT_quadratic_non_local_median.nc"] + } \ No newline at end of file From c1723a4c1c87441d6a9f930de271a4115feca2bf Mon Sep 17 00:00:00 2001 From: hollyhan Date: Mon, 3 Oct 2022 14:19:48 -0600 Subject: [PATCH 51/58] Remove the config section for the ocean_basal testcase Previously, the ocean basal testcase had a config option for the remapping method. However, as we always want to use the "nearest neighborhood" method for this testcase, we decide to hard code the method in the main code "process_basal_melt.py" and remove the config option from the config file. --- compass/landice/tests/ismip6_forcing/ismip6_forcing.cfg | 6 ------ 1 file changed, 6 deletions(-) diff --git a/compass/landice/tests/ismip6_forcing/ismip6_forcing.cfg b/compass/landice/tests/ismip6_forcing/ismip6_forcing.cfg index 4b07f076d0..71f290be05 100644 --- a/compass/landice/tests/ismip6_forcing/ismip6_forcing.cfg +++ b/compass/landice/tests/ismip6_forcing/ismip6_forcing.cfg @@ -50,9 +50,3 @@ method_remap = bilinear # Set to True if want to process observational thermal forcing data. Set to False if want to process model thermal forcing data. process_obs_data = False -# config options for ismip6 ocean basal test case -[ismip6_ais_ocean_basal] - -# Remapping method used in building a mapping file. Note: ocean basal testcase will always want to -# use neareststod method. -method_remap = neareststod From 8e973a91e77632b1a4e60221f8f07c2839dac02e Mon Sep 17 00:00:00 2001 From: hollyhan Date: Thu, 10 Nov 2022 15:27:39 -0700 Subject: [PATCH 52/58] Add RACMO file from the database --- .../ismip6_forcing/atmosphere/process_smb_racmo.py | 12 ++++-------- .../landice/tests/ismip6_forcing/ismip6_forcing.cfg | 3 --- .../landice/test_groups/ismip6_forcing.rst | 13 +++---------- 3 files changed, 7 insertions(+), 21 deletions(-) diff --git a/compass/landice/tests/ismip6_forcing/atmosphere/process_smb_racmo.py b/compass/landice/tests/ismip6_forcing/atmosphere/process_smb_racmo.py index 54c5badb5c..ce3542bd47 100644 --- a/compass/landice/tests/ismip6_forcing/atmosphere/process_smb_racmo.py +++ b/compass/landice/tests/ismip6_forcing/atmosphere/process_smb_racmo.py @@ -45,11 +45,10 @@ def setup(self): mali_mesh_file)) if process_smb_racmo: - base_path_racmo = section.get("base_path_racmo") - input_file_list = self._files - self.add_input_file(filename=input_file_list[0], - target=os.path.join(base_path_racmo, - input_file_list[0])) + self.add_input_file(filename="RACMO2.3p2_ANT27_smb_yearly_" + "1979_2018.nc", + database="RACMO2.3p2_ANT27_smb_yearly_" + "1979_2018.nc") output_file = f"processed_RACMO2.3p2_ANT27" \ f"_smb_climatology_1995-2017.nc" @@ -250,6 +249,3 @@ def correct_smb_anomaly_for_base_smb(self, output_file, mali_mesh_file): # write to a new netCDF file write_netcdf(ds, output_file) ds.close() - - # create a dictionary for the regional climate RACMO dataset and the - _files = ["RACMO2.3p2_ANT27_smb_yearly_1979_2018.nc"] diff --git a/compass/landice/tests/ismip6_forcing/ismip6_forcing.cfg b/compass/landice/tests/ismip6_forcing/ismip6_forcing.cfg index 71f290be05..5a94a1c3ab 100644 --- a/compass/landice/tests/ismip6_forcing/ismip6_forcing.cfg +++ b/compass/landice/tests/ismip6_forcing/ismip6_forcing.cfg @@ -38,9 +38,6 @@ method_remap = bilinear # Set True to process RACMO modern climatology process_smb_racmo = True -# Full path to the RACMO regional climate model SMB data for modern climatology -base_path_racmo = NotAvailable - # config options for ismip6 ocean thermal forcing data test cases [ismip6_ais_ocean_thermal] diff --git a/docs/users_guide/landice/test_groups/ismip6_forcing.rst b/docs/users_guide/landice/test_groups/ismip6_forcing.rst index 503efc896d..11679fd3ae 100644 --- a/docs/users_guide/landice/test_groups/ismip6_forcing.rst +++ b/docs/users_guide/landice/test_groups/ismip6_forcing.rst @@ -82,10 +82,9 @@ lower case "f" for the ``AIS/Atmospheric_forcing/`` and ``AIS/Ocean_forcing/``. In addition to atmospheric and ocean thermal forcing files that correspond to specific climate model (e.g., UKESM1-0-LL, CCSM4) and scenarios (e.g., SSP585, RCP85, RCP26-repeat), modern -climatology files are needed. For the ``atmosphere`` testcase, users -need to download the RACMO modern climatology data -``RACMO2.3p2_ANT27_smb_yearly_1979_2018.nc`` from -`here. `_ +climatology files are needed. For the ``atmosphere`` testcase, +``RACMO2.3p2_ANT27_smb_yearly_1979_2018.nc`` will be automatically downloaded +from the database when the testcase is being set up. The RACMO file is used to correct the ISMIP6 the surface mass balance (SMB) data with the modern climatology. For the ``ocean_thermal`` case, users need to download the modern ocean thermal forcing climatology file named @@ -190,9 +189,6 @@ Below is the default config options: # Set True to process RACMO modern climatology process_smb_racmo = True - # Full path to the RACMO regional climate model SMB data for modern climatology - base_path_racmo = /Users/hollyhan/Desktop/ISMIP6_2300_Protocol/ISMIP6-Projections-Forcing-2300/RACMO2.3p2_ANT27_SMB_yearly_1979_2018/ - # config options for ismip6 ocean thermal forcing data test cases [ismip6_ais_ocean_thermal] @@ -253,9 +249,6 @@ process the RACMO modern SMB climatology but not the modern thermal forcing. # Set True to process RACMO modern climatology process_smb_racmo = True - # Full path to the RACMO regional climate model SMB data for modern climatology - base_path_racmo = NotAvailable - # config options for ismip6 ocean thermal forcing data test cases [ismip6_ais_ocean_thermal] From 5e4d994658595a0eea8ec32c1f7fb23bb73990df Mon Sep 17 00:00:00 2001 From: hollyhan Date: Thu, 10 Nov 2022 15:45:58 -0700 Subject: [PATCH 53/58] Specify MALI mesh name in the processed file name --- .../tests/ismip6_forcing/atmosphere/process_smb.py | 11 ++++++----- .../ismip6_forcing/atmosphere/process_smb_racmo.py | 12 ++++++------ .../ismip6_forcing/ocean_basal/process_basal_melt.py | 6 ++++-- .../ocean_thermal/process_thermal_forcing.py | 9 +++++---- 4 files changed, 21 insertions(+), 17 deletions(-) diff --git a/compass/landice/tests/ismip6_forcing/atmosphere/process_smb.py b/compass/landice/tests/ismip6_forcing/atmosphere/process_smb.py index 23ffd145d3..22c12aa34a 100644 --- a/compass/landice/tests/ismip6_forcing/atmosphere/process_smb.py +++ b/compass/landice/tests/ismip6_forcing/atmosphere/process_smb.py @@ -38,6 +38,7 @@ def setup(self): base_path_ismip6 = section.get("base_path_ismip6") base_path_mali = section.get("base_path_mali") output_base_path = section.get("output_base_path") + mali_mesh_name = section.get("mali_mesh_name") mali_mesh_file = section.get("mali_mesh_file") period_endyear = section.get("period_endyear") model = section.get("model") @@ -53,12 +54,12 @@ def setup(self): target=os.path.join(base_path_ismip6, file)) - output_file_esm = f"processed_SMB_{model}_{scenario}_" \ + output_file_esm = f"{mali_mesh_name}_SMB_{model}_{scenario}_" \ f"{period_endyear}.nc" self.add_output_file(filename=output_file_esm) # add processed racmo data as input as it is needed for smb correction - racmo_clim_file = f"processed_RACMO2.3p2_ANT27" \ + racmo_clim_file = f"{mali_mesh_name}_RACMO2.3p2_ANT27" \ f"_smb_climatology_1995-2017.nc" racmo_path = f"{output_base_path}/atmosphere_forcing/" \ f"RACMO_climatology_1995-2017" @@ -87,7 +88,7 @@ def run(self): # define file names needed # input racmo climotology file - racmo_clim_file = f"processed_RACMO2.3p2_ANT27" \ + racmo_clim_file = f"{mali_mesh_name}_RACMO2.3p2_ANT27" \ f"_smb_climatology_1995-2017.nc" racmo_path = f"{output_base_path}/atmosphere_forcing/" \ f"RACMO_climatology_1995-2017" @@ -105,9 +106,9 @@ def run(self): remapped_clim_ismip6_temp = "remapped_clim_ismip6.nc" remapped_anomaly_ismip6_temp = "remapped_anomaly_ismip6.nc" # renamed remapped climatology and anomaly files (final outputs) - output_clim_ismip6 = f"processed_SMB_climatology_1995-2017_" \ + output_clim_ismip6 = f"{mali_mesh_name}_SMB_climatology_1995-2017_" \ f"{model}_{scenario}.nc" - output_anomaly_ismip6 = f"processed_SMB_{model}_{scenario}_" \ + output_anomaly_ismip6 = f"{mali_mesh_name}_SMB_{model}_{scenario}_" \ f"{period_endyear}.nc" # combine ismip6 forcing data covering different periods diff --git a/compass/landice/tests/ismip6_forcing/atmosphere/process_smb_racmo.py b/compass/landice/tests/ismip6_forcing/atmosphere/process_smb_racmo.py index ce3542bd47..8eaaaf36b7 100644 --- a/compass/landice/tests/ismip6_forcing/atmosphere/process_smb_racmo.py +++ b/compass/landice/tests/ismip6_forcing/atmosphere/process_smb_racmo.py @@ -35,6 +35,7 @@ def setup(self): config = self.config section = config["ismip6_ais"] base_path_mali = section.get("base_path_mali") + mali_mesh_name = section.get("mali_mesh_name") mali_mesh_file = section.get("mali_mesh_file") section = config["ismip6_ais_atmosphere"] @@ -45,11 +46,10 @@ def setup(self): mali_mesh_file)) if process_smb_racmo: - self.add_input_file(filename="RACMO2.3p2_ANT27_smb_yearly_" - "1979_2018.nc", - database="RACMO2.3p2_ANT27_smb_yearly_" - "1979_2018.nc") - output_file = f"processed_RACMO2.3p2_ANT27" \ + self.add_input_file( + target="RACMO2.3p2_ANT27_smb_yearly_1979_2018.nc", + database="RACMO2.3p2_ANT27_SMB_yearly_1979_2018") + output_file = f"{mali_mesh_name}_RACMO2.3p2_ANT27" \ f"_smb_climatology_1995-2017.nc" self.add_output_file(filename=output_file) @@ -76,7 +76,7 @@ def run(self): racmo_file_temp1 = "RACMO2.3p2_smb_climatology_1995_2017.nc" racmo_file_temp2 = "RACMO2.3p2_smb_climatology_1995_2017_" \ "correct_unit.nc" - output_file = f"processed_RACMO2.3p2_ANT27" \ + output_file = f"{mali_mesh_name}_RACMO2.3p2_ANT27" \ f"_smb_climatology_1995-2017.nc" output_file_final = os.path.join(output_base_path, output_file) diff --git a/compass/landice/tests/ismip6_forcing/ocean_basal/process_basal_melt.py b/compass/landice/tests/ismip6_forcing/ocean_basal/process_basal_melt.py index 84c995f74e..9497b8f3d6 100644 --- a/compass/landice/tests/ismip6_forcing/ocean_basal/process_basal_melt.py +++ b/compass/landice/tests/ismip6_forcing/ocean_basal/process_basal_melt.py @@ -34,6 +34,7 @@ def setup(self): section = config["ismip6_ais"] base_path_ismip6 = section.get("base_path_ismip6") base_path_mali = section.get("base_path_mali") + mali_mesh_name = section.get("mali_mesh_name") mali_mesh_file = section.get("mali_mesh_file") period_endyear = section.get("period_endyear") @@ -49,7 +50,7 @@ def setup(self): for file in input_files_coeff: self.add_input_file(filename=os.path.basename(file), target=os.path.join(base_path_ismip6, file)) - self.add_output_file(filename=f"processed_basin_and_" + self.add_output_file(filename=f"{mali_mesh_name}_basin_and_" f"{os.path.basename(file)}") def run(self): @@ -97,7 +98,8 @@ def run(self): mali_mesh_file, method_remap) - output_file = f"processed_basin_and_{os.path.basename(file)}" + output_file = f"{mali_mesh_name}_basin_and_" \ + f"{os.path.basename(file)}" # rename the ismip6 variables to MALI variables logger.info(f"Renaming the ismip6 variables to " diff --git a/compass/landice/tests/ismip6_forcing/ocean_thermal/process_thermal_forcing.py b/compass/landice/tests/ismip6_forcing/ocean_thermal/process_thermal_forcing.py index e027bc93da..e1230e4703 100644 --- a/compass/landice/tests/ismip6_forcing/ocean_thermal/process_thermal_forcing.py +++ b/compass/landice/tests/ismip6_forcing/ocean_thermal/process_thermal_forcing.py @@ -35,6 +35,7 @@ def setup(self): section = config["ismip6_ais"] base_path_ismip6 = section.get("base_path_ismip6") base_path_mali = section.get("base_path_mali") + mali_mesh_name = section.get("mali_mesh_name") mali_mesh_file = section.get("mali_mesh_file") period_endyear = section.get("period_endyear") model = section.get("model") @@ -49,10 +50,10 @@ def setup(self): if process_obs_data: input_file = self._file_obs - output_file = f"processed_obs_TF_1995-2017_8km_x_60m.nc" + output_file = f"{mali_mesh_name}_obs_TF_1995-2017_8km_x_60m.nc" else: input_file = self._files[period_endyear][model][scenario] - output_file = f"processed_TF_{model}_{scenario}_" \ + output_file = f"{mali_mesh_name}_TF_{model}_{scenario}_" \ f"{period_endyear}.nc" self.add_input_file(filename=os.path.basename(input_file[0]), @@ -81,12 +82,12 @@ def run(self): if process_obs_data: input_file_list = self._file_obs - output_file = f"processed_obs_TF_1995-2017_8km_x_60m.nc" + output_file = f"{mali_mesh_name}_obs_TF_1995-2017_8km_x_60m.nc" output_path = f"{output_base_path}/ocean_thermal_forcing/"\ f"obs" else: input_file_list = self._files[period_endyear][model][scenario] - output_file = f"processed_TF_" \ + output_file = f"{mali_mesh_name}_TF_" \ f"{model}_{scenario}_{period_endyear}.nc" output_path = f"{output_base_path}/ocean_thermal_forcing/" \ f"{model}_{scenario}/1995-{period_endyear}" From c5956ec5fe2f281f6a0295e9ab8a129c15fbc61c Mon Sep 17 00:00:00 2001 From: hollyhan Date: Thu, 10 Nov 2022 16:59:43 -0700 Subject: [PATCH 54/58] Add metadata to the processed SMB file --- .../landice/tests/ismip6_forcing/atmosphere/process_smb.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/compass/landice/tests/ismip6_forcing/atmosphere/process_smb.py b/compass/landice/tests/ismip6_forcing/atmosphere/process_smb.py index 22c12aa34a..c4851f7188 100644 --- a/compass/landice/tests/ismip6_forcing/atmosphere/process_smb.py +++ b/compass/landice/tests/ismip6_forcing/atmosphere/process_smb.py @@ -323,6 +323,11 @@ def correct_smb_anomaly_for_climatology(self, # correct the ismip6 smb anomaly ds["sfcMassBal"] = ds["sfcMassBal"] + corr_clim + # write metadata + ds["sfcMassBal"].attrs = {"long_name" : "surface mass balance", + "units" : "kg m-2 s-1", + "coordinates" : "lat lon"} + # write to a new netCDF file write_netcdf(ds, output_file_final) ds.close() From 52ed5bd9c1ff00e42c2bb6edb50313200a567719 Mon Sep 17 00:00:00 2001 From: hollyhan Date: Thu, 10 Nov 2022 18:15:53 -0700 Subject: [PATCH 55/58] Add a separate testcase for observation ocean thermal forcing data --- .../landice/tests/ismip6_forcing/__init__.py | 3 +- .../tests/ismip6_forcing/ismip6_forcing.cfg | 4 - .../ismip6_forcing/ocean_thermal/__init__.py | 23 ++++-- .../ocean_thermal/process_thermal_forcing.py | 19 +++-- .../landice/test_groups/ismip6_forcing.rst | 30 ++++---- .../landice/test_groups/ismip6_forcing.rst | 76 ++++++++++++------- 6 files changed, 95 insertions(+), 60 deletions(-) diff --git a/compass/landice/tests/ismip6_forcing/__init__.py b/compass/landice/tests/ismip6_forcing/__init__.py index 8cc1734dc6..0d0fed1383 100644 --- a/compass/landice/tests/ismip6_forcing/__init__.py +++ b/compass/landice/tests/ismip6_forcing/__init__.py @@ -22,4 +22,5 @@ def __init__(self, mpas_core): self.add_test_case(Atmosphere(test_group=self)) self.add_test_case(OceanBasal(test_group=self)) - self.add_test_case(OceanThermal(test_group=self)) + self.add_test_case(OceanThermal(test_group=self, process_obs=True)) + self.add_test_case(OceanThermal(test_group=self, process_obs=False)) diff --git a/compass/landice/tests/ismip6_forcing/ismip6_forcing.cfg b/compass/landice/tests/ismip6_forcing/ismip6_forcing.cfg index 5a94a1c3ab..6b88abcd80 100644 --- a/compass/landice/tests/ismip6_forcing/ismip6_forcing.cfg +++ b/compass/landice/tests/ismip6_forcing/ismip6_forcing.cfg @@ -43,7 +43,3 @@ process_smb_racmo = True # Remapping method used in building a mapping file. Options include: bilinear, neareststod, conserve method_remap = bilinear - -# Set to True if want to process observational thermal forcing data. Set to False if want to process model thermal forcing data. -process_obs_data = False - diff --git a/compass/landice/tests/ismip6_forcing/ocean_thermal/__init__.py b/compass/landice/tests/ismip6_forcing/ocean_thermal/__init__.py index 958b5fd4c0..5b627c5c9a 100644 --- a/compass/landice/tests/ismip6_forcing/ocean_thermal/__init__.py +++ b/compass/landice/tests/ismip6_forcing/ocean_thermal/__init__.py @@ -11,9 +11,14 @@ class OceanThermal(TestCase): The test case builds a mapping file for interpolation between the ISMIP6 8km polarstereo grid and MALI mesh, regrids the forcing data and renames the ISMIP6 variables to corresponding MALI variables. + + Attributes + ---------- + process_obs : bool + Whether we are processing observations rather than CMIP model data """ - def __init__(self, test_group): + def __init__(self, test_group, process_obs): """ Create the test case @@ -21,19 +26,25 @@ def __init__(self, test_group): ---------- test_group : compass.landice.tests.ismip6_forcing.Ismip6Forcing The test group that this test case belongs to + + process_obs: bool + Whether we are processing observations rather than CMIP model data """ - name = "ocean_thermal" + if process_obs: + name = "ocean_thermal_obs" + else: + name = "ocean_thermal" + self.process_obs = process_obs super().__init__(test_group=test_group, name=name) - step = ProcessThermalForcing(test_case=self) + step = ProcessThermalForcing(test_case=self, + process_obs=self.process_obs) self.add_step(step) def configure(self): """ Configures test case """ - process_obs_data = self.config.getboolean("ismip6_ais_ocean_thermal", - "process_obs_data") - check_model_options = not process_obs_data + check_model_options = not self.process_obs configure_testgroup(config=self.config, check_model_options=check_model_options) diff --git a/compass/landice/tests/ismip6_forcing/ocean_thermal/process_thermal_forcing.py b/compass/landice/tests/ismip6_forcing/ocean_thermal/process_thermal_forcing.py index e1230e4703..1fb29fa9c9 100644 --- a/compass/landice/tests/ismip6_forcing/ocean_thermal/process_thermal_forcing.py +++ b/compass/landice/tests/ismip6_forcing/ocean_thermal/process_thermal_forcing.py @@ -13,8 +13,13 @@ class ProcessThermalForcing(Step): """ A step for creating a mesh and initial condition for dome test cases + + Attributes + ---------- + process_obs : bool + Whether we are processing observations rather than CMIP model data """ - def __init__(self, test_case): + def __init__(self, test_case, process_obs): """ Create the step @@ -23,9 +28,13 @@ def __init__(self, test_case): test_case : compass.landice.tests.ismip6_forcing.ocean_thermal. OceanThermal The test case this step belongs to + + process_obs : bool + Whether we are processing observations rather than CMIP model data """ super().__init__(test_case=test_case, name="process_thermal_forcing", ntasks=4, min_tasks=1) + self.process_obs = process_obs def setup(self): """ @@ -41,8 +50,7 @@ def setup(self): model = section.get("model") scenario = section.get("scenario") - section = config["ismip6_ais_ocean_thermal"] - process_obs_data = section.getboolean("process_obs_data") + process_obs_data = self.process_obs self.add_input_file(filename=mali_mesh_file, target=os.path.join(base_path_mali, @@ -78,7 +86,7 @@ def run(self): section = config["ismip6_ais_ocean_thermal"] method_remap = section.get("method_remap") - process_obs_data = section.getboolean("process_obs_data") + process_obs_data = self.process_obs if process_obs_data: input_file_list = self._file_obs @@ -191,8 +199,7 @@ def rename_ismip6_thermal_forcing_to_mali_vars(self, """ config = self.config - section = config["ismip6_ais_ocean_thermal"] - process_obs_data = section.getboolean("process_obs_data") + process_obs_data = self.process_obs # open dataset in 20 years chunk ds = xr.open_dataset(remapped_file_temp, chunks=dict(time=20), diff --git a/docs/developers_guide/landice/test_groups/ismip6_forcing.rst b/docs/developers_guide/landice/test_groups/ismip6_forcing.rst index 8f853d8ec1..281fef9d8d 100644 --- a/docs/developers_guide/landice/test_groups/ismip6_forcing.rst +++ b/docs/developers_guide/landice/test_groups/ismip6_forcing.rst @@ -7,11 +7,11 @@ The ``ismip6_forcing`` test group (:py:class:`compass.landice.tests.ismip6_ forcing.Ismip6Forcing`) processes (i.e., remapping and renaming) the atmospheric and oceanic forcing data of the Ice Sheet Model Intercomparison for CMIP6 (ISMIP6) protocol from its native polarstereo grid to -the unstructure MALI mesh. The test group includes three test cases, -``atmosphere``, ``ocean_basal`` and ``ocean_thermal``. The ``atmosphere`` -test case has two steps: ``process_smb`` and ``process_smb_racmo``; +the unstructure MALI mesh. The test group includes four test cases, +``atmosphere``, ``ocean_basal``, ``ocean_thermal_obs`` and ``ocean_thermal``. +The ``atmosphere`` test case has two steps: ``process_smb`` and ``process_smb_racmo``; the ``ocean_basal`` test case has one step, ``process_basal_melt``; -the ``ocean_thermal`` has one step, ``process_thermal_forcing``. +the ``ocean_thermal_obs`` and ``ocean_thermal`` share one step, ``process_thermal_forcing``. Each step has the local methods (functions) of remapping and renaming the original ismip6 data to the format that MALI can incorporate in its forward simulations. In remapping the data, all test cases import the @@ -40,16 +40,6 @@ Processing data includes regridding the original ISMIP6 SMB data from its native polarstereo grid to MALI's unstructured grid, renaming variables and correcting the SMB anomaly field for the MALI base SMB. -.. _dev_landice_ismip6_forcing_ocean_thermal: - -ocean_thermal -------------- - -The :py:class:`compass.landice.tests.ismip6_forcing.ocean_thermal.OceanThermal` -performs the processing of ocean thermal forcing. Processing data includes -regridding the original ISMIP6 thermal forcing data from its native -polarstereo grid to MALI's unstructured grid and renaming variables. - .. _dev_landice_ismip6_forcing_ocean_basal: ocean_basal @@ -60,3 +50,15 @@ performs processing of the coefficients for the basal melt parameterization utilized by the ISMIP6 protocol. Processing data includes combining the IMBIE2 basin number file and parameterization coefficients and remapping onto the MALI mesh. + +.. _dev_landice_ismip6_forcing_ocean_thermal: + +ocean_thermal +------------- + +The :py:class:`compass.landice.tests.ismip6_forcing.ocean_thermal.OceanThermal` +performs the processing of ocean thermal forcing, both observational climatology +(in ``ocean_thermal_obs``) and the CMIP model data (in ``ocean_thermal``). +Processing data includes regridding the original ISMIP6 thermal forcing data +from its native polarstereo grid to MALI's unstructured grid and renaming variables. + diff --git a/docs/users_guide/landice/test_groups/ismip6_forcing.rst b/docs/users_guide/landice/test_groups/ismip6_forcing.rst index 11679fd3ae..fdb48a3a19 100644 --- a/docs/users_guide/landice/test_groups/ismip6_forcing.rst +++ b/docs/users_guide/landice/test_groups/ismip6_forcing.rst @@ -7,17 +7,19 @@ The ``landice/ismip6_forcing`` test group processes (i.e., remaps and renames) the atmospheric and ocean forcing data of the Ice Sheet Model Intercomparison for CMIP6 (ISMIP6) protocol. The processed data is used to force MALI in its simulations under a relevant ISMIP6 (either the 2100 or 2300) experimental protocol. -The test group includes three test cases, ``atmosphere``, ``ocean_basal`` and -``ocean_thermal``; the ``atmosphere`` test case has two steps: -``process_smb`` and ``process_smb_racmo``. The ``ocean_basal`` and the -``ocean_thermal`` test case each has one step, ``process_basal_melt``, and -``process_thermal_forcing``, respectively. (For more details on the steps of +The test group includes three test cases, ``atmosphere``, ``ocean_basal``, +``ocean_thermal_obs`` and ``ocean_thermal``; the ``atmosphere`` test case +has two steps: ``process_smb`` and ``process_smb_racmo``. The ``ocean_basal`` +and the ``ocean_thermal`` test case each has one step, ``process_basal_melt``, +and ``process_thermal_forcing``, respectively. (For more details on the steps of each test case, see :ref:`landice_ismip6_forcing_atmosphere`, -:ref:`landice_ismip6_forcing_ocean_basal` and +:ref:`landice_ismip6_forcing_ocean_basal`, +:ref:`landice_ismip6_forcing_ocean_thermal_obs` and :ref:`landice_ismip6_forcing_ocean_thermal`.) Approximated time for processing a single forcing file -on Cori (single core) is 2, 7, and 1 minutes for the atmosphere, ocean basal, -and ocean thermal testcase, respectively. +on Cori (single core) is 2 and 7 minutes for the atmosphere and ocean basal +testcases, and less than a minute for ocean thermal obs and ocean thermal +testcases, respectively. Before providing the details of necessary source data of the ISMIP6 protocols, we provide a summary of instructions of an overall process of this test group: @@ -28,16 +30,20 @@ be provided as soon as they are publicly available. Then, for a given MALI mesh, 1. run :ref:`landice_ismip6_forcing_ocean_basal` once, independent of the model, scenario and end year. -2. run :ref:`landice_ismip6_forcing_ocean_thermal` once with -``process_obs_data = True`` in the config file (:ref:`landice_ismip6_forcing_config`) -to process the thermal forcing from the observational climatology (used for +2. run :ref:`landice_ismip6_forcing_ocean_thermal_obs` once with, independent +of the model, scenario and end year, to process the thermal forcing from the observational climatology (used for control runs). -3. for each model, scenario and end year, run :ref:`landice_ismip6_forcing_ocean_thermal`, -and run :ref:`landice_ismip6_forcing_ocean_atmosphere` with -``process_racmo_smb = True`` once with any model, and run with subsequent -models with ``process_racmo_smb = False``, but it is harmless to run it again -with ``process_racmo_smb = True`` as it does nothing if data is already +3. run :ref:`landice_ismip6_forcing_ocean_thermal` for each model, scenario +and end year. + +4. run :ref:`landice_ismip6_forcing_ocean_atmosphere` with +``process_racmo_smb = True`` once with any model. + +5. run :ref:`landice_ismip6_forcing_ocean_thermal` for each model, +scenario and end year. Users can keep ``process_racmo_smb = False`` as far as +the RACMO SMB has been processed once in Step 4, but it is harmless to leave +``process_racmo_smb = True`` as it does nothing if data is already available, and the processing is very quick (less than a minute). There are six different of input data sets other than the MALI mesh that @@ -84,7 +90,8 @@ correspond to specific climate model (e.g., UKESM1-0-LL, CCSM4) and scenarios (e.g., SSP585, RCP85, RCP26-repeat), modern climatology files are needed. For the ``atmosphere`` testcase, ``RACMO2.3p2_ANT27_smb_yearly_1979_2018.nc`` will be automatically downloaded -from the database when the testcase is being set up. +from the MALI public database when the testcase is being set up and saved +to the directory that users define in the config option `landice_database_root`. The RACMO file is used to correct the ISMIP6 the surface mass balance (SMB) data with the modern climatology. For the ``ocean_thermal`` case, users need to download the modern ocean thermal forcing climatology file named @@ -119,7 +126,7 @@ and where processed files will be saved. config options -------------- -All three test cases share some set of default config options under the section +All four test cases share some set of default config options under the section ``[ismip6_ais]`` and have separate config options for each test case: ``[ismip6_ais_atmosphere]``, ``[ismip6_ais_ocean_thermal]``, and ``[ismip6_ais_ocean_basal]``. In the general config section @@ -130,12 +137,7 @@ from the available options as given in the config file (see the example file below.) In the ``ismip6_ais_atmosphere`` section, users need to indicate ``True`` or ``False`` on whether to process the RACMO modern climatology (``True`` is required to run the ``process_smb_racmo`` step, which needs to be -run before the ``process_smb`` step). In the ``ismip6_ais_ocean_thermal`` -section, users need to indicate ``True`` or ``False`` and on whether to process -the observational thermal forcing data, which represents the modern ocean -climatology between 1995-2017. When ``process_obs_data`` is set to ``True``, -ocean forcing anomaly files from climate models are not processed (these files -will only be processed when ``process_obs_data`` is set to ``False``) +run before the ``process_smb`` step). For most the ``[ismip6_ais_atmosphere]`` and ``[ismip6_ais_ocean_thermal]`` config sections users may choose the interpolation scheme among @@ -148,6 +150,10 @@ Below is the default config options: .. code-block:: cfg # config options for ismip6 antarctic ice sheet data set + [paths] + # The root to a location where data files for MALI will be cached + landice_database_root = /Users/hollyhan/Desktop/RESEARCH/MALI/database/ + [ismip6_ais] # Base path to the input ismip6 ocean and smb forcing files. User has to supply. @@ -195,8 +201,7 @@ Below is the default config options: # Remapping method used in building a mapping file. Options include: bilinear, neareststod, conserve method_remap = bilinear - # Set to True if want to process observational thermal forcing data. Set to False if want to process model thermal forcing data. - # Note: when set True, the ['ismip6_ais'] config options 'period_endyear', 'model' and 'scenario' will be ignored. + # Set to True if the want to process observational thermal forcing data. Set to False if want to process model thermal forcing data. process_obs_data = True Below is the example config options that users might create in running @@ -208,6 +213,10 @@ process the RACMO modern SMB climatology but not the modern thermal forcing. .. code-block:: cfg # config options for ismip6 antarctic ice sheet data set + [paths] + # The root to a location where data files for MALI will be cached + landice_database_root = NotAvailable + [ismip6_ais] # Base path to the input ismip6 ocean and smb forcing files. User has to supply. @@ -255,11 +264,9 @@ process the RACMO modern SMB climatology but not the modern thermal forcing. # Remapping method used in building a mapping file. Options include: bilinear, neareststod, conserve method_remap = bilinear - # Set to True if want to process observational thermal forcing data. Set to False if want to process model thermal forcing data. - # Note: when set True, the ['ismip6_ais'] config options 'period_endyear', 'model' and 'scenario' will be ignored. + # Set to True if the want to process observational thermal forcing data. Set to False if want to process model thermal forcing data. process_obs_data = True - .. _landice_ismip6_forcing_atmosphere: atmosphere @@ -284,6 +291,17 @@ utilized by the ISMIP6 protocol. Processing data includes combining the IMBIE2 basin numbers file and parameterization coefficients and remapping onto the MALI mesh. +.. _landice_ismip6_forcing_ocean_thermal_obs: + +ocean_thermal_obs +----------------- + +The ``landice/ismip6_forcing/ocean_thermal_obs`` test case +performs the processing of the observational climatology of +ocean thermal forcing. Processing data includes regridding the original ISMIP6 +thermal forcing data from its native polarstereo grid to MALI's unstructured +grid and renaming variables. + .. _landice_ismip6_forcing_ocean_thermal: ocean_thermal From 42c9f1a2e5e9b498447dadb0994d5fe3e24daaa7 Mon Sep 17 00:00:00 2001 From: hollyhan Date: Tue, 15 Nov 2022 10:29:02 -0700 Subject: [PATCH 56/58] Update the user guide --- .../landice/test_groups/ismip6_forcing.rst | 21 ++++++++++--------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/docs/users_guide/landice/test_groups/ismip6_forcing.rst b/docs/users_guide/landice/test_groups/ismip6_forcing.rst index fdb48a3a19..a16436d33b 100644 --- a/docs/users_guide/landice/test_groups/ismip6_forcing.rst +++ b/docs/users_guide/landice/test_groups/ismip6_forcing.rst @@ -30,18 +30,18 @@ be provided as soon as they are publicly available. Then, for a given MALI mesh, 1. run :ref:`landice_ismip6_forcing_ocean_basal` once, independent of the model, scenario and end year. -2. run :ref:`landice_ismip6_forcing_ocean_thermal_obs` once with, independent +2. run :ref:`landice_ismip6_forcing_ocean_thermal_obs` once, independent of the model, scenario and end year, to process the thermal forcing from the observational climatology (used for control runs). 3. run :ref:`landice_ismip6_forcing_ocean_thermal` for each model, scenario and end year. -4. run :ref:`landice_ismip6_forcing_ocean_atmosphere` with +4. run :ref:`landice_ismip6_forcing_atmosphere` with ``process_racmo_smb = True`` once with any model. -5. run :ref:`landice_ismip6_forcing_ocean_thermal` for each model, -scenario and end year. Users can keep ``process_racmo_smb = False`` as far as +5. run :ref:`landice_ismip6_forcing_atmosphere` for each model, +scenario and end year. Users can keep ``process_racmo_smb = False`` as long as the RACMO SMB has been processed once in Step 4, but it is harmless to leave ``process_racmo_smb = True`` as it does nothing if data is already available, and the processing is very quick (less than a minute). @@ -69,19 +69,20 @@ provided by the ISMIP6-2100 protocol (i.e., from the ``GHub-ISMIP6-Forcing`` Globus endpoint), they must create the directory path ``AIS/Atmosphere_Forcing/UKESM1-0-LL/Regridded_8km/`` in their local system where they will download the file ``UKESM1-0-LL_anomaly_ssp585_1995-2100_8km.nc``. -Note that the users only need to download SMB anomaly files (not the climatology -file) that cover the period from 1995 to ``period_endyear`` -(either 2100 or 2300, defined in the config file; +Note that the users only need to download SMB anomaly files in 8km resolution +(the climatology file is not needed) that cover the period from 1995 to +``period_endyear`` (either 2100 or 2300, defined in the config file; see :ref:`landice_ismip6_forcing_config`). Equivalently, for the -ocean forcing, the user should create the directory path +ocean forcing in this example, users should create the directory path ``AIS/Ocean_Forcing/ukesm1-0-ll_ssp585/1995-2100/`` and download the file ``UKESM1-0-LL_ssp585_thermal_forcing_8km_x_60m.nc`` from the same endpoint. Users do not need to download the thermal forcing files for the years previous to 1995 as only the files downloaded from -``1995-{period_endyear}`` will be processed. -Also note to be ware that unlike those in the ``GHub-ISMIP6-Forcing`` endpoint, +``1995-{period_endyear}`` will be processed. Users also do not need to download +the temperature and salinity files, as these will not be used by MALI. +Also note to be aware that unlike those in the ``GHub-ISMIP6-Forcing`` endpoint, the directory names in the ``ISMIP6-Projections-Forcing-2300`` endpoint have a lower case "f" for the ``AIS/Atmospheric_forcing/`` and ``AIS/Ocean_forcing/``. From 94138aac6bcf0a33fe3013518bd9a34220aa8870 Mon Sep 17 00:00:00 2001 From: hollyhan Date: Tue, 15 Nov 2022 13:50:49 -0700 Subject: [PATCH 57/58] Fix input file list and the use of 'f-string' arguments --- .../atmosphere/process_smb_racmo.py | 35 +++++++++++-------- 1 file changed, 21 insertions(+), 14 deletions(-) diff --git a/compass/landice/tests/ismip6_forcing/atmosphere/process_smb_racmo.py b/compass/landice/tests/ismip6_forcing/atmosphere/process_smb_racmo.py index 8eaaaf36b7..5ffdd35da7 100644 --- a/compass/landice/tests/ismip6_forcing/atmosphere/process_smb_racmo.py +++ b/compass/landice/tests/ismip6_forcing/atmosphere/process_smb_racmo.py @@ -49,9 +49,9 @@ def setup(self): self.add_input_file( target="RACMO2.3p2_ANT27_smb_yearly_1979_2018.nc", database="RACMO2.3p2_ANT27_SMB_yearly_1979_2018") + output_file = f"{mali_mesh_name}_RACMO2.3p2_ANT27" \ f"_smb_climatology_1995-2017.nc" - self.add_output_file(filename=output_file) else: print(f"\n'Warning: process_smb_racmo' is set to 'False'." @@ -65,6 +65,12 @@ def run(self): logger = self.logger config = self.config + section = config["ismip6_ais_atmosphere"] + process_smb_racmo = section.getboolean("process_smb_racmo") + if not process_smb_racmo: + # we don't want to run this step + return + section = config["ismip6_ais"] mali_mesh_name = section.get("mali_mesh_name") mali_mesh_file = section.get("mali_mesh_file") @@ -78,20 +84,21 @@ def run(self): "correct_unit.nc" output_file = f"{mali_mesh_name}_RACMO2.3p2_ANT27" \ f"_smb_climatology_1995-2017.nc" + output_path = f"{output_base_path}/atmosphere_forcing/" \ + f"RACMO_climatology_1995-2017" + output_path_final = os.path.join(output_base_path, output_path) - output_file_final = os.path.join(output_base_path, output_file) - - if os.path.exists(output_file_final): - logger.info("Processed RACMO SMB data already exists in the " - "output path {output_base_path}. " - "Not processing the source data...") + if os.path.exists(os.path.join(output_path_final, output_file)): + logger.info(f"Processed RACMO SMB data already exists in the " + f"output path {output_base_path}. " + f"Not processing the source RACMO data...") return - input_file_list = self._files + input_file = self.inputs[1] # take the time average over the period 1995-2017 args = ["ncra", "-O", "-F", "-d", "time,17,39", - f"{input_file_list[0]}", - f"{racmo_file_temp1}"] + input_file, + racmo_file_temp1] check_call(args, logger=logger) @@ -100,7 +107,7 @@ def run(self): # call the function that reads in, remap and rename the file. logger.info("Calling the remapping function...") - self.remap_source_smb_to_mali(f"{racmo_file_temp1}", + self.remap_source_smb_to_mali(racmo_file_temp1, remapped_file_temp, mali_mesh_name, mali_mesh_file, @@ -110,15 +117,15 @@ def run(self): # to be in unit of kg/m^2/s args = ["ncap2", "-O", "-v", "-s", "sfcMassBal=smb/(60*60*24*365)", - f"{remapped_file_temp}", - f"{racmo_file_temp2}"] + remapped_file_temp, + racmo_file_temp2] check_call(args, logger=logger) # change the unit attribute to kg/m^2/s args = ["ncatted", "-O", "-a", "units,sfcMassBal,m,c,'kg m-2 s-1'", - f"{racmo_file_temp2}"] + racmo_file_temp2] check_call(args, logger=logger) From 867bab78c3b8af109069c571acc9c87a02885fe3 Mon Sep 17 00:00:00 2001 From: hollyhan Date: Mon, 21 Nov 2022 20:46:53 -0700 Subject: [PATCH 58/58] Fix typos --- .../tests/ismip6_forcing/atmosphere/create_mapfile_smb.py | 2 +- compass/landice/tests/ismip6_forcing/atmosphere/process_smb.py | 2 +- .../tests/ismip6_forcing/ocean_basal/process_basal_melt.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/compass/landice/tests/ismip6_forcing/atmosphere/create_mapfile_smb.py b/compass/landice/tests/ismip6_forcing/atmosphere/create_mapfile_smb.py index a8d3a2e6a9..f17c39329a 100644 --- a/compass/landice/tests/ismip6_forcing/atmosphere/create_mapfile_smb.py +++ b/compass/landice/tests/ismip6_forcing/atmosphere/create_mapfile_smb.py @@ -45,7 +45,7 @@ def build_mapping_file(config, cores, logger, ismip6_grid_file, # create the scrip files if mapping file does not exist logger.info(f"Mapping file does not exist. Building one based on the" - f" input/ouptut meshes") + f" input/output meshes") logger.info(f"Creating temporary scrip files for source and " f"destination grids...") diff --git a/compass/landice/tests/ismip6_forcing/atmosphere/process_smb.py b/compass/landice/tests/ismip6_forcing/atmosphere/process_smb.py index c4851f7188..c0cc95f9a8 100644 --- a/compass/landice/tests/ismip6_forcing/atmosphere/process_smb.py +++ b/compass/landice/tests/ismip6_forcing/atmosphere/process_smb.py @@ -99,7 +99,7 @@ def run(self): "but it is required as an input file " "to run this step. Please run `process_smb_racmo`" "step prior to running this step by setting" - "the config option 'process_smb_racmo' to 'True.") + "the config option 'process_smb_racmo' to 'True'.") # temporary remapped climatology and anomaly files clim_ismip6_temp = "clim_ismip6.nc" diff --git a/compass/landice/tests/ismip6_forcing/ocean_basal/process_basal_melt.py b/compass/landice/tests/ismip6_forcing/ocean_basal/process_basal_melt.py index 9497b8f3d6..f3ba4eec0c 100644 --- a/compass/landice/tests/ismip6_forcing/ocean_basal/process_basal_melt.py +++ b/compass/landice/tests/ismip6_forcing/ocean_basal/process_basal_melt.py @@ -107,7 +107,7 @@ def run(self): self.rename_ismip6_basal_melt_to_mali_vars(remapped_file_temp, output_file) - logger.info(f"Remapping and renamping process done successfully. " + logger.info(f"Remapping and renaming process done successfully. " f"Removing the temporary files...") # remove the temporary combined file