diff --git a/.buildkite/longruns/pipeline.yml b/.buildkite/longruns/pipeline.yml index 8d765fe5d0..01276e1fb3 100644 --- a/.buildkite/longruns/pipeline.yml +++ b/.buildkite/longruns/pipeline.yml @@ -16,7 +16,6 @@ env: LD_LIBRARY_PATH: "/central/scratch/esm/slurm-buildkite/climaatmos-ci/depot/cpu/artifacts/e9feeabfa989549ac1dac61e4df5f2849149ad6d/lib/:${LD_LIBRARY_PATH}" - timeout_in_minutes: 1440 steps: @@ -51,14 +50,70 @@ steps: - wait - - group: "Targeted resolution coupled AMIP long runs" + - group: "Coupler integration and conservation tests" steps: - - label: "Slabplanet: default" - key: "slabplanet_default_longrun" - command: "julia --color=yes --project=experiments/AMIP/modular/ experiments/AMIP/modular/coupler_driver_modular.jl --config_file $CONFIG_PATH/slabplanet_default_longrun.yml" - artifact_paths: "experiments/AMIP/modular/output/slabplanet/slabplanet_default_longrun_artifacts/*" + # Integration tests - the expected results were also confirmed locally + - label: "Slabplanet_aqua: nocouple" + key: "slabplanet_aqua_atmos_sf_nocouple" # SF at each Atmos stage, no coupling, prescribed SST from atmos + command: "julia --color=yes --project=experiments/AMIP/modular/ experiments/AMIP/modular/coupler_driver_modular.jl --config_file $CONFIG_PATH/slabplanet_aqua_atmos_sf_nocouple.yml" + artifact_paths: "experiments/AMIP/modular/output/slabplanet_aqua/slabplanet_aqua_atmos_sf_nocouple_artifacts/*" + env: + BUILD_HISTORY_HANDLE: "" + agents: + slurm_ntasks_per_node: 1 + slurm_nodes: 1 + slurm_mem_per_cpu: 16G + + - label: "Slabplanet_aqua: couple" + key: "slabplanet_aqua_atmos_sf_couple" # SF at each Atmos stage, coupling, prescribed SST from coupler - identical results to the above confirm 1) initial conditions in Atmos are unchanged compared to the slab, 2) coupling not introducing variability when constant surface + command: "julia --color=yes --project=experiments/AMIP/modular/ experiments/AMIP/modular/coupler_driver_modular.jl --config_file $CONFIG_PATH/slabplanet_aqua_atmos_sf_couple.yml" + artifact_paths: "experiments/AMIP/modular/output/slabplanet_aqua/slabplanet_aqua_atmos_sf_couple_artifacts/*" + env: + BUILD_HISTORY_HANDLE: "" + agents: + slurm_ntasks_per_node: 1 + slurm_nodes: 1 + slurm_mem_per_cpu: 16G + + - label: "Slabplanet_aqua: coupler fluxes" + key: "slabplanet_aqua_coupler_sf" # SF at each coupler timestep, constant ocean - comparing to the above runs, this tests the sensitivity of less frequent flux calculation + command: "julia --color=yes --project=experiments/AMIP/modular/ experiments/AMIP/modular/coupler_driver_modular.jl --config_file $CONFIG_PATH/slabplanet_aqua_coupler_sf.yml" + artifact_paths: "experiments/AMIP/modular/output/slabplanet_aqua/slabplanet_aqua_coupler_sf_artifacts/*" + env: + BUILD_HISTORY_HANDLE: "" + agents: + slurm_ntasks_per_node: 1 + slurm_nodes: 1 + slurm_mem_per_cpu: 16G + + - label: "Slabplanet_aqua: coupler fluxes, evolving ocean" + key: "slabplanet_aqua_coupler_sf_evolve_ocn" # SF at each coupler timestep, evolving ocean - comparing to the above run, tests the sensitivity of evolving ocean + command: "julia --color=yes --project=experiments/AMIP/modular/ experiments/AMIP/modular/coupler_driver_modular.jl --config_file $CONFIG_PATH/slabplanet_aqua_coupler_sf_evolve_ocn.yml" + artifact_paths: "experiments/AMIP/modular/output/slabplanet_aqua/slabplanet_aqua_coupler_sf_evolve_ocn_artifacts/*" + env: + BUILD_HISTORY_HANDLE: "" + agents: + slurm_ntasks_per_node: 1 + slurm_nodes: 1 + slurm_mem_per_cpu: 16G + + - label: "Slabplanet_terra: coupler fluxes, evolving bucket" + key: "slabplanet_terra" # SF at each coupler timestep, evolving ocean - comparing to the above run, tests the sensitivity of evolving bucket + command: "julia --color=yes --project=experiments/AMIP/modular/ experiments/AMIP/modular/coupler_driver_modular.jl --config_file $CONFIG_PATH/slabplanet_terra.yml" + artifact_paths: "experiments/AMIP/modular/output/slabplanet_terra/slabplanet_terra_artifacts/*" + env: + BUILD_HISTORY_HANDLE: "" + agents: + slurm_ntasks_per_node: 1 + slurm_nodes: 1 + slurm_mem_per_cpu: 16G + + - label: "Slabplanet: coupler fluxes, evolving ocean and land" + key: "slabplanet_coupler_sf_evolve_ocn" + command: "julia --color=yes --project=experiments/AMIP/modular/ experiments/AMIP/modular/coupler_driver_modular.jl --config_file $CONFIG_PATH/slabplanet_coupler_sf_evolve_ocn.yml" + artifact_paths: "experiments/AMIP/modular/output/slabplanet_aqua/slabplanet_coupler_sf_evolve_ocn_artifacts/*" env: BUILD_HISTORY_HANDLE: "" agents: @@ -66,6 +121,116 @@ steps: slurm_nodes: 1 slurm_mem_per_cpu: 16G + - group: "Current target tests: idealized surface" + + steps: + + - label: "ClimaAtmos standalone target" + command: + - srun julia --project=experiments/AMIP/modular/ target/atmos_driver.jl --config_file target/longrun_aquaplanet_rhoe_equilmoist_nz63_0M_55km_rs35km_clearsky_tvinsolation.yml + artifact_paths: "output/longrun_aquaplanet_rhoe_equilmoist_nz63_0M_55km_rs35km_clearsky_tvinsolation/*" + env: + BUILD_HISTORY_HANDLE: "" + CLIMACORE_DISTRIBUTED: "MPI" + agents: + slurm_ntasks_per_node: 16 + slurm_nodes: 4 + slurm_mem_per_cpu: 16G + + - label: "TARGET IDEALIZED: new target aqua - fixed ocean T, nocouple, atmos flux calc" + key: "slabplanet_aqua_new_target_nocouple" + command: "mpiexec julia --color=yes --project=experiments/AMIP/modular/ experiments/AMIP/modular/coupler_driver_modular.jl --config_file $CONFIG_PATH/slabplanet_aqua_new_target_nocouple.yml" + artifact_paths: "experiments/AMIP/modular/output/slabplanet_aqua/slabplanet_aqua_new_target_nocouple_artifacts/*" + env: + CLIMACORE_DISTRIBUTED: "MPI" + BUILD_HISTORY_HANDLE: "" + agents: + slurm_ntasks_per_node: 16 + slurm_nodes: 4 + slurm_mem_per_cpu: 16G + + - label: "TARGET IDEALIZED: new target aqua - fixed ocean T, coupler flux calc" + key: "slabplanet_aqua_new_target" + command: "mpiexec julia --color=yes --project=experiments/AMIP/modular/ experiments/AMIP/modular/coupler_driver_modular.jl --config_file $CONFIG_PATH/slabplanet_aqua_new_target.yml" + artifact_paths: "experiments/AMIP/modular/output/slabplanet_aqua/slabplanet_aqua_new_target_artifacts/*" + env: + CLIMACORE_DISTRIBUTED: "MPI" + BUILD_HISTORY_HANDLE: "" + agents: + slurm_ntasks_per_node: 16 + slurm_nodes: 4 + slurm_mem_per_cpu: 16G + + - label: "TARGET IDEALIZED: new target slab aqua - evolving slab ocean T" + key: "slabplanet_aqua_new_target_evolve_ocn" + command: "mpiexec julia --color=yes --project=experiments/AMIP/modular/ experiments/AMIP/modular/coupler_driver_modular.jl --config_file $CONFIG_PATH/slabplanet_aqua_new_target_evolve_ocn.yml" + artifact_paths: "experiments/AMIP/modular/output/slabplanet/slabplanet_aqua_new_target_evolve_ocn_artifacts/*" + env: + CLIMACORE_DISTRIBUTED: "MPI" + BUILD_HISTORY_HANDLE: "" + agents: + slurm_ntasks_per_node: 16 + slurm_nodes: 4 + slurm_mem_per_cpu: 16G + + - label: "TARGET IDEALIZED: new target slab - fixed ocean T, bucket" + key: "slabplanet_new_target" + command: "mpiexec julia --color=yes --project=experiments/AMIP/modular/ experiments/AMIP/modular/coupler_driver_modular.jl --config_file $CONFIG_PATH/slabplanet_new_target.yml" + artifact_paths: "experiments/AMIP/modular/output/slabplanet/slabplanet_new_target_artifacts/*" + env: + CLIMACORE_DISTRIBUTED: "MPI" + BUILD_HISTORY_HANDLE: "" + agents: + slurm_ntasks_per_node: 16 + slurm_nodes: 4 + slurm_mem_per_cpu: 16G + + - label: "TARGET IDEALIZED: new target slab - evolving slab ocean T, bucket" + key: "slabplanet_new_target_evolve_ocn" + command: "mpiexec julia --color=yes --project=experiments/AMIP/modular/ experiments/AMIP/modular/coupler_driver_modular.jl --config_file $CONFIG_PATH/slabplanet_new_target_evolve_ocn.yml" + artifact_paths: "experiments/AMIP/modular/output/slabplanet/slabplanet_new_target_evolve_ocn_artifacts/*" + env: + CLIMACORE_DISTRIBUTED: "MPI" + BUILD_HISTORY_HANDLE: "" + agents: + slurm_ntasks_per_node: 16 + slurm_nodes: 4 + slurm_mem_per_cpu: 16G + + + - group: "Current target tests: AMIP surface" + + steps: + + - label: "MPI AMIP FINE: new target amip" + key: "amip_new_target" + command: "mpiexec julia --color=yes --project=experiments/AMIP/modular/ experiments/AMIP/modular/coupler_driver_modular.jl --config_file $CONFIG_PATH/amip_new_target.yml" + artifact_paths: "experiments/AMIP/modular/output/amip/amip_new_target_artifacts/*" + env: + CLIMACORE_DISTRIBUTED: "MPI" + BUILD_HISTORY_HANDLE: "" + agents: + slurm_ntasks_per_node: 16 + slurm_nodes: 4 + slurm_mem_per_cpu: 16G + + - label: "MPI AMIP FINE: new target amip: topo" + key: "amip_new_target_topo" + command: "mpiexec julia --color=yes --project=experiments/AMIP/modular/ experiments/AMIP/modular/coupler_driver_modular.jl --config_file $CONFIG_PATH/amip_new_target_topo.yml" + artifact_paths: "experiments/AMIP/modular/output/amip/amip_new_target_topo_artifacts/*" + env: + CLIMACORE_DISTRIBUTED: "MPI" + BUILD_HISTORY_HANDLE: "" + agents: + slurm_ntasks_per_node: 16 + slurm_nodes: 4 + slurm_mem_per_cpu: 16G + + + - group: "Other AMIP targets" + + steps: + # DYAMOND AMIP: 1 day (convection resolving) - label: "MPI AMIP SUPERFINE: dyamond_target" @@ -80,11 +245,10 @@ steps: slurm_nodes: 4 slurm_mem_per_cpu: 16G - # mid-resolution AMIP: longrun (140 days) - - label: "MPI AMIP FINE: target longrun" - key: "amip_longrun_target" - command: "mpiexec julia --color=yes --project=experiments/AMIP/modular/ experiments/AMIP/modular/coupler_driver_modular.jl --config_file $CONFIG_PATH/amip_longrun_target.yml" - artifact_paths: "experiments/AMIP/modular/output/amip/amip_longrun_target_artifacts/*" + - label: "MPI AMIP FINE: august" + key: "august" + command: "mpiexec julia --color=yes --project=experiments/AMIP/modular/ experiments/AMIP/modular/coupler_driver_modular.jl --config_file $CONFIG_PATH/august.yml" + artifact_paths: "experiments/AMIP/modular/output/amip/august_artifacts/*" env: CLIMACORE_DISTRIBUTED: "MPI" BUILD_HISTORY_HANDLE: "" @@ -190,14 +354,14 @@ steps: - label: ":envelope: Slack report: Slabplanet" command: - - slack-upload -c "#coupler-report" -f experiments/AMIP/modular/output/slabplanet/slabplanet_default_longrun_artifacts/total_energy_log_bucket.png -m png -n slab_coarse_log -x "Slabplanet energy conservation (log error)" - - slack-upload -c "#coupler-report" -f experiments/AMIP/modular/output/slabplanet/slabplanet_default_longrun_artifacts/total_energy_bucket.png -m png -n slab_coarse -x "Slabplanet energy conservation" - - slack-upload -c "#coupler-report" -f experiments/AMIP/modular/output/slabplanet/slabplanet_default_longrun_artifacts/total_water_log_bucket.png -m png -n slab_coarse_w_log -x "Slabplanet water conservation (log error)" - - slack-upload -c "#coupler-report" -f experiments/AMIP/modular/output/slabplanet/slabplanet_default_longrun_artifacts/total_water_bucket.png -m png -n slab_coarse_w -x "Slabplanet water conservation" + - slack-upload -c "#coupler-report" -f experiments/AMIP/modular/output/slabplanet/slabplanet_coupler_sf_evolve_ocn_artifacts/total_energy_log_bucket.png -m png -n slab_coarse_log -x "Slabplanet energy conservation (log error)" + - slack-upload -c "#coupler-report" -f experiments/AMIP/modular/output/slabplanet/slabplanet_coupler_sf_evolve_ocn_artifacts/total_energy_bucket.png -m png -n slab_coarse -x "Slabplanet energy conservation" + - slack-upload -c "#coupler-report" -f experiments/AMIP/modular/output/slabplanet/slabplanet_coupler_sf_evolve_ocn_artifacts/total_water_log_bucket.png -m png -n slab_coarse_w_log -x "Slabplanet water conservation (log error)" + - slack-upload -c "#coupler-report" -f experiments/AMIP/modular/output/slabplanet/slabplanet_coupler_sf_evolve_ocn_artifacts/total_water_bucket.png -m png -n slab_coarse_w -x "Slabplanet water conservation" - label: ":envelope: Slack report: target AMIP" command: - - slack-upload -c "#coupler-report" -f experiments/AMIP/modular/output/amip/amip_longrun_target_artifacts/amip_paperplots.png -m png -n amip_fine -x "AMIP Target Longrun" + - slack-upload -c "#coupler-report" -f experiments/AMIP/modular/output/amip/amip_new_target_artifacts/amip_paperplots.png -m png -n amip_fine -x "AMIP Target Longrun" - label: ":envelope: Slack report: Flame Diff" command: diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index f650e7d07d..a84edd7795 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -1,6 +1,5 @@ agents: queue: central - slurm_mem: 8G slurm_time: 24:00:00 modules: julia/1.9.3 cuda/12.2 ucx/1.14.1_cuda-12.2 openmpi/4.1.5_cuda-12.2 hdf5/1.12.2-ompi415 nsight-systems/2023.2.1 @@ -24,15 +23,14 @@ steps: - label: "init environment :computer:" key: "init_cpu_env" command: - # - "echo $$JULIA_DEPOT_PATH" - echo "--- Configure MPI" - julia -e 'using Pkg; Pkg.add("MPIPreferences"); using MPIPreferences; use_system_binary()' - - echo "--- Instantiate package env" - - "julia --project -e 'using Pkg; Pkg.instantiate(;verbose=true)'" - - "julia --project -e 'using Pkg; Pkg.precompile()'" - - "julia --project -e 'using Pkg; Pkg.status()'" + # - echo "--- Instantiate package env" + # - "julia --project -e 'using Pkg; Pkg.instantiate(;verbose=true)'" + # - "julia --project -e 'using Pkg; Pkg.precompile()'" + # - "julia --project -e 'using Pkg; Pkg.status()'" - echo "--- Instantiate sea breeze env" - "julia --project=experiments/ClimaCore/sea_breeze -e 'using Pkg; Pkg.instantiate(;verbose=true)'" diff --git a/Project.toml b/Project.toml index 761dcec110..64c62ec568 100644 --- a/Project.toml +++ b/Project.toml @@ -18,6 +18,7 @@ NCDatasets = "85f8d34a-cbdd-5861-8df4-14fed0d494ab" OrdinaryDiffEq = "1dea7af3-3e70-54e6-95c3-0bf5283fa5ed" Plots = "91a5bcdd-55d7-5caf-9e0b-520d859cae80" SciMLBase = "0bca4576-84f4-4d90-8ffe-ffa030f20462" +SimpleNonlinearSolve = "727e6d20-b764-4bd8-a329-72de5adea6c7" StaticArrays = "90137ffa-7385-5640-81b9-e52037218182" Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" SurfaceFluxes = "49b00bb7-8bd4-4f2b-b78c-51cd0450215f" @@ -28,7 +29,7 @@ UnPack = "3a884ed6-31ef-47d7-9d2a-63182c4928ed" [compat] CLIMAParameters = "0.4, 0.5, 0.6, 0.7" -ClimaAtmos = "0.16" +ClimaAtmos = "=0.16.2" ClimaComms = "0.5" ClimaCore = "0.10" ClimaCoreTempestRemap = "0.3" @@ -40,6 +41,7 @@ NCDatasets = "0.11, 0.12" OrdinaryDiffEq = "5, 6" Plots = "1.39.0" SciMLBase = "1" +SimpleNonlinearSolve = "=0.1.23" StaticArrays = "1" SurfaceFluxes = "0.7" TempestRemap_jll = "2" diff --git a/config/longrun_configs/amip_longrun_target.yml b/config/longrun_configs/amip_longrun_target.yml deleted file mode 100644 index 1b2ff5114b..0000000000 --- a/config/longrun_configs/amip_longrun_target.yml +++ /dev/null @@ -1,22 +0,0 @@ -run_name: "amip_longrun_target" -anim: true -dt_cpl: 150 -energy_check: false -mode_name: "amip" -mono_surface: false -vert_diff: "true" -moist: "equil" -rad: "clearsky" -precip_model: "0M" -z_elem: 35 -dz_bottom: 50 -h_elem: 12 -kappa_4: 3e16 -rayleigh_sponge: true -alpha_rayleigh_uh: 0 -dt: "100secs" -t_end: "100days" # TODO this has been decreased from 140 days to avoid instability #460 -job_id: "amip_longrun_target" -dt_save_to_sol: "5days" -dt_save_to_disk: "1days" -apply_limiter: false diff --git a/config/longrun_configs/amip_new_target.yml b/config/longrun_configs/amip_new_target.yml new file mode 100644 index 0000000000..d4e30b4959 --- /dev/null +++ b/config/longrun_configs/amip_new_target.yml @@ -0,0 +1,18 @@ +run_name: "amip_new_target" +anim: true +dt_cpl: 150 +energy_check: false +mode_name: "amip" +mono_surface: false +dt: "150secs" +t_end: "300days" # TODO this has been decreased from 140 days to avoid instability #460 +job_id: "amip_new_target" +dt_save_to_sol: "5days" +dt_save_to_disk: "5days" +apply_limiter: false +hourly_checkpoint: true +turb_flux_partition: "CombinedStateFluxes" +atmos_config_file: "config/longrun_configs/longrun_aquaplanet_rhoe_equilmoist_nz63_0M_55km_rs35km_clearsky_tvinsolation.yml" +surface_setup: "PrescribedSurface" +land_albedo_type: "map_temporal" +start_date: "19790301" \ No newline at end of file diff --git a/config/longrun_configs/amip_new_target_topo.yml b/config/longrun_configs/amip_new_target_topo.yml new file mode 100644 index 0000000000..505bcc0bd0 --- /dev/null +++ b/config/longrun_configs/amip_new_target_topo.yml @@ -0,0 +1,21 @@ +run_name: "amip_new_target_topo" +anim: true +dt_cpl: 75 +energy_check: false +mode_name: "amip" +mono_surface: false +dt: "150secs" +t_end: "100days" # TODO this has been decreased from 140 days to avoid instability #460 +job_id: "amip_new_target_topo" +dt_save_to_sol: "5days" +dt_save_to_disk: "5days" +apply_limiter: false +hourly_checkpoint: true +turb_flux_partition: "CombinedStateFluxes" +atmos_config_file: "config/longrun_configs/longrun_aquaplanet_rhoe_equilmoist_nz63_0M_55km_rs35km_clearsky_tvinsolation.yml" +surface_setup: "PrescribedSurface" +land_albedo_type: "map_temporal" +start_date: "19790301" +topography: "Earth" +topo_smoothing: true +use_reference_state: false \ No newline at end of file diff --git a/config/longrun_configs/august.yml b/config/longrun_configs/august.yml new file mode 100644 index 0000000000..845f500f58 --- /dev/null +++ b/config/longrun_configs/august.yml @@ -0,0 +1,33 @@ +run_name: "august" +coupled: true +start_date: "19790301" +monthly_checkpoint: true +surface_setup: "PrescribedSurface" +dt_cpl: 100 +energy_check: false +mode_name: "amip" +mono_surface: false +vert_diff: "true" +moist: "equil" +rad: "clearsky" +precip_model: "0M" +z_elem: 35 +dz_bottom: 50 +h_elem: 12 +kappa_4: 4e16 +rayleigh_sponge: true +alpha_rayleigh_uh: 0 +dt: "100secs" +t_end: "400days" +job_id: "august" +dt_save_to_sol: "10days" +dt_save_to_disk: "1days" +apply_limiter: false +FLOAT_TYPE: "Float64" +post_process: false +anim: true +hourly_checkpoint: true +apply_limiter: false +surface_setup: "PrescribedSurface" +land_albedo_type: "function" +coupler_toml_file: "toml/august.toml" \ No newline at end of file diff --git a/config/longrun_configs/dyamond_target.yml b/config/longrun_configs/dyamond_target.yml index fbae6b5f37..291b813b64 100644 --- a/config/longrun_configs/dyamond_target.yml +++ b/config/longrun_configs/dyamond_target.yml @@ -10,5 +10,6 @@ dt_save_to_sol: "0.5days" dt_save_to_disk: "0.5days" turb_flux_partition: "CombinedStateFluxes" atmos_config_file: "config/longrun_configs/longrun_aquaplanet_dyamond.yml" -atmos_toml_file: "toml/longrun_aquaplanet_dyamond.toml" -hourly_checkpoint: false +monthly_checkpoint: false +land_albedo_type: "map_temporal" +start_date: "19790301" \ No newline at end of file diff --git a/config/longrun_configs/slabplanet_aqua_atmos_sf_couple.yml b/config/longrun_configs/slabplanet_aqua_atmos_sf_couple.yml new file mode 100644 index 0000000000..89e592ccd2 --- /dev/null +++ b/config/longrun_configs/slabplanet_aqua_atmos_sf_couple.yml @@ -0,0 +1,21 @@ +run_name: "slabplanet_aqua_atmos_sf_couple" +job_id: "slabplanet_aqua_atmos_sf_couple" +moist: "equil" +vert_diff: "true" +rad: "gray" +precip_model: "0M" +energy_check: true +mode_name: "slabplanet_aqua" +t_end: "20days" +dt_save_to_sol: "10days" +dt_cpl: 200 +dt: "200secs" +mono_surface: true +h_elem: 6 +anim: true +apply_limiter: false +turb_flux_partition: "CombinedStateFluxes" +surface_setup: "DefaultMoninObukhov" +land_albedo_type: "function" +evolving_ocean: false +start_date: "19790301" \ No newline at end of file diff --git a/config/longrun_configs/slabplanet_aqua_atmos_sf_nocouple.yml b/config/longrun_configs/slabplanet_aqua_atmos_sf_nocouple.yml new file mode 100644 index 0000000000..e7420b0c2a --- /dev/null +++ b/config/longrun_configs/slabplanet_aqua_atmos_sf_nocouple.yml @@ -0,0 +1,21 @@ +run_name: "slabplanet_aqua_atmos_sf_nocouple" +job_id: "slabplanet_aqua_atmos_sf_nocouple" +moist: "equil" +vert_diff: "true" +rad: "gray" +precip_model: "0M" +energy_check: true +mode_name: "slabplanet_aqua" +t_end: "20days" +dt_save_to_sol: "10days" +dt_cpl: 1728000 +dt: "200secs" +mono_surface: true +h_elem: 6 +anim: true +apply_limiter: false +turb_flux_partition: "CombinedStateFluxes" +surface_setup: "DefaultMoninObukhov" +land_albedo_type: "function" +evolving_ocean: false +start_date: "19790301" \ No newline at end of file diff --git a/config/longrun_configs/slabplanet_aqua_coupler_sf.yml b/config/longrun_configs/slabplanet_aqua_coupler_sf.yml new file mode 100644 index 0000000000..d02caca226 --- /dev/null +++ b/config/longrun_configs/slabplanet_aqua_coupler_sf.yml @@ -0,0 +1,21 @@ +run_name: "slabplanet_aqua_coupler_sf" +job_id: "slabplanet_aqua_coupler_sf" +moist: "equil" +vert_diff: "true" +rad: "gray" +precip_model: "0M" +energy_check: true +mode_name: "slabplanet_aqua" +t_end: "20days" +dt_save_to_sol: "10days" +dt_cpl: 200 +dt: "200secs" +mono_surface: true +h_elem: 6 +anim: true +apply_limiter: false +turb_flux_partition: "CombinedStateFluxes" +surface_setup: "PrescribedSurface" +land_albedo_type: "function" +evolving_ocean: false +start_date: "19790301" \ No newline at end of file diff --git a/config/longrun_configs/slabplanet_aqua_coupler_sf_evolve_ocn.yml b/config/longrun_configs/slabplanet_aqua_coupler_sf_evolve_ocn.yml new file mode 100644 index 0000000000..1a3689004d --- /dev/null +++ b/config/longrun_configs/slabplanet_aqua_coupler_sf_evolve_ocn.yml @@ -0,0 +1,21 @@ +run_name: "slabplanet_aqua_coupler_sf_evolve_ocn" +job_id: "slabplanet_aqua_coupler_sf_evolve_ocn" +moist: "equil" +vert_diff: "true" +rad: "gray" +precip_model: "0M" +energy_check: true +mode_name: "slabplanet_aqua" +t_end: "20days" +dt_save_to_sol: "10days" +dt_cpl: 200 +dt: "200secs" +mono_surface: true +h_elem: 6 +anim: true +apply_limiter: false +turb_flux_partition: "CombinedStateFluxes" +surface_setup: "PrescribedSurface" +land_albedo_type: "function" +evolving_ocean: true +start_date: "19790301" \ No newline at end of file diff --git a/config/longrun_configs/slabplanet_aqua_new_target.yml b/config/longrun_configs/slabplanet_aqua_new_target.yml new file mode 100644 index 0000000000..5715e2f6db --- /dev/null +++ b/config/longrun_configs/slabplanet_aqua_new_target.yml @@ -0,0 +1,19 @@ +run_name: "slabplanet_aqua_new_target" +anim: true +dt_cpl: 150 +energy_check: false +mode_name: "slabplanet_aqua" +mono_surface: false +dt: "150secs" +t_end: "200days" # TODO this has been decreased from 140 days to avoid instability #460 +job_id: "slabplanet_aqua_new_target" +dt_save_to_sol: "5days" +dt_save_to_disk: "5days" +apply_limiter: false +hourly_checkpoint: true +turb_flux_partition: "CombinedStateFluxes" +atmos_config_file: "config/longrun_configs/longrun_aquaplanet_rhoe_equilmoist_nz63_0M_55km_rs35km_clearsky_tvinsolation.yml" +surface_setup: "PrescribedSurface" +land_albedo_type: "function" +evolving_ocean: false +start_date: "19790301" diff --git a/config/longrun_configs/slabplanet_aqua_new_target_evolve_ocn.yml b/config/longrun_configs/slabplanet_aqua_new_target_evolve_ocn.yml new file mode 100644 index 0000000000..5ea01c8498 --- /dev/null +++ b/config/longrun_configs/slabplanet_aqua_new_target_evolve_ocn.yml @@ -0,0 +1,19 @@ +run_name: "slabplanet_aqua_new_target_evolve_ocn" +anim: true +dt_cpl: 150 +energy_check: false +mode_name: "slabplanet" +mono_surface: false +dt: "150secs" +t_end: "300days" # TODO this has been decreased from 140 days to avoid instability #460 +job_id: "slabplanet_new_target_evolve_ocn" +dt_save_to_sol: "5days" +dt_save_to_disk: "5days" +apply_limiter: false +hourly_checkpoint: true +turb_flux_partition: "CombinedStateFluxes" +atmos_config_file: "config/longrun_configs/longrun_aquaplanet_rhoe_equilmoist_nz63_0M_55km_rs35km_clearsky_tvinsolation.yml" +surface_setup: "PrescribedSurface" +land_albedo_type: "function" +evolving_ocean: true +start_date: "19790301" diff --git a/config/longrun_configs/slabplanet_aqua_new_target_nocouple.yml b/config/longrun_configs/slabplanet_aqua_new_target_nocouple.yml new file mode 100644 index 0000000000..9077686103 --- /dev/null +++ b/config/longrun_configs/slabplanet_aqua_new_target_nocouple.yml @@ -0,0 +1,19 @@ +run_name: "slabplanet_aqua_new_target_nocouple" +anim: true +dt_cpl: 8639800 +energy_check: false +mode_name: "slabplanet_aqua" +mono_surface: false +dt: "150secs" +t_end: "200days" # TODO this has been decreased from 140 days to avoid instability #460 +job_id: "slabplanet_aqua_new_target_nocouple" +dt_save_to_sol: "5days" +dt_save_to_disk: "5days" +apply_limiter: false +hourly_checkpoint: true +turb_flux_partition: "CombinedStateFluxes" +atmos_config_file: "config/longrun_configs/longrun_aquaplanet_rhoe_equilmoist_nz63_0M_55km_rs35km_clearsky_tvinsolation.yml" +surface_setup: "DefaultMoninObukhov" +land_albedo_type: "function" +evolving_ocean: false +start_date: "19790301" diff --git a/config/longrun_configs/slabplanet_default_longrun.yml b/config/longrun_configs/slabplanet_coarse_longrun.yml similarity index 63% rename from config/longrun_configs/slabplanet_default_longrun.yml rename to config/longrun_configs/slabplanet_coarse_longrun.yml index e635c8b447..df5bc107c4 100644 --- a/config/longrun_configs/slabplanet_default_longrun.yml +++ b/config/longrun_configs/slabplanet_coarse_longrun.yml @@ -1,4 +1,4 @@ -run_name: "slabplanet_default_longrun" +run_name: "slabplanet_coarse_longrun" moist: "equil" vert_diff: "true" rad: "gray" @@ -13,4 +13,6 @@ h_elem: 6 precip_model: "0M" anim: true apply_limiter: false -job_id: "slabplanet_default_longrun" +job_id: "slabplanet_coarse_longrun" +surface_setup: "PrescribedSurface" +land_albedo_type: "function" diff --git a/config/longrun_configs/slabplanet_coupler_sf_evolve_ocn.yml b/config/longrun_configs/slabplanet_coupler_sf_evolve_ocn.yml new file mode 100644 index 0000000000..2ff58dddf7 --- /dev/null +++ b/config/longrun_configs/slabplanet_coupler_sf_evolve_ocn.yml @@ -0,0 +1,21 @@ +run_name: "slabplanet_coupler_sf_evolve_ocn" +job_id: "slabplanet_coupler_sf_evolve_ocn" +moist: "equil" +vert_diff: "true" +rad: "gray" +precip_model: "0M" +energy_check: true +mode_name: "slabplanet" +t_end: "20days" +dt_save_to_sol: "10days" +dt_cpl: 200 +dt: "200secs" +mono_surface: true +h_elem: 6 +anim: true +apply_limiter: false +turb_flux_partition: "CombinedStateFluxes" +surface_setup: "PrescribedSurface" +land_albedo_type: "function" +evolving_ocean: true +start_date: "19790301" \ No newline at end of file diff --git a/config/longrun_configs/slabplanet_new_target.yml b/config/longrun_configs/slabplanet_new_target.yml new file mode 100644 index 0000000000..76b92537b4 --- /dev/null +++ b/config/longrun_configs/slabplanet_new_target.yml @@ -0,0 +1,19 @@ +run_name: "slabplanet_new_target" +anim: true +dt_cpl: 150 +energy_check: false +mode_name: "slabplanet" +mono_surface: false +dt: "150secs" +t_end: "200days" # TODO this has been decreased from 140 days to avoid instability #460 +job_id: "slabplanet_new_target" +dt_save_to_sol: "5days" +dt_save_to_disk: "5days" +apply_limiter: false +hourly_checkpoint: true +turb_flux_partition: "CombinedStateFluxes" +atmos_config_file: "config/longrun_configs/longrun_aquaplanet_rhoe_equilmoist_nz63_0M_55km_rs35km_clearsky_tvinsolation.yml" +surface_setup: "PrescribedSurface" +land_albedo_type: "function" +evolving_ocean: false +start_date: "19790301" diff --git a/config/longrun_configs/slabplanet_new_target_evolve_ocn.yml b/config/longrun_configs/slabplanet_new_target_evolve_ocn.yml new file mode 100644 index 0000000000..c17a14a949 --- /dev/null +++ b/config/longrun_configs/slabplanet_new_target_evolve_ocn.yml @@ -0,0 +1,19 @@ +run_name: "slabplanet_new_target_evolve_ocn" +anim: true +dt_cpl: 150 +energy_check: false +mode_name: "slabplanet" +mono_surface: false +dt: "150secs" +t_end: "200days" # TODO this has been decreased from 140 days to avoid instability #460 +job_id: "slabplanet_new_target_evolve_ocn" +dt_save_to_sol: "5days" +dt_save_to_disk: "5days" +apply_limiter: false +hourly_checkpoint: true +turb_flux_partition: "CombinedStateFluxes" +atmos_config_file: "config/longrun_configs/longrun_aquaplanet_rhoe_equilmoist_nz63_0M_55km_rs35km_clearsky_tvinsolation.yml" +surface_setup: "PrescribedSurface" +land_albedo_type: "function" +evolving_ocean: true +start_date: "19790301" diff --git a/config/longrun_configs/slabplanet_terra.yml b/config/longrun_configs/slabplanet_terra.yml new file mode 100644 index 0000000000..affb7b48e2 --- /dev/null +++ b/config/longrun_configs/slabplanet_terra.yml @@ -0,0 +1,20 @@ +run_name: "slabplanet_terra" +job_id: "slabplanet_terra" +moist: "equil" +vert_diff: "true" +rad: "gray" +precip_model: "0M" +energy_check: true +mode_name: "slabplanet_terra" +t_end: "20days" +dt_save_to_sol: "10days" +dt_cpl: 200 +dt: "200secs" +mono_surface: true +h_elem: 4 +anim: true +apply_limiter: false +turb_flux_partition: "CombinedStateFluxes" +surface_setup: "PrescribedSurface" +land_albedo_type: "function" +start_date: "19790301" \ No newline at end of file diff --git a/config/model_configs/interactive_debug.yml b/config/model_configs/interactive_debug.yml index a5ec7c15dc..e817a5f201 100644 --- a/config/model_configs/interactive_debug.yml +++ b/config/model_configs/interactive_debug.yml @@ -1,17 +1,21 @@ +run_name: "interactive_debug_run" +job_id: "interactive_debug_run" moist: "equil" vert_diff: "true" -rad: "allskywithclear" +rad: "gray" +precip_model: "0M" energy_check: true mode_name: "slabplanet" -t_end: "10days" -dt_save_to_sol: "0.5days" -dt_cpl: 400 -dt: "400secs" +t_end: "20days" +dt_save_to_sol: "10days" +dt_cpl: 200 +dt: "200secs" mono_surface: true -turb_flux_partition: "CombinedStateFluxes" h_elem: 4 -dt_save_restart: "5days" -precip_model: "0M" -run_name: "interactive_debug_run" -job_id: "interactive_debug_run" -hourly_checkpoint: true \ No newline at end of file +anim: true +apply_limiter: false +turb_flux_partition: "CombinedStateFluxes" +surface_setup: "PrescribedSurface" +land_albedo_type: "function" +evolving_ocean: true +start_date: "19790301" \ No newline at end of file diff --git a/experiments/AMIP/modular/cli_options.jl b/experiments/AMIP/modular/cli_options.jl index 335b5f1c2d..13a31b8a35 100644 --- a/experiments/AMIP/modular/cli_options.jl +++ b/experiments/AMIP/modular/cli_options.jl @@ -52,6 +52,12 @@ function argparse_settings() help = "Floating point precision [`Float64` (default), `Float32`]" arg_type = String default = "Float64" + "--coupler_toml_file" + help = "A toml file used to overwrite the model parameters. If nothing is specified, the default parameters are used." + "--evolving_ocean" + help = "Boolean flag indicating whether to use a dynamic slab ocean model or constant surface temperatures" + arg_type = Bool + default = true # ClimaAtmos specific "--surface_setup" help = "Triggers ClimaAtmos into the coupled mode [`PrescribedSurface` (default)]" # retained here for standalone Atmos benchmarks @@ -59,8 +65,6 @@ function argparse_settings() default = "PrescribedSurface" "--atmos_config_file" help = "A yaml file used to set the atmospheric model configuration. If nothing is specified, the default configuration is used." - "--atmos_toml_file" - help = "A toml file used to overwrite the atmospheric model parameters. If nothing is specified, the default parameters are used." # ClimaLSM specific "--land_albedo_type" help = "Access land surface albedo information from data file. [`function`, `map_static`, `map_temporal`]" @@ -70,6 +74,10 @@ function argparse_settings() help = "Type of land domain. [`sphere` (default), `single_column`]" arg_type = String default = "sphere" + "--land_temperature_anomaly" + help = "Type of temperature anomaly for bucket model. [`zonally_asymmetric`, `amip`, `aquaplanet` (default)]" + arg_type = String + default = "aquaplanet" end return s end diff --git a/experiments/AMIP/modular/components/atmosphere/climaatmos_init.jl b/experiments/AMIP/modular/components/atmosphere/climaatmos_init.jl index fa507fe712..2bbe1b2045 100644 --- a/experiments/AMIP/modular/components/atmosphere/climaatmos_init.jl +++ b/experiments/AMIP/modular/components/atmosphere/climaatmos_init.jl @@ -1,6 +1,8 @@ # atmos_init: for ClimaAtmos pre-AMIP interface -using ClimaAtmos: RRTMGPI +import ClimaAtmos as CA import ClimaAtmos: CT1, CT2, CT12, CT3, C3, C12, unit_basis_vector_data, ⊗ +import SurfaceFluxes as SF + import ClimaCoupler.FluxCalculator: atmos_turbulent_fluxes!, calculate_surface_air_density, @@ -41,10 +43,19 @@ function get_atmos_config(coupler_dict) Dict("config_file" => atmos_config_file), ) end - atmos_toml_file = coupler_dict["atmos_toml_file"] - if !isnothing(atmos_toml_file) - @info "Overwriting Atmos parameters from $atmos_toml_file" - atmos_config = merge(atmos_config, Dict("toml" => [joinpath(pkgdir(CA), atmos_toml_file)])) + + # use coupler toml if atmos is not defined + atmos_toml_file = atmos_config["toml"] + coupler_toml_file = coupler_dict["coupler_toml_file"] + default_toml_file = "toml/default_coarse.toml" + + toml_file = !isempty(atmos_toml_file) ? joinpath(pkgdir(CA), atmos_toml_file[1]) : nothing + toml_file = !isnothing(coupler_toml_file) ? joinpath(pkgdir(ClimaCoupler), coupler_toml_file) : toml_file + toml_file = isnothing(toml_file) ? joinpath(pkgdir(ClimaCoupler), default_toml_file) : toml_file + + if !isnothing(toml_file) + @info "Overwriting Atmos parameters from $toml_file" + atmos_config = merge(atmos_config, Dict("toml" => [toml_file])) end return atmos_config end @@ -123,14 +134,14 @@ function update_field!(atmos_sim::ClimaAtmosSimulation, ::Val{:co2_gm}, field) end # extensions required by the Interfacer function update_field!(sim::ClimaAtmosSimulation, ::Val{:surface_temperature}, csf) - sim.integrator.p.radiation_model.surface_temperature .= RRTMGPI.field2array(csf.T_S) + sim.integrator.p.radiation_model.surface_temperature .= CA.RRTMGPI.field2array(csf.T_S) end function update_field!(sim::ClimaAtmosSimulation, ::Val{:albedo}, field) sim.integrator.p.radiation_model.diffuse_sw_surface_albedo .= - reshape(RRTMGPI.field2array(field), 1, length(parent(field))) + reshape(CA.RRTMGPI.field2array(field), 1, length(parent(field))) sim.integrator.p.radiation_model.direct_sw_surface_albedo .= - reshape(RRTMGPI.field2array(field), 1, length(parent(field))) + reshape(CA.RRTMGPI.field2array(field), 1, length(parent(field))) end # get_surface_params required by FluxCalculator (partitioned fluxes) @@ -241,11 +252,20 @@ requires that we define a new temporary parameter Tuple, `new_p`, and save the n in it. We do not want `new_p` to live in the atmospheric model permanently, because that would also trigger flux calculation during Atmos `step!`. We only want to trigger this once per coupling timestep from ClimaCoupler. + +For debigging atmos, we can set the following atmos defaults: + csf.z0m_S .= 1.0e-5 + csf.z0b_S .= 1.0e-5 + csf.beta .= 1 + csf = merge(csf, (;q_sfc = nothing)) """ function atmos_turbulent_fluxes!(atmos_sim::ClimaAtmosSimulation, csf) - new_p = get_new_cache(atmos_sim, csf) - CA.SurfaceConditions.update_surface_conditions!(atmos_sim.integrator.u, new_p, atmos_sim.integrator.t) - atmos_sim.integrator.p.sfc_conditions .= new_p.sfc_conditions + + if isnothing(atmos_sim.integrator.p.sfc_setup) # trigger flux calculation if not done in Atmos internally + new_p = get_new_cache(atmos_sim, csf) + CA.SurfaceConditions.update_surface_conditions!(atmos_sim.integrator.u, new_p, atmos_sim.integrator.t) + atmos_sim.integrator.p.sfc_conditions .= new_p.sfc_conditions + end end """ @@ -292,19 +312,19 @@ function get_field(atmos_sim::ClimaAtmosSimulation, ::Val{:F_radiative_TOA}) n_faces = length(z[:, 1, 1, 1, 1]) LWd_TOA = Fields.level( - RRTMGPI.array2field(FT.(atmos_sim.integrator.p.radiation_model.face_lw_flux_dn), face_space), + CA.RRTMGPI.array2field(FT.(atmos_sim.integrator.p.radiation_model.face_lw_flux_dn), face_space), n_faces - half, ) LWu_TOA = Fields.level( - RRTMGPI.array2field(FT.(atmos_sim.integrator.p.radiation_model.face_lw_flux_up), face_space), + CA.RRTMGPI.array2field(FT.(atmos_sim.integrator.p.radiation_model.face_lw_flux_up), face_space), n_faces - half, ) SWd_TOA = Fields.level( - RRTMGPI.array2field(FT.(atmos_sim.integrator.p.radiation_model.face_sw_flux_dn), face_space), + CA.RRTMGPI.array2field(FT.(atmos_sim.integrator.p.radiation_model.face_sw_flux_dn), face_space), n_faces - half, ) SWu_TOA = Fields.level( - RRTMGPI.array2field(FT.(atmos_sim.integrator.p.radiation_model.face_sw_flux_up), face_space), + CA.RRTMGPI.array2field(FT.(atmos_sim.integrator.p.radiation_model.face_sw_flux_up), face_space), n_faces - half, ) diff --git a/experiments/AMIP/modular/components/land/bucket_init.jl b/experiments/AMIP/modular/components/land/bucket_init.jl index 6548277370..2b103d984c 100644 --- a/experiments/AMIP/modular/components/land/bucket_init.jl +++ b/experiments/AMIP/modular/components/land/bucket_init.jl @@ -166,6 +166,7 @@ function bucket_init( tspan::Tuple{FT, FT}, config::String, albedo_type::String, + land_temperature_anomaly::String, comms_ctx::AbstractCommsContext, regrid_dirpath::String; space, @@ -188,14 +189,22 @@ function bucket_init( α_snow = FT(0.8) # snow albedo if albedo_type == "map_static" # Read in albedo from static data file (default type) # By default, this uses a file containing bareground albedo without a time component. Snow albedo is specified separately. + if ClimaComms.iamroot(comms_ctx) + albedo = BulkAlbedoStatic{FT}(regrid_dirpath, α_snow = α_snow, comms = comms_ctx) + end + ClimaComms.barrier(comms_ctx) albedo = BulkAlbedoStatic{FT}(regrid_dirpath, α_snow = α_snow, comms = comms_ctx) elseif albedo_type == "map_temporal" # Read in albedo from data file containing data over time # By default, this uses a file containing linearly-interpolated monthly data of total albedo, generated by CESM2's land model (CLM). + if ClimaComms.iamroot(comms_ctx) + albedo = BulkAlbedoTemporal{FT}(regrid_dirpath, date_ref, t_start, space) + end + ClimaComms.barrier(comms_ctx) albedo = BulkAlbedoTemporal{FT}(regrid_dirpath, date_ref, t_start, space) elseif albedo_type == "function" # Use prescribed function of lat/lon for surface albedo function α_sfc(coordinate_point) (; lat, long) = coordinate_point - return typeof(lat)(0.4) + return typeof(lat)(0.38) end albedo = BulkAlbedoFunction{FT}(α_snow, α_sfc) else @@ -203,12 +212,12 @@ function bucket_init( end σS_c = FT(0.2) - W_f = FT(0.5) + W_f = FT(10) d_soil = FT(3.5) # soil depth z_0m = FT(1e-2) z_0b = FT(1e-3) κ_soil = FT(0.7) - ρc_soil = FT(2e6) + ρc_soil = FT(2e8) t_crit = dt # This is the timescale on which snow exponentially damps to zero, in the case where all # the snow would melt in time t_crit. It prevents us from having to specially time step in cases where # all the snow melts in a single timestep. @@ -222,29 +231,28 @@ function bucket_init( # Initial conditions with no moisture Y, p, coords = initialize(model) - anomaly = false - anomaly_tropics = false - hs_sfc = false Y.bucket.T = map(coords.subsurface) do coord - T_sfc_0 = FT(285.0) + T_sfc_0 = FT(271.0) radlat = coord.lat / FT(180) * pi ΔT = FT(0) - if anomaly == true + if land_temperature_anomaly == "zonally_asymmetric" anom_ampl = FT(0)# this is zero, no anomaly lat_0 = FT(60) / FT(180) * pi lon_0 = FT(-90) / FT(180) * pi radlon = coord.long / FT(180) * pi stdev = FT(5) / FT(180) * pi ΔT = anom_ampl * exp(-((radlat - lat_0)^2 / 2stdev^2 + (radlon - lon_0)^2 / 2stdev^2)) - elseif hs_sfc == true - ΔT = -FT(60) * sin(radlat)^2 - elseif anomaly_tropics == true + elseif land_temperature_anomaly == "aquaplanet" + ΔT = FT(29) * exp(-coord.lat^2 / (2 * 26^2)) + elseif land_temperature_anomaly == "amip" ΔT = FT(40 * cos(radlat)^4) + else + ΔT = FT(0) end T_sfc_0 + ΔT end - Y.bucket.W .= 0.5#0.14 + Y.bucket.W .= 10#0.14 Y.bucket.Ws .= 0.0 Y.bucket.σS .= 0.0 P_liq = zeros(axes(Y.bucket.W)) .+ FT(0.0) diff --git a/experiments/AMIP/modular/components/ocean/prescr_seaice_init.jl b/experiments/AMIP/modular/components/ocean/prescr_seaice_init.jl index 7f4c7a9d56..e0fb56f439 100644 --- a/experiments/AMIP/modular/components/ocean/prescr_seaice_init.jl +++ b/experiments/AMIP/modular/components/ocean/prescr_seaice_init.jl @@ -78,8 +78,8 @@ function ice_rhs!(du, u, p, _) rhs = @. (-F_turb_energy - F_radiative + F_conductive) / (params.h * params.ρ * params.c) # do not count tendencies that lead to temperatures above freezing, and mask out no-ice areas - unphysical = @. Regridder.binary_mask.(T_freeze - (Y.T_sfc + FT(rhs) * p.dt), threshold = FT(0)) .* - Regridder.binary_mask.(area_fraction, threshold = eps(FT)) + area_mask = Regridder.binary_mask.(area_fraction, threshold = eps(FT)) + unphysical = @. Regridder.binary_mask.(T_freeze - (Y.T_sfc + FT(rhs) * p.dt), threshold = FT(0)) .* area_mask parent(dY.T_sfc) .= parent(rhs .* unphysical) @. p.q_sfc = TD.q_vap_saturation_generic.(p.thermo_params, Y.T_sfc, p.ρ_sfc, TD.Ice()) diff --git a/experiments/AMIP/modular/components/ocean/slab_ocean_init.jl b/experiments/AMIP/modular/components/ocean/slab_ocean_init.jl index 42ee4fb345..44752c2bee 100644 --- a/experiments/AMIP/modular/components/ocean/slab_ocean_init.jl +++ b/experiments/AMIP/modular/components/ocean/slab_ocean_init.jl @@ -32,6 +32,7 @@ struct OceanSlabParameters{FT <: AbstractFloat} z0m::FT z0b::FT α::FT + evolving_switch::FT end name(::SlabOceanSimulation) = "SlabOceanSimulation" @@ -45,7 +46,7 @@ function slab_ocean_space_init(::Type{FT}, space, p) where {FT} T_sfc_0 = FT(p.T_init) #- FT(275) # close to the average of T_1 in atmos anomaly = false - anomaly_tropics = false + anomaly_tropics = true anom = FT(0) radlat = coord.lat / FT(180) * pi if anomaly == true @@ -55,7 +56,7 @@ function slab_ocean_space_init(::Type{FT}, space, p) where {FT} stdev = FT(5) / FT(180) * pi anom = anom_ampl * exp(-((radlat - lat_0)^2 / 2stdev^2 + (radlon - lon_0)^2 / 2stdev^2)) elseif anomaly_tropics == true - anom = FT(20 * cos(radlat)^4) + anom = FT(29) * exp(-coord.lat^2 / (2 * 26^2)) end T_sfc = T_sfc_0 + anom end @@ -71,7 +72,7 @@ function slab_ocean_rhs!(dY, Y, cache, t) p, F_turb_energy, F_radiative, area_fraction = cache FT = eltype(Y.T_sfc) rhs = @. -(F_turb_energy + F_radiative) / (p.h * p.ρ * p.c) - parent(dY.T_sfc) .= parent(rhs .* Regridder.binary_mask.(area_fraction, threshold = eps(FT))) + parent(dY.T_sfc) .= parent(rhs .* Regridder.binary_mask.(area_fraction, threshold = eps(FT))) * p.evolving_switch @. cache.q_sfc = TD.q_vap_saturation_generic.(cache.thermo_params, Y.T_sfc, cache.ρ_sfc, TD.Liquid()) end @@ -81,9 +82,21 @@ end Initializes the `DiffEq` problem, and creates a Simulation-type object containing the necessary information for `step!` in the coupling loop. """ -function ocean_init(::Type{FT}; tspan, dt, saveat, space, area_fraction, thermo_params, stepper = CTS.RK4()) where {FT} - - params = OceanSlabParameters(FT(20), FT(1500.0), FT(800.0), FT(280.0), FT(1e-3), FT(1e-5), FT(0.06)) +function ocean_init( + ::Type{FT}; + tspan, + dt, + saveat, + space, + area_fraction, + thermo_params, + stepper = CTS.RK4(), + evolving = true, +) where {FT} + + evolving_switch = evolving ? FT(1) : FT(0) + params = + OceanSlabParameters(FT(20), FT(1500.0), FT(800.0), FT(271.0), FT(1e-5), FT(1e-5), FT(0.38), evolving_switch) Y, space = slab_ocean_space_init(FT, space, params) cache = ( diff --git a/experiments/AMIP/modular/coupler_driver_modular.jl b/experiments/AMIP/modular/coupler_driver_modular.jl index 43cb8d1de6..9839c4e811 100644 --- a/experiments/AMIP/modular/coupler_driver_modular.jl +++ b/experiments/AMIP/modular/coupler_driver_modular.jl @@ -167,6 +167,7 @@ mono_surface = config_dict["mono_surface"] hourly_checkpoint = config_dict["hourly_checkpoint"] restart_dir = config_dict["restart_dir"] restart_t = Int(config_dict["restart_t"]) +evolving_ocean = config_dict["evolving_ocean"] ## I/O directory setup if isinteractive() @@ -217,37 +218,38 @@ boundary_space = atmos_sim.domain.face_space.horizontal_space land_fraction = Regridder.land_fraction(FT, REGRID_DIR, comms_ctx, land_mask_data, "LSMASK", boundary_space, mono = mono_surface) -#= -### Land -We use `ClimaLSM.jl`'s bucket model. -=# -land_sim = bucket_init( - FT, - FT.(tspan), - config_dict["land_domain_type"], - config_dict["land_albedo_type"], - comms_ctx, - REGRID_DIR; - dt = FT(Δt_cpl), - space = boundary_space, - saveat = FT(saveat), - area_fraction = land_fraction, - date_ref = date0, - t_start = FT(0), -) - #= ### Ocean and Sea Ice In the `AMIP` mode, all ocean properties are prescribed from a file, while sea-ice temperatures are calculated using observed SIC and assuming a 2m thickness of the ice. In the `SlabPlanet` mode, all ocean and sea ice are dynamical models, namely thermal slabs, with different parameters. + +### Land +If evolving, use `ClimaLSM.jl`'s bucket model. =# @info mode_name if mode_name == "amip" @info "AMIP boundary conditions - do not expect energy conservation" + ## land + land_sim = bucket_init( + FT, + FT.(tspan), + config_dict["land_domain_type"], + config_dict["land_albedo_type"], + config_dict["land_temperature_anomaly"], + comms_ctx, + REGRID_DIR; + dt = FT(Δt_cpl), + space = boundary_space, + saveat = FT(saveat), + area_fraction = land_fraction, + date_ref = date0, + t_start = FT(0), + ) + ## ocean SST_info = bcfile_info_init( FT, @@ -276,6 +278,7 @@ if mode_name == "amip" phase = TD.Liquid(), thermo_params = thermo_params, )) + ## sea ice SIC_info = bcfile_info_init( FT, @@ -323,7 +326,28 @@ if mode_name == "amip" mode_specifics = (; name = mode_name, SST_info = SST_info, SIC_info = SIC_info, CO2_info = CO2_info) -elseif mode_name == "slabplanet" +elseif mode_name in ("slabplanet", "slabplanet_aqua", "slabplanet_terra") + + land_fraction = mode_name == "slabplanet_aqua" ? land_fraction .* 0 : land_fraction + land_fraction = mode_name == "slabplanet_terra" ? land_fraction .* 0 .+ 1 : land_fraction + + ## land + land_sim = bucket_init( + FT, + FT.(tspan), + config_dict["land_domain_type"], + config_dict["land_albedo_type"], + config_dict["land_temperature_anomaly"], + comms_ctx, + REGRID_DIR; + dt = FT(Δt_cpl), + space = boundary_space, + saveat = FT(saveat), + area_fraction = land_fraction, + date_ref = date0, + t_start = FT(0), + ) + ## ocean ocean_sim = ocean_init( FT; @@ -333,6 +357,7 @@ elseif mode_name == "slabplanet" saveat = saveat, area_fraction = (FT(1) .- land_fraction), ## NB: this ocean fraction includes areas covered by sea ice (unlike the one contained in the cs) thermo_params = thermo_params, + evolving = evolving_ocean, ) ## sea ice (here set to zero area coverage) @@ -351,6 +376,24 @@ elseif mode_name == "slabplanet" mode_specifics = (; name = mode_name, SST_info = nothing, SIC_info = nothing) elseif mode_name == "slabplanet_eisenman" + + ## land + land_sim = bucket_init( + FT, + FT.(tspan), + config_dict["land_domain_type"], + config_dict["land_albedo_type"], + config_dict["land_temperature_anomaly"], + comms_ctx, + REGRID_DIR; + dt = FT(Δt_cpl), + space = boundary_space, + saveat = FT(saveat), + area_fraction = land_fraction, + date_ref = date0, + t_start = FT(0), + ) + ## ocean ocean_sim = ocean_init( FT; @@ -449,7 +492,7 @@ end dir_paths = (; output = COUPLER_OUTPUT_DIR, artifacts = COUPLER_ARTIFACTS_DIR) checkpoint_cb = - HourlyCallback(dt = FT(48), func = checkpoint_sims, ref_date = [dates.date[1]], active = hourly_checkpoint) # bi-daily + HourlyCallback(dt = FT(480), func = checkpoint_sims, ref_date = [dates.date[1]], active = hourly_checkpoint) # bi-daily update_firstdayofmonth!_cb = MonthlyCallback(dt = FT(1), func = update_firstdayofmonth!, ref_date = [dates.date1[1]], active = true) # for BCReader callbacks = (; checkpoint = checkpoint_cb, update_firstdayofmonth! = update_firstdayofmonth!_cb) @@ -550,7 +593,7 @@ function solve_coupler!(cs) ## print date on the first of month if cs.dates.date[1] >= cs.dates.date1[1] - @show(cs.dates.date[1]) + ClimaComms.iamroot(comms_ctx) ? @show(cs.dates.date[1]) : nothing end if cs.mode.name == "amip" diff --git a/experiments/AMIP/modular/user_io/user_diagnostics.jl b/experiments/AMIP/modular/user_io/user_diagnostics.jl index 42e28c4e65..f455538b55 100644 --- a/experiments/AMIP/modular/user_io/user_diagnostics.jl +++ b/experiments/AMIP/modular/user_io/user_diagnostics.jl @@ -59,19 +59,19 @@ function get_var(cs::CoupledSimulation, ::Val{:toa_fluxes}) n_faces = length(z[:, 1, 1, 1, 1]) LWd_TOA = Fields.level( - RRTMGPI.array2field(FT.(atmos_sim.integrator.p.radiation_model.face_lw_flux_dn), face_space), + CA.RRTMGPI.array2field(FT.(atmos_sim.integrator.p.radiation_model.face_lw_flux_dn), face_space), n_faces - half, ) LWu_TOA = Fields.level( - RRTMGPI.array2field(FT.(atmos_sim.integrator.p.radiation_model.face_lw_flux_up), face_space), + CA.RRTMGPI.array2field(FT.(atmos_sim.integrator.p.radiation_model.face_lw_flux_up), face_space), n_faces - half, ) SWd_TOA = Fields.level( - RRTMGPI.array2field(FT.(atmos_sim.integrator.p.radiation_model.face_sw_flux_dn), face_space), + CA.RRTMGPI.array2field(FT.(atmos_sim.integrator.p.radiation_model.face_sw_flux_dn), face_space), n_faces - half, ) SWu_TOA = Fields.level( - RRTMGPI.array2field(FT.(atmos_sim.integrator.p.radiation_model.face_sw_flux_up), face_space), + CA.RRTMGPI.array2field(FT.(atmos_sim.integrator.p.radiation_model.face_sw_flux_up), face_space), n_faces - half, ) diff --git a/target/atmos_driver.jl b/target/atmos_driver.jl new file mode 100644 index 0000000000..09eb21798f --- /dev/null +++ b/target/atmos_driver.jl @@ -0,0 +1,33 @@ +using ClimaComms +using Logging +using ClimaAtmos + +const is_distributed = get(ENV, "CLIMACORE_DISTRIBUTED", "") == "MPI" +if is_distributed + const comms_ctx = ClimaComms.context(ClimaComms.CPUSingleThreaded()) + const pid, nprocs = ClimaComms.init(comms_ctx) + + if ClimaComms.iamroot(comms_ctx) + Logging.global_logger(Logging.ConsoleLogger(stderr, Logging.Info)) + else + Logging.global_logger(Logging.NullLogger()) + end +else + using TerminalLoggers: TerminalLogger + const comms_ctx = ClimaComms.SingletonCommsContext() + const pid, nprocs = ClimaComms.init(comms_ctx) +end + + +ClimaComms.barrier(comms_ctx) + +# this tests the standalone runs, using the hybrid/driver +import Random +Random.seed!(1234) + +config = ClimaAtmos.AtmosConfig() +ClimaComms.barrier(comms_ctx) +# ClimaComms.iamroot(comms_ctx) ? @info(config) : nothing +integrator = ClimaAtmos.get_integrator(config) +ClimaComms.barrier(comms_ctx) +sol_res = ClimaAtmos.solve_atmos!(integrator) diff --git a/target/longrun_aquaplanet_rhoe_equilmoist_nz63_0M_55km_rs35km_clearsky.toml b/target/longrun_aquaplanet_rhoe_equilmoist_nz63_0M_55km_rs35km_clearsky.toml new file mode 100644 index 0000000000..7469c2505f --- /dev/null +++ b/target/longrun_aquaplanet_rhoe_equilmoist_nz63_0M_55km_rs35km_clearsky.toml @@ -0,0 +1,10 @@ +[alpha_rayleigh_w] +alias = "alpha_rayleigh_w" +value = 1.0 +type = "float" + +[zd_rayleigh] +alias = "zd_rayleigh" +value = 35000.0 +type = "float" + diff --git a/target/longrun_aquaplanet_rhoe_equilmoist_nz63_0M_55km_rs35km_clearsky_tvinsolation.yml b/target/longrun_aquaplanet_rhoe_equilmoist_nz63_0M_55km_rs35km_clearsky_tvinsolation.yml new file mode 100644 index 0000000000..aa67c24ad3 --- /dev/null +++ b/target/longrun_aquaplanet_rhoe_equilmoist_nz63_0M_55km_rs35km_clearsky_tvinsolation.yml @@ -0,0 +1,21 @@ +FLOAT_TYPE: "Float64" +dt_save_to_disk: "10days" +dt: "150secs" +t_end: "200days" +h_elem: 16 +z_elem: 63 +dz_bottom: 30.0 +dz_top: 3000.0 +z_max: 55000.0 +rayleigh_sponge: true +kappa_4: 2.0e16 +moist: "equil" +precip_model: "0M" +surface_setup: "DefaultMoninObukhov" +vert_diff: "true" +rad: "clearsky" +idealized_insolation: false +dt_rad: "1hours" +job_id: "longrun_aquaplanet_rhoe_equilmoist_nz63_0M_55km_rs35km_clearsky_tvinsolation" +toml: [target/longrun_aquaplanet_rhoe_equilmoist_nz63_0M_55km_rs35km_clearsky.toml] +start_date: "19790301" \ No newline at end of file diff --git a/test/Project.toml b/test/Project.toml index acd91714fd..dcfe5d305d 100644 --- a/test/Project.toml +++ b/test/Project.toml @@ -23,6 +23,7 @@ PrettyTables = "08abe8d2-0d0c-5749-adfa-8a2ac140af0d" Printf = "de0858da-6303-5e67-8744-51eddeeeb8d7" Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" SafeTestsets = "1bc83da4-3b8d-516f-aca4-4fe02f6d838f" +SimpleNonlinearSolve = "727e6d20-b764-4bd8-a329-72de5adea6c7" StaticArrays = "90137ffa-7385-5640-81b9-e52037218182" Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" SurfaceFluxes = "49b00bb7-8bd4-4f2b-b78c-51cd0450215f" @@ -39,6 +40,7 @@ MPI = "0.20" NCDatasets = "0.11, 0.12" Pkg = "1" PrettyTables = "2" +SimpleNonlinearSolve = "=0.1.23" StaticArrays = "1" Unitful = "1" julia = "1.8" diff --git a/toml/amip_longrun_target.toml b/toml/amip_longrun_target.toml new file mode 100644 index 0000000000..e50d88c0ca --- /dev/null +++ b/toml/amip_longrun_target.toml @@ -0,0 +1,9 @@ +[alpha_rayleigh_w] +alias = "alpha_rayleigh_w" +value = 1.0 +type = "float" + +[zd_rayleigh] +alias = "zd_rayleigh" +value = 35000.0 +type = "float" \ No newline at end of file diff --git a/toml/august.toml b/toml/august.toml new file mode 100644 index 0000000000..e50d88c0ca --- /dev/null +++ b/toml/august.toml @@ -0,0 +1,9 @@ +[alpha_rayleigh_w] +alias = "alpha_rayleigh_w" +value = 1.0 +type = "float" + +[zd_rayleigh] +alias = "zd_rayleigh" +value = 35000.0 +type = "float" \ No newline at end of file diff --git a/toml/default_coarse.toml b/toml/default_coarse.toml new file mode 100644 index 0000000000..b887173907 --- /dev/null +++ b/toml/default_coarse.toml @@ -0,0 +1,4 @@ +[C_E] +alias = "C_E" +value = 0.044 +type = "float" diff --git a/toml/slabplanet_default_longrun.toml b/toml/slabplanet_default_longrun.toml new file mode 100644 index 0000000000..b887173907 --- /dev/null +++ b/toml/slabplanet_default_longrun.toml @@ -0,0 +1,4 @@ +[C_E] +alias = "C_E" +value = 0.044 +type = "float"