set up surface hooks

add bk pipelines

clean

fix long pip

clean interactive setup

temp manifest check in

try

no pkg precomp

revert manifest

revert gitignore

spec atmos version

down SimpleNonlinearSolve

.buildkite/longruns/pipeline.yml

@@ -16,7 +16,6 @@ env:
 
   LD_LIBRARY_PATH: "/central/scratch/esm/slurm-buildkite/climaatmos-ci/depot/cpu/artifacts/e9feeabfa989549ac1dac61e4df5f2849149ad6d/lib/:${LD_LIBRARY_PATH}"
 
-
 timeout_in_minutes: 1440
 
 steps:
@@ -51,21 +50,187 @@ steps:
 
   - wait
 
-  - group: "Targeted resolution coupled AMIP long runs"
+  - group: "Coupler integration and conservation tests"
 
     steps:
 
-      - label: "Slabplanet: default"
-        key: "slabplanet_default_longrun"
-        command: "julia --color=yes --project=experiments/AMIP/modular/ experiments/AMIP/modular/coupler_driver_modular.jl --config_file $CONFIG_PATH/slabplanet_default_longrun.yml"
-        artifact_paths: "experiments/AMIP/modular/output/slabplanet/slabplanet_default_longrun_artifacts/*"
+      # Integration tests - the expected results were also confirmed locally
+      - label: "Slabplanet_aqua: nocouple"
+        key: "slabplanet_aqua_atmos_sf_nocouple" # SF at each Atmos stage, no coupling, prescribed SST from atmos
+        command: "julia --color=yes --project=experiments/AMIP/modular/ experiments/AMIP/modular/coupler_driver_modular.jl --config_file $CONFIG_PATH/slabplanet_aqua_atmos_sf_nocouple.yml"
+        artifact_paths: "experiments/AMIP/modular/output/slabplanet_aqua/slabplanet_aqua_atmos_sf_nocouple_artifacts/*"
+        env:
+          BUILD_HISTORY_HANDLE: ""
+        agents:
+          slurm_ntasks_per_node: 1
+          slurm_nodes: 1
+          slurm_mem_per_cpu: 16G
+
+      - label: "Slabplanet_aqua: couple"
+        key: "slabplanet_aqua_atmos_sf_couple" # SF at each Atmos stage, coupling, prescribed SST from coupler - identical results to the above confirm 1) initial conditions in Atmos are unchanged compared to the slab, 2) coupling not introducing variability when constant surface
+        command: "julia --color=yes --project=experiments/AMIP/modular/ experiments/AMIP/modular/coupler_driver_modular.jl --config_file $CONFIG_PATH/slabplanet_aqua_atmos_sf_couple.yml"
+        artifact_paths: "experiments/AMIP/modular/output/slabplanet_aqua/slabplanet_aqua_atmos_sf_couple_artifacts/*"
+        env:
+          BUILD_HISTORY_HANDLE: ""
+        agents:
+          slurm_ntasks_per_node: 1
+          slurm_nodes: 1
+          slurm_mem_per_cpu: 16G
+
+      - label: "Slabplanet_aqua: coupler fluxes"
+        key: "slabplanet_aqua_coupler_sf" # SF at each coupler timestep, constant ocean - comparing to the above runs, this tests the sensitivity of less frequent flux calculation
+        command: "julia --color=yes --project=experiments/AMIP/modular/ experiments/AMIP/modular/coupler_driver_modular.jl --config_file $CONFIG_PATH/slabplanet_aqua_coupler_sf.yml"
+        artifact_paths: "experiments/AMIP/modular/output/slabplanet_aqua/slabplanet_aqua_coupler_sf_artifacts/*"
+        env:
+          BUILD_HISTORY_HANDLE: ""
+        agents:
+          slurm_ntasks_per_node: 1
+          slurm_nodes: 1
+          slurm_mem_per_cpu: 16G
+
+      - label: "Slabplanet_aqua: coupler fluxes, evolving ocean"
+        key: "slabplanet_aqua_coupler_sf_evolve_ocn" # SF at each coupler timestep, evolving ocean - comparing to the above run, tests the sensitivity of evolving ocean
+        command: "julia --color=yes --project=experiments/AMIP/modular/ experiments/AMIP/modular/coupler_driver_modular.jl --config_file $CONFIG_PATH/slabplanet_aqua_coupler_sf_evolve_ocn.yml"
+        artifact_paths: "experiments/AMIP/modular/output/slabplanet_aqua/slabplanet_aqua_coupler_sf_evolve_ocn_artifacts/*"
+        env:
+          BUILD_HISTORY_HANDLE: ""
+        agents:
+          slurm_ntasks_per_node: 1
+          slurm_nodes: 1
+          slurm_mem_per_cpu: 16G
+
+      - label: "Slabplanet_terra: coupler fluxes, evolving bucket"
+        key: "slabplanet_terra" # SF at each coupler timestep, evolving ocean - comparing to the above run, tests the sensitivity of evolving bucket
+        command: "julia --color=yes --project=experiments/AMIP/modular/ experiments/AMIP/modular/coupler_driver_modular.jl --config_file $CONFIG_PATH/slabplanet_terra.yml"
+        artifact_paths: "experiments/AMIP/modular/output/slabplanet_terra/slabplanet_terra_artifacts/*"
+        env:
+          BUILD_HISTORY_HANDLE: ""
+        agents:
+          slurm_ntasks_per_node: 1
+          slurm_nodes: 1
+          slurm_mem_per_cpu: 16G
+
+      - label: "Slabplanet: coupler fluxes, evolving ocean and land"
+        key: "slabplanet_coupler_sf_evolve_ocn"
+        command: "julia --color=yes --project=experiments/AMIP/modular/ experiments/AMIP/modular/coupler_driver_modular.jl --config_file $CONFIG_PATH/slabplanet_coupler_sf_evolve_ocn.yml"
+        artifact_paths: "experiments/AMIP/modular/output/slabplanet_aqua/slabplanet_coupler_sf_evolve_ocn_artifacts/*"
         env:
           BUILD_HISTORY_HANDLE: ""
         agents:
           slurm_ntasks_per_node: 1
           slurm_nodes: 1
           slurm_mem_per_cpu: 16G
 
+  - group: "Current target tests: idealized surface"
+
+    steps:
+
+      - label: "ClimaAtmos standalone target"
+        command:
+          - srun julia --project=experiments/AMIP/modular/ target/atmos_driver.jl --config_file target/longrun_aquaplanet_rhoe_equilmoist_nz63_0M_55km_rs35km_clearsky_tvinsolation.yml
+        artifact_paths: "output/longrun_aquaplanet_rhoe_equilmoist_nz63_0M_55km_rs35km_clearsky_tvinsolation/*"
+        env:
+          BUILD_HISTORY_HANDLE: ""
+          CLIMACORE_DISTRIBUTED: "MPI"
+        agents:
+          slurm_ntasks_per_node: 16
+          slurm_nodes: 4
+          slurm_mem_per_cpu: 16G
+
+      - label: "TARGET IDEALIZED: new target aqua - fixed ocean T, nocouple, atmos flux calc"
+        key: "slabplanet_aqua_new_target_nocouple"
+        command: "mpiexec julia --color=yes --project=experiments/AMIP/modular/ experiments/AMIP/modular/coupler_driver_modular.jl --config_file $CONFIG_PATH/slabplanet_aqua_new_target_nocouple.yml"
+        artifact_paths: "experiments/AMIP/modular/output/slabplanet_aqua/slabplanet_aqua_new_target_nocouple_artifacts/*"
+        env:
+          CLIMACORE_DISTRIBUTED: "MPI"
+          BUILD_HISTORY_HANDLE: ""
+        agents:
+          slurm_ntasks_per_node: 16
+          slurm_nodes: 4
+          slurm_mem_per_cpu: 16G
+
+      - label: "TARGET IDEALIZED: new target aqua - fixed ocean T, coupler flux calc"
+        key: "slabplanet_aqua_new_target"
+        command: "mpiexec julia --color=yes --project=experiments/AMIP/modular/ experiments/AMIP/modular/coupler_driver_modular.jl --config_file $CONFIG_PATH/slabplanet_aqua_new_target.yml"
+        artifact_paths: "experiments/AMIP/modular/output/slabplanet_aqua/slabplanet_aqua_new_target_artifacts/*"
+        env:
+          CLIMACORE_DISTRIBUTED: "MPI"
+          BUILD_HISTORY_HANDLE: ""
+        agents:
+          slurm_ntasks_per_node: 16
+          slurm_nodes: 4
+          slurm_mem_per_cpu: 16G
+
+      - label: "TARGET IDEALIZED: new target slab aqua - evolving slab ocean T"
+        key: "slabplanet_aqua_new_target_evolve_ocn"
+        command: "mpiexec julia --color=yes --project=experiments/AMIP/modular/ experiments/AMIP/modular/coupler_driver_modular.jl --config_file $CONFIG_PATH/slabplanet_aqua_new_target_evolve_ocn.yml"
+        artifact_paths: "experiments/AMIP/modular/output/slabplanet/slabplanet_aqua_new_target_evolve_ocn_artifacts/*"
+        env:
+          CLIMACORE_DISTRIBUTED: "MPI"
+          BUILD_HISTORY_HANDLE: ""
+        agents:
+          slurm_ntasks_per_node: 16
+          slurm_nodes: 4
+          slurm_mem_per_cpu: 16G
+
+      - label: "TARGET IDEALIZED: new target slab - fixed ocean T, bucket"
+        key: "slabplanet_new_target"
+        command: "mpiexec julia --color=yes --project=experiments/AMIP/modular/ experiments/AMIP/modular/coupler_driver_modular.jl --config_file $CONFIG_PATH/slabplanet_new_target.yml"
+        artifact_paths: "experiments/AMIP/modular/output/slabplanet/slabplanet_new_target_artifacts/*"
+        env:
+          CLIMACORE_DISTRIBUTED: "MPI"
+          BUILD_HISTORY_HANDLE: ""
+        agents:
+          slurm_ntasks_per_node: 16
+          slurm_nodes: 4
+          slurm_mem_per_cpu: 16G
+
+      - label: "TARGET IDEALIZED: new target slab - evolving slab ocean T, bucket"
+        key: "slabplanet_new_target_evolve_ocn"
+        command: "mpiexec julia --color=yes --project=experiments/AMIP/modular/ experiments/AMIP/modular/coupler_driver_modular.jl --config_file $CONFIG_PATH/slabplanet_new_target_evolve_ocn.yml"
+        artifact_paths: "experiments/AMIP/modular/output/slabplanet/slabplanet_new_target_evolve_ocn_artifacts/*"
+        env:
+          CLIMACORE_DISTRIBUTED: "MPI"
+          BUILD_HISTORY_HANDLE: ""
+        agents:
+          slurm_ntasks_per_node: 16
+          slurm_nodes: 4
+          slurm_mem_per_cpu: 16G
+
+
+  - group: "Current target tests: AMIP surface"
+
+    steps:
+
+      - label: "MPI AMIP FINE: new target amip"
+        key: "amip_new_target"
+        command: "mpiexec julia --color=yes --project=experiments/AMIP/modular/ experiments/AMIP/modular/coupler_driver_modular.jl --config_file $CONFIG_PATH/amip_new_target.yml"
+        artifact_paths: "experiments/AMIP/modular/output/amip/amip_new_target_artifacts/*"
+        env:
+          CLIMACORE_DISTRIBUTED: "MPI"
+          BUILD_HISTORY_HANDLE: ""
+        agents:
+          slurm_ntasks_per_node: 16
+          slurm_nodes: 4
+          slurm_mem_per_cpu: 16G
+
+      - label: "MPI AMIP FINE: new target amip: topo"
+        key: "amip_new_target_topo"
+        command: "mpiexec julia --color=yes --project=experiments/AMIP/modular/ experiments/AMIP/modular/coupler_driver_modular.jl --config_file $CONFIG_PATH/amip_new_target_topo.yml"
+        artifact_paths: "experiments/AMIP/modular/output/amip/amip_new_target_topo_artifacts/*"
+        env:
+          CLIMACORE_DISTRIBUTED: "MPI"
+          BUILD_HISTORY_HANDLE: ""
+        agents:
+          slurm_ntasks_per_node: 16
+          slurm_nodes: 4
+          slurm_mem_per_cpu: 16G
+
+
+  - group: "Other AMIP targets"
+
+    steps:
+
       # DYAMOND AMIP: 1 day (convection resolving)
 
       - label: "MPI AMIP SUPERFINE: dyamond_target"
@@ -80,11 +245,10 @@ steps:
           slurm_nodes: 4
           slurm_mem_per_cpu: 16G
 
-      # mid-resolution AMIP: longrun (140 days)
-      - label: "MPI AMIP FINE: target longrun"
-        key: "amip_longrun_target"
-        command: "mpiexec julia --color=yes --project=experiments/AMIP/modular/ experiments/AMIP/modular/coupler_driver_modular.jl --config_file $CONFIG_PATH/amip_longrun_target.yml"
-        artifact_paths: "experiments/AMIP/modular/output/amip/amip_longrun_target_artifacts/*"
+      - label: "MPI AMIP FINE: august"
+        key: "august"
+        command: "mpiexec julia --color=yes --project=experiments/AMIP/modular/ experiments/AMIP/modular/coupler_driver_modular.jl --config_file $CONFIG_PATH/august.yml"
+        artifact_paths: "experiments/AMIP/modular/output/amip/august_artifacts/*"
         env:
           CLIMACORE_DISTRIBUTED: "MPI"
           BUILD_HISTORY_HANDLE: ""
@@ -190,14 +354,14 @@ steps:
 
       - label: ":envelope: Slack report: Slabplanet"
         command:
-          - slack-upload -c "#coupler-report" -f experiments/AMIP/modular/output/slabplanet/slabplanet_default_longrun_artifacts/total_energy_log_bucket.png -m png -n slab_coarse_log -x "Slabplanet energy conservation (log error)"
-          - slack-upload -c "#coupler-report" -f experiments/AMIP/modular/output/slabplanet/slabplanet_default_longrun_artifacts/total_energy_bucket.png -m png -n slab_coarse -x "Slabplanet energy conservation"
-          - slack-upload -c "#coupler-report" -f experiments/AMIP/modular/output/slabplanet/slabplanet_default_longrun_artifacts/total_water_log_bucket.png -m png -n slab_coarse_w_log -x "Slabplanet water conservation (log error)"
-          - slack-upload -c "#coupler-report" -f experiments/AMIP/modular/output/slabplanet/slabplanet_default_longrun_artifacts/total_water_bucket.png -m png -n slab_coarse_w -x "Slabplanet water conservation"
+          - slack-upload -c "#coupler-report" -f experiments/AMIP/modular/output/slabplanet/slabplanet_coupler_sf_evolve_ocn_artifacts/total_energy_log_bucket.png -m png -n slab_coarse_log -x "Slabplanet energy conservation (log error)"
+          - slack-upload -c "#coupler-report" -f experiments/AMIP/modular/output/slabplanet/slabplanet_coupler_sf_evolve_ocn_artifacts/total_energy_bucket.png -m png -n slab_coarse -x "Slabplanet energy conservation"
+          - slack-upload -c "#coupler-report" -f experiments/AMIP/modular/output/slabplanet/slabplanet_coupler_sf_evolve_ocn_artifacts/total_water_log_bucket.png -m png -n slab_coarse_w_log -x "Slabplanet water conservation (log error)"
+          - slack-upload -c "#coupler-report" -f experiments/AMIP/modular/output/slabplanet/slabplanet_coupler_sf_evolve_ocn_artifacts/total_water_bucket.png -m png -n slab_coarse_w -x "Slabplanet water conservation"
 
       - label: ":envelope: Slack report: target AMIP"
         command:
-          - slack-upload -c "#coupler-report" -f experiments/AMIP/modular/output/amip/amip_longrun_target_artifacts/amip_paperplots.png -m png -n amip_fine -x "AMIP Target Longrun"
+          - slack-upload -c "#coupler-report" -f experiments/AMIP/modular/output/amip/amip_new_target_artifacts/amip_paperplots.png -m png -n amip_fine -x "AMIP Target Longrun"
 
       - label: ":envelope: Slack report: Flame Diff"
         command:

.buildkite/pipeline.yml

@@ -1,6 +1,5 @@
 agents:
   queue: central
-  slurm_mem: 8G
   slurm_time: 24:00:00
   modules: julia/1.9.3 cuda/12.2 ucx/1.14.1_cuda-12.2 openmpi/4.1.5_cuda-12.2 hdf5/1.12.2-ompi415 nsight-systems/2023.2.1
 
@@ -24,15 +23,14 @@ steps:
   - label: "init environment :computer:"
     key: "init_cpu_env"
     command:
-      # - "echo $$JULIA_DEPOT_PATH"
 
       - echo "--- Configure MPI"
       - julia -e 'using Pkg; Pkg.add("MPIPreferences"); using MPIPreferences; use_system_binary()'
 
-      - echo "--- Instantiate package env"
-      - "julia --project -e 'using Pkg; Pkg.instantiate(;verbose=true)'"
-      - "julia --project -e 'using Pkg; Pkg.precompile()'"
-      - "julia --project -e 'using Pkg; Pkg.status()'"
+      # - echo "--- Instantiate package env"
+      # - "julia --project -e 'using Pkg; Pkg.instantiate(;verbose=true)'"
+      # - "julia --project -e 'using Pkg; Pkg.precompile()'"
+      # - "julia --project -e 'using Pkg; Pkg.status()'"
 
       - echo "--- Instantiate sea breeze env"
       - "julia --project=experiments/ClimaCore/sea_breeze -e 'using Pkg; Pkg.instantiate(;verbose=true)'"

Project.toml

@@ -18,6 +18,7 @@ NCDatasets = "85f8d34a-cbdd-5861-8df4-14fed0d494ab"
 OrdinaryDiffEq = "1dea7af3-3e70-54e6-95c3-0bf5283fa5ed"
 Plots = "91a5bcdd-55d7-5caf-9e0b-520d859cae80"
 SciMLBase = "0bca4576-84f4-4d90-8ffe-ffa030f20462"
+SimpleNonlinearSolve = "727e6d20-b764-4bd8-a329-72de5adea6c7"
 StaticArrays = "90137ffa-7385-5640-81b9-e52037218182"
 Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2"
 SurfaceFluxes = "49b00bb7-8bd4-4f2b-b78c-51cd0450215f"
@@ -28,7 +29,7 @@ UnPack = "3a884ed6-31ef-47d7-9d2a-63182c4928ed"
 
 [compat]
 CLIMAParameters = "0.4, 0.5, 0.6, 0.7"
-ClimaAtmos = "0.16"
+ClimaAtmos = "=0.16.2"
 ClimaComms = "0.5"
 ClimaCore = "0.10"
 ClimaCoreTempestRemap = "0.3"
@@ -40,6 +41,7 @@ NCDatasets = "0.11, 0.12"
 OrdinaryDiffEq = "5, 6"
 Plots = "1.39.0"
 SciMLBase = "1"
+SimpleNonlinearSolve = "=0.1.23"
 StaticArrays = "1"
 SurfaceFluxes = "0.7"
 TempestRemap_jll = "2"

config/longrun_configs/amip_new_target.yml

@@ -0,0 +1,18 @@
+run_name: "amip_new_target"
+anim: true
+dt_cpl: 150
+energy_check: false
+mode_name: "amip"
+mono_surface: false
+dt: "150secs"
+t_end: "300days" # TODO this has been decreased from 140 days to avoid instability #460
+job_id: "amip_new_target"
+dt_save_to_sol: "5days"
+dt_save_to_disk: "5days"
+apply_limiter: false
+hourly_checkpoint: true
+turb_flux_partition: "CombinedStateFluxes"
+atmos_config_file: "config/longrun_configs/longrun_aquaplanet_rhoe_equilmoist_nz63_0M_55km_rs35km_clearsky_tvinsolation.yml"
+surface_setup: "PrescribedSurface"
+land_albedo_type: "map_temporal"
+start_date: "19790301"

config/longrun_configs/amip_new_target_topo.yml

@@ -0,0 +1,21 @@
+run_name: "amip_new_target_topo"
+anim: true
+dt_cpl: 75
+energy_check: false
+mode_name: "amip"
+mono_surface: false
+dt: "150secs"
+t_end: "100days" # TODO this has been decreased from 140 days to avoid instability #460
+job_id: "amip_new_target_topo"
+dt_save_to_sol: "5days"
+dt_save_to_disk: "5days"
+apply_limiter: false
+hourly_checkpoint: true
+turb_flux_partition: "CombinedStateFluxes"
+atmos_config_file: "config/longrun_configs/longrun_aquaplanet_rhoe_equilmoist_nz63_0M_55km_rs35km_clearsky_tvinsolation.yml"
+surface_setup: "PrescribedSurface"
+land_albedo_type: "map_temporal"
+start_date: "19790301"
+topography: "Earth"
+topo_smoothing: true
+use_reference_state: false

config/longrun_configs/august.yml

@@ -0,0 +1,33 @@
+run_name: "august"
+coupled: true
+start_date: "19790301"
+monthly_checkpoint: true
+surface_setup: "PrescribedSurface"
+dt_cpl: 100
+energy_check: false
+mode_name: "amip"
+mono_surface: false
+vert_diff: "true"
+moist: "equil"
+rad: "clearsky"
+precip_model: "0M"
+z_elem: 35
+dz_bottom: 50
+h_elem: 12
+kappa_4: 4e16
+rayleigh_sponge: true
+alpha_rayleigh_uh: 0
+dt: "100secs"
+t_end: "400days"
+job_id: "august"
+dt_save_to_sol: "10days"
+dt_save_to_disk: "1days"
+apply_limiter: false
+FLOAT_TYPE: "Float64"
+post_process: false
+anim: true
+hourly_checkpoint: true
+apply_limiter: false
+surface_setup: "PrescribedSurface"
+land_albedo_type: "function"
+coupler_toml_file: "toml/august.toml"

config/longrun_configs/dyamond_target.yml

@@ -10,5 +10,6 @@ dt_save_to_sol: "0.5days"
 dt_save_to_disk: "0.5days"
 turb_flux_partition: "CombinedStateFluxes"
 atmos_config_file: "config/longrun_configs/longrun_aquaplanet_dyamond.yml"
-atmos_toml_file: "toml/longrun_aquaplanet_dyamond.toml"
-hourly_checkpoint: false
+monthly_checkpoint: false
+land_albedo_type: "map_temporal"
+start_date: "19790301"