Scaling pipeline for moist baroclinic wave

.buildkite/cpu_scaling_pipeline/generate_pipeline.jl

@@ -0,0 +1,69 @@
+# Generate a YAML pipeline to run CPU scaling tests on buildkite
+# To run: `julia --project=.buildkite .buildkite/cpu_scaling_pipeline/generate_pipeline.jl`
+# nodes = (1, 2, 4, 8, 16, 32)
+# helems = (30, 42, 60, 84, 120, 170)
+nodes = (1, 2, 4)
+helems = (30, 42, 60)
+
+import YAML
+
+agents = Dict("modules" => "climacommon/2024_12_16", "queue" => "new-central")
+env = Dict(
+    "SLURM_KILL_BAD_EXIT" => 1,
+    "OPENBLAS_NUM_THREADS" => 1,
+    "JULIA_NVTX_CALLBACKS" => "gc",
+    "JULIA_LOAD_PATH" => "\${JULIA_LOAD_PATH}:\${BUILDKITE_BUILD_CHECKOUT_PATH}/.buildkite",
+)
+
+CPU_TYPE = "icelake"
+MBW_SCALING_CONFIG_PATH = "config/mbw_scaling_configs"
+
+init_step = Dict(
+    "label" => "init :computer:",
+    "key" => "init_cpu_env",
+    "agents" => Dict("slurm_cpus_per_task" => 8),
+    "command" => [
+        """echo --- Instantiate project""",
+        "julia --project -e 'using Pkg; Pkg.instantiate(;verbose=true)'",
+        "julia --project -e 'using Pkg; Pkg.precompile(); Pkg.status()'",
+        """echo --- Instantiate .buildkite""",
+        "julia --project=.buildkite -e 'using Pkg; Pkg.instantiate(;verbose=true)'",
+        "julia --project=.buildkite -e 'using Pkg; Pkg.precompile(); Pkg.status()'",
+    ],
+    "env" => Dict(
+        "JULIA_NUM_PRECOMPILE_TASKS" => 8,
+        "JULIA_MAX_NUM_PRECOMPILE_FILES" => 50,
+    ),
+)
+
+function generate_step(nodes::Int, helems::Int)
+    return Dict(
+        "label" => ":computer: $nodes node, 16 processes per node, helem = $helems",
+        "command" => "srun julia --color=yes --project=.buildkite .buildkite/ci_driver.jl --config_file $MBW_SCALING_CONFIG_PATH/moist_baroclinic_wave_helem_$(helems)_0M_ws.yml --job_id moist_baroclinic_wave_helem_$(helems)_0M_ws",
+        "artifact_paths" => "moist_baroclinic_wave_helem_$(helems)_0M_ws/output_active/*",
+        "agents" => Dict(
+            "slurm_constraint" => CPU_TYPE,
+            "queue" => "new-central",
+            "slurm_nodes" => nodes,
+            "slurm_tasks_per_node" => 16,
+            "slurm_cpus_per_task" => 1,
+            "slurm_mem" => 0,
+            "slurm_time" => "1:00:00",
+            "slurm_exclusive" => true,
+        ),
+    )
+end
+
+pipeline = Dict(
+    "agents" => agents,
+    "env" => env,
+    "steps" => [
+        init_step,
+        "wait",
+        Dict(
+            "group" => "Moist Baroclinic Wave, weak scaling",
+            "steps" => [generate_step.(nodes, helems)...],
+        ),
+    ],
+)
+YAML.write_file(".buildkite/cpu_scaling_pipeline/pipeline.yml", pipeline)

.buildkite/cpu_scaling_pipeline/pipeline.yml

@@ -0,0 +1,62 @@
+agents:
+  modules: "climacommon/2024_12_16"
+  queue: "new-central"
+steps:
+  - label: "init :computer:"
+    key: "init_cpu_env"
+    agents:
+      slurm_cpus_per_task: 8
+    command:
+      - "echo --- Instantiate project"
+      - "julia --project -e 'using Pkg; Pkg.instantiate(;verbose=true)'"
+      - "julia --project -e 'using Pkg; Pkg.precompile(); Pkg.status()'"
+      - "echo --- Instantiate .buildkite"
+      - "julia --project=.buildkite -e 'using Pkg; Pkg.instantiate(;verbose=true)'"
+      - "julia --project=.buildkite -e 'using Pkg; Pkg.precompile(); Pkg.status()'"
+    env:
+      JULIA_NUM_PRECOMPILE_TASKS: 8
+      JULIA_MAX_NUM_PRECOMPILE_FILES: 50
+  - "wait"
+  - steps:
+      - label: ":computer: 1 node, 16 processes per node, helem = 30"
+        agents:
+          slurm_time: "1:00:00"
+          slurm_cpus_per_task: 1
+          slurm_tasks_per_node: 16
+          queue: "new-central"
+          slurm_nodes: 1
+          slurm_exclusive: true
+          slurm_mem: 0
+          slurm_constraint: "icelake"
+        command: "srun julia --color=yes --project=.buildkite .buildkite/ci_driver.jl --config_file config/mbw_scaling_configs/moist_baroclinic_wave_helem_30_0M_ws.yml --job_id moist_baroclinic_wave_helem_30_0M_ws"
+        artifact_paths: "moist_baroclinic_wave_helem_30_0M_ws/output_active/*"
+      - label: ":computer: 2 node, 16 processes per node, helem = 42"
+        agents:
+          slurm_time: "1:00:00"
+          slurm_cpus_per_task: 1
+          slurm_tasks_per_node: 16
+          queue: "new-central"
+          slurm_nodes: 2
+          slurm_exclusive: true
+          slurm_mem: 0
+          slurm_constraint: "icelake"
+        command: "srun julia --color=yes --project=.buildkite .buildkite/ci_driver.jl --config_file config/mbw_scaling_configs/moist_baroclinic_wave_helem_42_0M_ws.yml --job_id moist_baroclinic_wave_helem_42_0M_ws"
+        artifact_paths: "moist_baroclinic_wave_helem_42_0M_ws/output_active/*"
+      - label: ":computer: 4 node, 16 processes per node, helem = 60"
+        agents:
+          slurm_time: "1:00:00"
+          slurm_cpus_per_task: 1
+          slurm_tasks_per_node: 16
+          queue: "new-central"
+          slurm_nodes: 4
+          slurm_exclusive: true
+          slurm_mem: 0
+          slurm_constraint: "icelake"
+        command: "srun julia --color=yes --project=.buildkite .buildkite/ci_driver.jl --config_file config/mbw_scaling_configs/moist_baroclinic_wave_helem_60_0M_ws.yml --job_id moist_baroclinic_wave_helem_60_0M_ws"
+        artifact_paths: "moist_baroclinic_wave_helem_60_0M_ws/output_active/*"
+    group: "Moist Baroclinic Wave, weak scaling"
+env:
+  SLURM_KILL_BAD_EXIT: 1
+  OPENBLAS_NUM_THREADS: 1
+  JULIA_NVTX_CALLBACKS: "gc"
+  JULIA_LOAD_PATH: "${JULIA_LOAD_PATH}:${BUILDKITE_BUILD_CHECKOUT_PATH}/.buildkite"

config/mbw_scaling_configs/moist_baroclinic_wave_helem_120_0M_ss.yml

@@ -0,0 +1,12 @@
+h_elem: 120
+z_elem: 43
+dz_bottom: 30.0
+dt: "40secs"
+t_end: "1days"
+ode_algo: ARS343
+initial_condition: "MoistBaroclinicWave"
+moist: "equil"
+precip_model: "0M"
+dt_save_state_to_disk: "Inf"
+dt_save_to_sol: "Inf"
+output_default_diagnostics: false

config/mbw_scaling_configs/moist_baroclinic_wave_helem_120_0M_ws.yml

@@ -0,0 +1,12 @@
+h_elem: 120
+z_elem: 43
+dz_bottom: 30.0
+dt: "6secs"
+t_end: "1days"
+ode_algo: ARS343
+initial_condition: "MoistBaroclinicWave"
+moist: "equil"
+precip_model: "0M"
+dt_save_state_to_disk: "Inf"
+dt_save_to_sol: "Inf"
+output_default_diagnostics: false

config/mbw_scaling_configs/moist_baroclinic_wave_helem_170_0M_ws.yml

@@ -0,0 +1,12 @@
+h_elem: 170
+z_elem: 43
+dz_bottom: 30.0
+dt: "6secs"
+t_end: "1days"
+ode_algo: ARS343
+initial_condition: "MoistBaroclinicWave"
+moist: "equil"
+precip_model: "0M"
+dt_save_state_to_disk: "Inf"
+dt_save_to_sol: "Inf"
+output_default_diagnostics: false