Make all steps in buildkite pipeline use reservation

Reduce flame graph: perf target (prognostic edmfx aquaplanet)
memory to 32 GB, remove threaded benchmark and flame graph,
and move calibration to its own pipeline.

Author: imreddyTeja

.buildkite/pipeline.yml

@@ -549,20 +549,6 @@ steps:
         agents:
           slurm_mem: 20GB
 
-  - group: "Calibration"
-    steps:
-      - label: "Calibration interface unit tests"
-        command: julia --project=calibration/test calibration/test/interface.jl
-      - label: "end to end test"
-        command: julia --project=calibration/test calibration/test/e2e_test.jl
-        agents:
-          slurm_ntasks: 10
-          slurm_cpus_per_task: 1
-          slurm_mem: 96GB
-          slurm_reservation: "false"
-          slurm_time: "00:30:00"
-        artifact_paths: "calibration_end_to_end_test/*"
-
   - group: "Diagnostic EDMFX"
     steps:
 
@@ -1047,16 +1033,6 @@ steps:
           slurm_mem: 24GB
           slurm_gpus: 1
 
-      - label: ":computer: Benchmark: CPU perf target (Threaded)"
-        command: >
-          julia --color=yes --threads 8 --project=.buildkite perf/benchmark.jl
-          --config_file $PERF_CONFIG_PATH/bm_perf_target_threaded.yml
-          --job_id bm_perf_target_threaded
-        agents:
-          slurm_mem: 24GB
-          slurm_cpus_per_task: 8
-          slurm_reservation: "false"
-
       - label: ":computer: Benchmark: GPU diag edmf"
         command: >
           julia --color=yes --project=.buildkite perf/benchmark.jl
@@ -1117,29 +1093,7 @@ steps:
           --job_id flame_perf_target_prognostic_edmfx_aquaplanet
         artifact_paths: "flame_perf_target_prognostic_edmfx_aquaplanet/*"
         agents:
-          slurm_mem: 48GB
-          slurm_reservation: "false"
-
-      - label: ":fire: Flame graph: perf target (frierson diffusion)"
-        command: >
-          julia --color=yes --project=.buildkite perf/flame.jl
-          --config_file $PERF_CONFIG_PATH/flame_perf_target_frierson.yml
-          --job_id flame_perf_target_frierson
-        artifact_paths: "flame_perf_target_frierson/*"
-        agents:
-          slurm_mem: 48GB
-          slurm_reservation: "false"
-
-      - label: ":fire: Flame graph: perf target (Threaded)"
-        command: >
-          julia --threads 8 --color=yes --project=.buildkite perf/flame.jl
-          --config_file $PERF_CONFIG_PATH/flame_perf_target_threaded.yml
-          --job_id flame_perf_target_threaded
-        artifact_paths: "flame_perf_target_threaded/*"
-        agents:
-          slurm_cpus_per_task: 8
-          slurm_mem: 24GB
-          slurm_reservation: "false"
+          slurm_mem: 32GB
 
       - label: ":fire: Flame graph: perf target (Callbacks)"
         command: >

calibration/test/pipeline.yml

@@ -0,0 +1,88 @@
+agents:
+  queue: new-central
+  slurm_mem: 8G
+  modules: climacommon/2024_12_16
+env:
+  JULIA_LOAD_PATH: "${JULIA_LOAD_PATH}:${BUILDKITE_BUILD_CHECKOUT_PATH}/.buildkite"
+  OPENBLAS_NUM_THREADS: 1
+  SLURM_KILL_BAD_EXIT: 1
+  JULIA_NVTX_CALLBACKS: gc
+  JULIA_MAX_NUM_PRECOMPILE_FILES: 100
+  JULIA_DEPOT_PATH: "${BUILDKITE_BUILD_PATH}/${BUILDKITE_PIPELINE_SLUG}/depot/default"
+  CONFIG_PATH: "config/model_configs"
+  GPU_CONFIG_PATH: "config/gpu_configs"
+  PERF_CONFIG_PATH: "config/perf_configs"
+  MPI_CONFIG_PATH: "config/mpi_configs"
+
+steps:
+  - label: "init :computer:"
+    key: "init_cpu_env"
+    concurrency: 1
+    concurrency_group: 'depot/climaatmos-ci'
+    command:
+      - "echo $$JULIA_DEPOT_PATH"
+
+      - echo "--- Instantiate project"
+      - "julia --project -e 'using Pkg; Pkg.instantiate(;verbose=true)'"
+      - "julia --project -e 'using Pkg; Pkg.precompile()'"
+      - "julia --project -e 'using Pkg; Pkg.status()'"
+
+      - echo "--- Instantiate .buildkite"
+      - "julia --project=.buildkite -e 'using Pkg; Pkg.instantiate(;verbose=true)'"
+      - "julia --project=.buildkite -e 'using Pkg; Pkg.precompile()'"
+      - "julia --project=.buildkite -e 'using CUDA; CUDA.precompile_runtime()'"
+      - "julia --project=.buildkite -e 'using Pkg; Pkg.status()'"
+
+      - echo "--- Instantiate calibration/test"
+      - "julia --project=calibration/test -e 'using Pkg; Pkg.develop(;path=\".\"); Pkg.instantiate(;verbose=true)'"
+      - "julia --project=calibration/test -e 'using Pkg; Pkg.precompile()'"
+      - "julia --project=calibration/test -e 'using Pkg; Pkg.status()'"
+
+    agents:
+      slurm_cpus_per_task: 8
+      slurm_gpus: 1
+    env:
+      JULIA_NUM_PRECOMPILE_TASKS: 8
+      JULIA_MAX_NUM_PRECOMPILE_FILES: 50
+
+  - wait
+
+  - group: "Calibration"
+      steps:
+        - label: "Calibration interface unit tests"
+          command: julia --project=calibration/test calibration/test/interface.jl
+        - label: "end to end test"
+          command: julia --project=calibration/test calibration/test/e2e_test.jl
+          agents:
+            slurm_ntasks: 10
+            slurm_cpus_per_task: 1
+            slurm_mem: 96GB
+            slurm_reservation: "false"
+            slurm_time: "00:30:00"
+          artifact_paths: "calibration_end_to_end_test/*"
+
+  -group: "Benchmarks"
+    - label: ":computer: Benchmark: CPU perf target (Threaded)"
+        command: >
+          julia --color=yes --threads 8 --project=.buildkite perf/benchmark.jl
+          --config_file $PERF_CONFIG_PATH/bm_perf_target_threaded.yml
+          --job_id bm_perf_target_threaded
+        agents:
+          slurm_mem: 24GB
+          slurm_cpus_per_task: 8
+          slurm_reservation: "false"
+
+  -group: "Flame Graphs"
+    - label: ":fire: Flame graph: perf target (Threaded)"
+        command: >
+          julia --threads 8 --color=yes --project=.buildkite perf/flame.jl
+          --config_file $PERF_CONFIG_PATH/flame_perf_target_threaded.yml
+          --job_id flame_perf_target_threaded
+        artifact_paths: "flame_perf_target_threaded/*"
+        agents:
+          slurm_cpus_per_task: 8
+          slurm_mem: 24GB
+          slurm_reservation: "false"
+
+  - wait: ~
+    continue_on_failure: true