Reapply reproducibility improvements + unit tests

debug

reproducibility_tests/print_new_mse.jl -> mse_summary

.buildkite/pipeline.yml

@@ -56,11 +56,11 @@ steps:
 
   - wait
 
-  - group: "Reproducibility tests"
+  - group: "Reproducibility infrastructure"
     steps:
 
-      - label: ":computer: Ensure mse tables are reset when necessary"
-        command: "julia --color=yes --project=examples reproducibility_tests/test_reset.jl"
+      - label: ":computer: Test reproducibility infrastructure"
+        command: "julia --color=yes --project=examples test/unit_reproducibility_infra.jl"
 
   - group: "Radiation"
     steps:
@@ -1186,7 +1186,7 @@ steps:
     continue_on_failure: true
 
   - label: ":robot_face: Print new mse tables"
-    command: "julia --color=yes --project=examples reproducibility_tests/print_new_mse.jl"
+    command: "julia --color=yes --project=examples reproducibility_tests/mse_summary.jl"
 
   - label: ":robot_face: Print new reference counter"
     command: "julia --color=yes --project=examples reproducibility_tests/print_new_ref_counter.jl"

examples/hybrid/driver.jl

@@ -95,21 +95,14 @@ if config.parsed_args["reproducibility_test"]
             "..",
             "..",
             "reproducibility_tests",
-            "reproducibility_tests.jl",
+            "compute_mse.jl", # TODO: improve filename
         ),
     )
-    @testset "Test reproducibility table entries" begin
-        mse_keys = sort(collect(keys(all_best_mse[simulation.job_id])))
-        pcs = collect(Fields.property_chains(sol.u[end]))
-        for prop_chain in mse_keys
-            @test prop_chain in pcs
-        end
-    end
-    perform_reproducibility_tests(
-        simulation.job_id,
+    export_reproducibility_results(
         sol.u[end],
-        all_best_mse,
-        simulation.output_dir,
+        config.comms_ctx;
+        job_id = simulation.job_id,
+        computed_dir = simulation.output_dir,
     )
 end
 
@@ -152,7 +145,7 @@ if ClimaComms.iamroot(config.comms_ctx)
         ),
     )
     @info "Plotting"
-    paths = latest_comparable_paths() # __build__ path (not job path)
+    paths = latest_comparable_dirs() # __build__ path (not job path)
     if isempty(paths)
         make_plots(Val(Symbol(reference_job_id)), simulation.output_dir)
     else