diff --git a/benches/_memory.py b/benches/_memory.py index ffa22140..bd9cd1d6 100644 --- a/benches/_memory.py +++ b/benches/_memory.py @@ -94,6 +94,11 @@ def peak_rss_bytes() -> int: def reset_peak_rss() -> bool: """Reset ``VmHWM`` to the current RSS, starting a new measurement window. + This also resets ``getrusage(...).ru_maxrss`` (and so Julia's ``Sys.maxrss()``): both + report the same kernel field, ``mm->hiwater_rss``. Any process that opens a window + therefore loses ``ru_maxrss`` as a whole-run ceiling, and must take the maximum over + its windows instead. + Returns: ``True`` if the reset took effect, ``False`` where ``/proc/self/clear_refs`` is unavailable (non-Linux, kernel < 4.0, or a restricted sandbox), in which case diff --git a/benches/third_party/README.md b/benches/third_party/README.md index bf0a1e65..baf90642 100644 --- a/benches/third_party/README.md +++ b/benches/third_party/README.md @@ -70,10 +70,14 @@ instead of squares with `--ny`. Two things to know when reading the output: -- **The memory column is not one quantity.** Each backend reports what it exposes — - monoprop and cuPauliProp their own operator footprint, `PauliPropagation.jl` - `Base.summarysize`, and ppvm and Qiskit only a process-RSS proxy. `MEMORY_METRICS` in - `backends.py` records which is which, and every record also carries `peak_rss_MB`. +- **The memory column is one quantity for every backend**: the peak resident set size over + each step, read from the kernel's `VmHWM` high-water mark and reset per step, so the + curves may be compared directly. Where a library also accounts for itself, that figure is + kept separately as `operator_memory_MB` / the `native_memory` series — those are *not* + commensurable across backends (one counts an operator, another an object graph, another a + device pool), so never plot them against each other. `HOST_MEMORY_METRIC` and + `OPERATOR_MEMORY_METRICS` in `backends.py` record which is which, and every record also + carries `peak_rss_MB` for the process lifetime. - **`PauliPropagation.jl` runs in its fastest documented configuration**, which is not its default: the `VectorPauliSum` container driven by `Performance.propagate!`, with coefficient truncation on. That combination needs the **dev branch (0.8.0)** — earlier diff --git a/benches/third_party/bench_common.jl b/benches/third_party/bench_common.jl new file mode 100644 index 00000000..d3c39475 --- /dev/null +++ b/benches/third_party/bench_common.jl @@ -0,0 +1,116 @@ +# Copyright 2026 Algorithmiq +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Shared across the third-party Julia benchmark scripts in this project (pauli_prop, +# majorana_prop). Included via `include(joinpath(@__DIR__, "..", "bench_common.jl"))`. +# +# Line-for-line mirror of `benches/_memory.py`. The two must stay in step: a cross-language +# memory comparison is only meaningful if both sides measure the same quantity the same way, +# down to how the process is settled before the window opens. + +# Reset VmHWM to the current RSS, starting a new measurement window +const _CLEAR_REFS_MM_HIWATER_RSS = "5\n" + +"""Return a `/proc/self` size field (kB -> bytes); 0 if unavailable.""" +function proc_field(path::AbstractString, key::AbstractString)::Int + try + for line in eachline(path) + if startswith(line, key) + return parse(Int, split(line)[2]) * 1024 # values are in kB + end + end + catch + return 0 # non-Linux or restricted /proc + end + return 0 +end + +"""Return this process's current resident set size (RSS) in bytes.""" +rss_bytes()::Int = proc_field("/proc/self/status", "VmRSS:") + +"""Return the kernel's high-water mark of this process's RSS, in bytes. + +VmHWM is maintained by the kernel on every RSS increase, so it is exact: unlike a polling +sampler it cannot miss a transient that is allocated and freed between two observations. +""" +peak_rss_bytes()::Int = proc_field("/proc/self/status", "VmHWM:") + +"""Reset VmHWM to the current RSS, starting a new measurement window. + +Returns `false` where `/proc/self/clear_refs` is unavailable (non-Linux, kernel < 4.0, or a +restricted sandbox), in which case VmHWM keeps counting from process start and callers must +fall back. +""" +function reset_peak_rss()::Bool + try + write("/proc/self/clear_refs", _CLEAR_REFS_MM_HIWATER_RSS) + catch + return false + end + return true +end + +"""Ask the C allocator to return unused heap pages to the OS.""" +function heap_trim() + try + ccall((:malloc_trim, "libc"), Cint, (Csize_t,), 0) + catch + nothing # unsupported platform / allocator + end + return nothing +end + +"""Return current RSS after collecting garbage and trimming the C heap.""" +function resting_rss_bytes()::Int + GC.gc() + heap_trim() + return rss_bytes() +end + +"""Exact peak RSS over a measurement window, straight from the kernel. + +`start!` settles the process (`GC.gc()` + `malloc_trim`) and resets VmHWM to that floor, so +the peak reported is this window's own and not an earlier window's retained garbage. The +settling is what makes the number comparable against the Python side: without it the figure +tracks the GC's willingness to return pages more than what the code needed. + +`exact` is `false` when the kernel would not reset the window (see `reset_peak_rss`); the +peak then degrades to the RSS observed at `stop!`, which is a lower bound. +""" +mutable struct HighWaterMark + baseline_bytes::Int + peak_bytes::Int + exact::Bool +end + +HighWaterMark() = HighWaterMark(0, 0, false) + +"""Open the window: settle, reset VmHWM, and record the floor.""" +function start!(hwm::HighWaterMark) + hwm.baseline_bytes = resting_rss_bytes() + hwm.exact = reset_peak_rss() + hwm.peak_bytes = hwm.baseline_bytes + return hwm +end + +"""Close the window and latch the peak.""" +function stop!(hwm::HighWaterMark) + observed = hwm.exact ? peak_rss_bytes() : rss_bytes() + hwm.peak_bytes = max(hwm.baseline_bytes, observed) + return hwm +end + +peak_mb(hwm::HighWaterMark) = hwm.peak_bytes / 1024^2 +baseline_mb(hwm::HighWaterMark) = hwm.baseline_bytes / 1024^2 +delta_mb(hwm::HighWaterMark) = (hwm.peak_bytes - hwm.baseline_bytes) / 1024^2 diff --git a/benches/third_party/majorana_prop/julia_hubbard1d_benchmark.jl b/benches/third_party/majorana_prop/julia_hubbard1d_benchmark.jl index b985b57b..1c5ae010 100644 --- a/benches/third_party/majorana_prop/julia_hubbard1d_benchmark.jl +++ b/benches/third_party/majorana_prop/julia_hubbard1d_benchmark.jl @@ -18,6 +18,8 @@ using BenchmarkTools using ArgParse using JSON +include(joinpath(@__DIR__, "..", "bench_common.jl")) + """CPU seconds (user + system) consumed by this process so far, summed over all threads. @@ -30,17 +32,6 @@ function process_cpu_seconds() end -"""Peak resident set size in MB, from VmHWM in /proc/self/status.""" -function peak_rss_mb() - for line in eachline("/proc/self/status") - if startswith(line, "VmHWM:") - return parse(Int, split(line)[2]) / 1024 - end - end - return NaN -end - - function experiment(N_spinful_sites, fock_state, circ_single, thetas_single, n_layers) site_index = N_spinful_sites ÷ 2 @@ -49,54 +40,91 @@ function experiment(N_spinful_sites, fock_state, circ_single, thetas_single, n_l obs = VectorMajoranaSum(MajoranaSum(N_spinful_sites, :nup, site_index)) - res = zeros(n_layers + 1) - res[1] = overlapwithfock(obs, fock_state) - - - cpu_start = process_cpu_seconds() - stats = @timed for k = 1:n_layers - propagate!(circ_single, obs, thetas_single, min_abs_coeff=min_abs_coeff, max_unpaired=max_unpaired) - res[k+1] = overlapwithfock(obs, fock_state) + values = zeros(n_layers + 1) + term_counts = zeros(Int, n_layers + 1) + cumulative_runtimes = zeros(n_layers + 1) + memory_size = zeros(n_layers + 1) + native_memory_size = zeros(n_layers + 1) + + sampler = HighWaterMark() + + # Accumulated across the timed regions only: opening a memory window forces a full GC, + # so a counter spanning the whole loop would charge that settling cost to the workload + # and inflate both gc_seconds and busy_cores. + cpu_seconds = 0.0 + gc_ns = 0 + + start!(sampler) + cpu_mark, gc_mark = process_cpu_seconds(), Base.gc_time_ns() + cumulative_runtimes[1] = @elapsed (values[1] = overlapwithfock(obs, fock_state)) + cpu_seconds += process_cpu_seconds() - cpu_mark + gc_ns += Base.gc_time_ns() - gc_mark + stop!(sampler) + term_counts[1] = length(obs) + memory_size[1] = peak_mb(sampler) + native_memory_size[1] = Base.summarysize(obs) / 1024^2 + for k = 1:n_layers + start!(sampler) + cpu_mark, gc_mark = process_cpu_seconds(), Base.gc_time_ns() + step_runtime = @elapsed propagate!(circ_single, obs, thetas_single, min_abs_coeff=min_abs_coeff, max_unpaired=max_unpaired) + cpu_seconds += process_cpu_seconds() - cpu_mark + gc_ns += Base.gc_time_ns() - gc_mark + stop!(sampler) + values[k+1] = overlapwithfock(obs, fock_state) + term_counts[k+1] = length(obs) + cumulative_runtimes[k+1] = cumulative_runtimes[k] + step_runtime + memory_size[k+1] = peak_mb(sampler) + native_memory_size[k+1] = Base.summarysize(obs) / 1024^2 end - cpu_seconds = process_cpu_seconds() - cpu_start - memory_size = Base.summarysize(obs) / 1024^2 - return ( - res=res, - num_terms=length(obs), - runtime_seconds=stats.time, - gc_seconds=stats.gctime, - cpu_seconds=cpu_seconds, - memory_MB=memory_size, + + total_runtime = cumulative_runtimes[end] + provenance = Dict( + "cpu_seconds" => cpu_seconds, + "gc_seconds" => gc_ns / 1e9, + "busy_cores" => total_runtime > 0 ? cpu_seconds / total_runtime : NaN, # Recorded, not hardcoded: only the Vector container dispatches into the # AcceleratedKernels path, so this is what proves the run was threaded at all. - container=string(nameof(typeof(obs))), - ) - - -end -function save_result(output_path, N_spinful_sites, n_layers, result) - """Append one benchmark result as a JSON line, creating the parent directory if needed.""" - record = Dict( - "n_spinful_sites" => N_spinful_sites, - "n_layers" => n_layers, - "num_terms" => result.num_terms, - "final_overlap" => result.res[end], - "runtime_seconds" => result.runtime_seconds, - "cpu_seconds" => result.cpu_seconds, - "busy_cores" => result.cpu_seconds / result.runtime_seconds, - "gc_seconds" => result.gc_seconds, - "num_threads" => Threads.nthreads(), - "container" => result.container, - "memory_MB" => result.memory_MB, - "peak_rss_MB" => peak_rss_mb(), + "container" => string(nameof(typeof(obs))), "library_version" => string(pkgversion(MajoranaPropagation)), "pauliprop_version" => string(pkgversion(PauliPropagation)), "host" => gethostname(), ) - open(output_path, "a") do io - JSON.print(io, record) - println(io) + return values, term_counts, cumulative_runtimes, memory_size, native_memory_size, provenance + + +end +function save_result(output_path, source, N_spinful_sites, n_layers, term_counts, values, cumulative_runtimes, memory_size, native_memory_size, num_threads, provenance) + """Merge this run's per-step data into the shared results JSON file, keyed by source label.""" + data = if isfile(output_path) + JSON.parsefile(output_path) + else + Dict( + "n_spinful_sites" => N_spinful_sites, + "n_layers" => n_layers, + "step_range" => collect(0:n_layers), + "num_threads" => Dict(), + "runtime_seconds" => Dict(), + "expectation_value" => Dict(), + "num_terms" => Dict(), + "memory_MB" => Dict(), + "native_memory_MB" => Dict(), + ) + end + data["num_threads"] = get(data, "num_threads", Dict()) + data["num_threads"][source] = num_threads + data["runtime_seconds"][source] = cumulative_runtimes + data["expectation_value"][source] = values + data["num_terms"][source] = term_counts + data["memory_MB"][source] = memory_size + data["native_memory_MB"] = get(data, "native_memory_MB", Dict()) + data["native_memory_MB"][source] = native_memory_size + data["provenance"] = get(data, "provenance", Dict()) + data["provenance"][source] = provenance + + mkpath(dirname(output_path)) + open(output_path, "w") do io + JSON.print(io, data, 4) end end @@ -105,28 +133,27 @@ function main(args) s = ArgParseSettings(description="Arguments for the 1D Hubbard model benchmark.") @add_arg_table! s begin - "--case", "-c" - help = "Case pair to run." + "--n-spins", "-n" + help = "Number of spinful sites." + arg_type = Int + default = 60 + dest_name = "n_spins" + "--max-layers", "-l" + help = "Number of Trotter layers." arg_type = Int - default = 1 + default = 20 + dest_name = "max_layers" "--output", "-o" - help = "Path to the JSONL file results are appended to." + help = "Path to the shared JSON file results are merged into." arg_type = String - default = joinpath(@__DIR__, "julia_hubbard1d_benchmark_results.jsonl") + default = joinpath(@__DIR__, "results.json") end parsed_args = parse_args(s) - spin_layers_pairs = [] - for i in [20, 40, 60] - for j in 10:2:18 - push!(spin_layers_pairs, (i, j)) - end - end - - case_pair = parsed_args["case"] - N_spinful_sites, n_layers = spin_layers_pairs[case_pair] + N_spinful_sites = parsed_args["n_spins"] + n_layers = parsed_args["max_layers"] t = 1. U = 1.5 @@ -161,12 +188,11 @@ function main(args) println("Number of threads: $(Threads.nthreads())") - result = experiment(N_spinful_sites, fock_state, circ_single, thetas_single, n_layers) - busy_cores = result.cpu_seconds / result.runtime_seconds - println("$N_spinful_sites n_spin $n_layers layers $(result.num_terms) num_terms $(result.res[end]) final overlap $(result.runtime_seconds) seconds") - println("container $(result.container) cpu $(round(result.cpu_seconds, digits=1)) s busy_cores $(round(busy_cores, digits=2)) gc $(round(result.gc_seconds, digits=1)) s") + values, term_counts, cumulative_runtimes, memory_size, native_memory_size, provenance = experiment(N_spinful_sites, fock_state, circ_single, thetas_single, n_layers) + println("$N_spinful_sites n_spin $n_layers layers $(term_counts[end]) num_terms $(values[end]) final overlap $(cumulative_runtimes[end]) seconds") + println("container $(provenance["container"]) cpu $(round(provenance["cpu_seconds"], digits=1)) s busy_cores $(round(provenance["busy_cores"], digits=2)) gc $(round(provenance["gc_seconds"], digits=1)) s") - save_result(parsed_args["output"], N_spinful_sites, n_layers, result) + save_result(parsed_args["output"], "MajoranaPropagation.jl", N_spinful_sites, n_layers, term_counts, values, cumulative_runtimes, memory_size, native_memory_size, Threads.nthreads(), provenance) end diff --git a/benches/third_party/majorana_prop/julia_hubbard1d_benchmark_results.jsonl b/benches/third_party/majorana_prop/julia_hubbard1d_benchmark_results.jsonl deleted file mode 100644 index af620c97..00000000 --- a/benches/third_party/majorana_prop/julia_hubbard1d_benchmark_results.jsonl +++ /dev/null @@ -1,15 +0,0 @@ -{"gc_seconds":0.362910689,"pauliprop_version":"0.7.3","final_overlap":0.5540635634956823,"runtime_seconds":2.917830575,"n_spinful_sites":20,"container":"VectorMajoranaSum","num_threads":28,"n_layers":10,"peak_rss_MB":575.765625,"num_terms":597051,"cpu_seconds":60.19,"library_version":"0.3.0","host":"lrdn4988.leonardo.local","busy_cores":20.62833960124638,"memory_MB":54.33390808105469} -{"gc_seconds":0.578300529,"pauliprop_version":"0.7.3","final_overlap":0.6265984784190225,"runtime_seconds":5.556186726,"n_spinful_sites":20,"container":"VectorMajoranaSum","num_threads":28,"n_layers":12,"peak_rss_MB":717.9921875,"num_terms":1754896,"cpu_seconds":103.85,"library_version":"0.3.0","host":"lrdn4988.leonardo.local","busy_cores":18.69087651680913,"memory_MB":109.1256332397461} -{"gc_seconds":1.276975719,"pauliprop_version":"0.7.3","final_overlap":0.646851959851176,"runtime_seconds":17.935291296,"n_spinful_sites":20,"container":"VectorMajoranaSum","num_threads":28,"n_layers":14,"peak_rss_MB":1111.484375,"num_terms":4870330,"cpu_seconds":320.28999999999996,"library_version":"0.3.0","host":"lrdn4988.leonardo.local","busy_cores":17.858087427408126,"memory_MB":438.11207580566406} -{"gc_seconds":2.693645094,"pauliprop_version":"0.7.3","final_overlap":0.6237908253934835,"runtime_seconds":59.278095613,"n_spinful_sites":20,"container":"VectorMajoranaSum","num_threads":28,"n_layers":16,"peak_rss_MB":2096.875,"num_terms":12875636,"cpu_seconds":1099.3999999999999,"library_version":"0.3.0","host":"lrdn4988.leonardo.local","busy_cores":18.54647975160146,"memory_MB":962.4855346679688} -{"gc_seconds":6.055282547,"pauliprop_version":"0.7.3","final_overlap":0.5748922977223003,"runtime_seconds":174.706799082,"n_spinful_sites":20,"container":"VectorMajoranaSum","num_threads":28,"n_layers":18,"peak_rss_MB":4697.16015625,"num_terms":32668247,"cpu_seconds":3263.1200000000003,"library_version":"0.3.0","host":"lrdn4988.leonardo.local","busy_cores":18.67769323887864,"memory_MB":2034.1456985473633} -{"gc_seconds":0.470775403,"pauliprop_version":"0.7.3","final_overlap":0.5540635642561327,"runtime_seconds":5.248756053,"n_spinful_sites":40,"container":"VectorMajoranaSum","num_threads":28,"n_layers":10,"peak_rss_MB":585.19921875,"num_terms":597311,"cpu_seconds":114.82000000000001,"library_version":"0.3.0","host":"lrdn4988.leonardo.local","busy_cores":21.87565945922997,"memory_MB":72.44517517089844} -{"gc_seconds":0.817695282,"pauliprop_version":"0.7.3","final_overlap":0.6265984871125446,"runtime_seconds":10.911880258,"n_spinful_sites":40,"container":"VectorMajoranaSum","num_threads":28,"n_layers":12,"peak_rss_MB":806.62109375,"num_terms":1754526,"cpu_seconds":214.44,"library_version":"0.3.0","host":"lrdn4988.leonardo.local","busy_cores":19.65197518024304,"memory_MB":145.5008087158203} -{"gc_seconds":1.683642663,"pauliprop_version":"0.7.3","final_overlap":0.6468519627999069,"runtime_seconds":36.575411074,"n_spinful_sites":40,"container":"VectorMajoranaSum","num_threads":28,"n_layers":14,"peak_rss_MB":1256.0625,"num_terms":4867455,"cpu_seconds":690.2,"library_version":"0.3.0","host":"lrdn4988.leonardo.local","busy_cores":18.870601306532837,"memory_MB":584.1493988037109} -{"gc_seconds":4.033128591,"pauliprop_version":"0.7.3","final_overlap":0.6237907699662404,"runtime_seconds":121.334406253,"n_spinful_sites":40,"container":"VectorMajoranaSum","num_threads":28,"n_layers":16,"peak_rss_MB":2595.28515625,"num_terms":12869611,"cpu_seconds":2328.31,"library_version":"0.3.0","host":"lrdn4988.leonardo.local","busy_cores":19.189198446689,"memory_MB":1283.3140106201172} -{"gc_seconds":9.323305056,"pauliprop_version":"0.7.3","final_overlap":0.5748922102989575,"runtime_seconds":359.845444315,"n_spinful_sites":40,"container":"VectorMajoranaSum","num_threads":28,"n_layers":18,"peak_rss_MB":5989.84765625,"num_terms":32655012,"cpu_seconds":6958.17,"library_version":"0.3.0","host":"lrdn4988.leonardo.local","busy_cores":19.33655159438113,"memory_MB":2712.1942291259766} -{"gc_seconds":0.576368469,"pauliprop_version":"0.7.3","final_overlap":0.5540635608915114,"runtime_seconds":7.755988538,"n_spinful_sites":60,"container":"VectorMajoranaSum","num_threads":28,"n_layers":10,"peak_rss_MB":629.94140625,"num_terms":597601,"cpu_seconds":173.48000000000002,"library_version":"0.3.0","host":"lrdn4988.leonardo.local","busy_cores":22.367232642240918,"memory_MB":90.55644226074219} -{"gc_seconds":0.915065652,"pauliprop_version":"0.7.3","final_overlap":0.6265984814098748,"runtime_seconds":17.317334301,"n_spinful_sites":60,"container":"VectorMajoranaSum","num_threads":28,"n_layers":12,"peak_rss_MB":829.578125,"num_terms":1754476,"cpu_seconds":347.95,"library_version":"0.3.0","host":"lrdn4988.leonardo.local","busy_cores":20.092584340761235,"memory_MB":181.87598419189453} -{"gc_seconds":1.894691961,"pauliprop_version":"0.7.3","final_overlap":0.6468519475075789,"runtime_seconds":60.244685642,"n_spinful_sites":60,"container":"VectorMajoranaSum","num_threads":28,"n_layers":14,"peak_rss_MB":1420.40234375,"num_terms":4865089,"cpu_seconds":1204.2900000000002,"library_version":"0.3.0","host":"lrdn4988.leonardo.local","busy_cores":19.989978985970858,"memory_MB":730.1867218017578} -{"gc_seconds":5.113325333,"pauliprop_version":"0.7.3","final_overlap":0.6237907501255578,"runtime_seconds":203.050861287,"n_spinful_sites":60,"container":"VectorMajoranaSum","num_threads":28,"n_layers":16,"peak_rss_MB":3089.06640625,"num_terms":12859755,"cpu_seconds":4054.7400000000002,"library_version":"0.3.0","host":"lrdn4988.leonardo.local","busy_cores":19.969085451299183,"memory_MB":1604.1424865722656} -{"gc_seconds":12.807711221,"pauliprop_version":"0.7.3","final_overlap":0.5748921471542362,"runtime_seconds":591.766174546,"n_spinful_sites":60,"container":"VectorMajoranaSum","num_threads":28,"n_layers":18,"peak_rss_MB":7273.3828125,"num_terms":32625568,"cpu_seconds":11814.2,"library_version":"0.3.0","host":"lrdn4988.leonardo.local","busy_cores":19.96430432858687,"memory_MB":3390.24275970459} diff --git a/benches/third_party/majorana_prop/monoprop_hubbard1d_benchmark.py b/benches/third_party/majorana_prop/monoprop_hubbard1d_benchmark.py index c0d60661..e4748b1d 100644 --- a/benches/third_party/majorana_prop/monoprop_hubbard1d_benchmark.py +++ b/benches/third_party/majorana_prop/monoprop_hubbard1d_benchmark.py @@ -19,6 +19,7 @@ import os import platform import resource +import sys from pathlib import Path from time import perf_counter @@ -27,7 +28,10 @@ from monoprop import Circuit, ExpGate, MajoranaPropagator from monoprop.fermi import FermiOperator -os.environ["YAQS_LOG_LEVEL"] = "INFO" +# The repository's own benchmark suite owns the memory instrumentation; this directory is a +# separate uv project, so reach it by path rather than by dependency. +sys.path.insert(0, str(Path(__file__).resolve().parents[2])) +from _memory import HighWaterMark # noqa: E402 def mode(site, spin): @@ -59,13 +63,19 @@ def hubbard_fermion_terms( """ terms = [] topology = bricklayer_topology(num_sites) + # Spin-up and spin-down modes are interleaved, so a spinful chain spans 2 * num_sites modes. + num_modes = 2 * num_sites for spin in ("up", "down"): for left_site, right_site in topology: left, right = mode(left_site, spin), mode(right_site, spin) op_terms = [((left, "+"), (right, "-")), ((right, "+"), (left, "-"))] terms.append( - FermiOperator(terms=op_terms, coefficients=[-hopping, -hopping]) + FermiOperator( + terms=op_terms, + coefficients=[-hopping, -hopping], + num_modes=num_modes, + ) ) for site in range(num_sites): @@ -74,6 +84,7 @@ def hubbard_fermion_terms( FermiOperator( terms=[((up, "+"), (up, "-"), (down, "+"), (down, "-"))], coefficients=[interaction], + num_modes=num_modes, ) ) @@ -85,6 +96,7 @@ def hubbard_fermion_terms( FermiOperator( terms=[((m, "+"), (m, "-"))], coefficients=[-chemical_potential], + num_modes=num_modes, ) ) @@ -117,28 +129,71 @@ def number_operator_majorana(site, spin, num_qubits): ) +SOURCE_LABEL = "monoprop" + + def process_cpu_seconds(): """Return CPU seconds (user + system) consumed by this process, summed over all threads.""" usage = resource.getrusage(resource.RUSAGE_SELF) return usage.ru_utime + usage.ru_stime -def save_result(output_path, record): - """Append one benchmark result as a JSON line, creating the parent directory if needed.""" +def save_result( + output_path, + n_spinful_sites, + n_layers, + values, + term_counts, + cumulative_runtimes, + memory_size, + native_memory_size, + provenance, +): + """Merge this run's per-step data into the shared results JSON file, keyed by source label.""" output_path = Path(output_path) output_path.parent.mkdir(parents=True, exist_ok=True) - with output_path.open("a") as f: - f.write(json.dumps(record) + "\n") + if output_path.exists(): + with output_path.open() as f: + data = json.load(f) + else: + data = { + "n_spinful_sites": n_spinful_sites, + "n_layers": n_layers, + "step_range": list(range(n_layers + 1)), + "num_threads": {}, + "runtime_seconds": {}, + "expectation_value": {}, + "num_terms": {}, + "memory_MB": {}, + "native_memory_MB": {}, + } + requested_threads = os.environ.get("monoprop_NUM_THREADS") + data.setdefault("num_threads", {})[SOURCE_LABEL] = ( + int(requested_threads) if requested_threads else None + ) + data["runtime_seconds"][SOURCE_LABEL] = cumulative_runtimes + data["expectation_value"][SOURCE_LABEL] = values + data["num_terms"][SOURCE_LABEL] = term_counts + data["memory_MB"][SOURCE_LABEL] = memory_size + data.setdefault("native_memory_MB", {})[SOURCE_LABEL] = native_memory_size + data.setdefault("provenance", {})[SOURCE_LABEL] = provenance + with output_path.open("w") as f: + json.dump(data, f, indent=4) def main(): parser = argparse.ArgumentParser(description="Benchmark for 1D Hubbard model") - parser.add_argument("--case", "-c", help="Case pair to run.", type=int, default=0) + parser.add_argument( + "--n-spins", "-n", help="Number of spinful sites.", type=int, default=60 + ) + parser.add_argument( + "--max-layers", "-l", help="Number of Trotter layers.", type=int, default=20 + ) parser.add_argument( "--output", "-o", - help="Path to the JSONL file results are appended to.", - default=Path(__file__).with_name("monoprop_hubbard1d_benchmark_results.jsonl"), + help="Path to the shared JSON file results are merged into.", + default=Path(__file__).with_name("results.json"), ) parser.add_argument( "--mu-gates", @@ -148,13 +203,7 @@ def main(): args = parser.parse_args() - spin_layer_cases = [] - for i in [20, 40, 60]: - for j in range(10, 19, 2): - spin_layer_cases.append((i, j)) - - case_pair = args.case - n_spinful_sites, n_layers = spin_layer_cases[case_pair] + n_spinful_sites, n_layers = args.n_spins, args.max_layers trotter_steps = n_layers t = 1.0 u = 1.5 @@ -192,23 +241,35 @@ def main(): values = np.empty(trotter_steps + 1) term_counts = np.empty(trotter_steps + 1, dtype=int) - - values[0] = simulator.expectation_value() - term_counts[0] = simulator.size() + cumulative_runtimes = np.empty(trotter_steps + 1) + memory_size = np.empty(trotter_steps + 1) + native_memory_size = np.empty(trotter_steps + 1) cpu_start = process_cpu_seconds() - t_start = perf_counter() + with HighWaterMark() as window: + t_start = perf_counter() + values[0] = simulator.expectation_value() + cumulative_runtimes[0] = perf_counter() - t_start + term_counts[0] = simulator.size() + memory_size[0] = window.peak_mb + native_memory_size[0] = simulator._simulator.operator_memory_bytes() / 1024**2 for step in range(trotter_steps): - simulator.propagate(fermi_circuit) - values[step + 1] = simulator.expectation_value() - term_counts[step + 1] = simulator.size() - t_total = perf_counter() - t_start + with HighWaterMark() as window: + step_start = perf_counter() + simulator.propagate(fermi_circuit) + step_runtime = perf_counter() - step_start + values[step + 1] = simulator.expectation_value() + term_counts[step + 1] = simulator.size() + cumulative_runtimes[step + 1] = cumulative_runtimes[step] + step_runtime + memory_size[step + 1] = window.peak_mb + native_memory_size[step + 1] = ( + simulator._simulator.operator_memory_bytes() / 1024**2 + ) cpu_total = process_cpu_seconds() - cpu_start - memory_size = simulator._simulator.operator_memory_bytes() / 1024**2 - peak_rss_mb = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1024 + # The engine picks one partition per core when the env var is unset, so an unset value is # not "1 thread"; busy_cores is the only measurement of what the threads actually did. - requested_threads = os.environ.get("monoprop_NUM_THREADS") + t_total = cumulative_runtimes[-1] busy_cores = cpu_total / t_total if t_total > 0 else float("nan") print( f"{n_spinful_sites} n_spin {n_layers} layers {term_counts[-1]} num_terms {values[-1]} final overlap runtime {t_total:.3f} seconds" @@ -216,20 +277,18 @@ def main(): ) save_result( args.output, + n_spinful_sites, + n_layers, + values.tolist(), + term_counts.tolist(), + cumulative_runtimes.tolist(), + memory_size.tolist(), + native_memory_size.tolist(), { - "n_spinful_sites": n_spinful_sites, - "n_layers": n_layers, - "num_threads": int(requested_threads) if requested_threads else None, "affinity_cores": len(os.sched_getaffinity(0)), "mu_gates": bool(args.mu_gates), - "runtime_seconds": t_total, "cpu_seconds": cpu_total, "busy_cores": busy_cores, - "final_overlap": float(values[-1]), - # np.int64 is not a subclass of int, so json.dumps rejects it as-is. - "num_terms": int(term_counts[-1]), - "memory_MB": memory_size, - "peak_rss_MB": peak_rss_mb, "library_version": monoprop.__version__, "host": platform.node(), }, diff --git a/benches/third_party/majorana_prop/monoprop_hubbard1d_benchmark_results.jsonl b/benches/third_party/majorana_prop/monoprop_hubbard1d_benchmark_results.jsonl deleted file mode 100644 index 6fa912bd..00000000 --- a/benches/third_party/majorana_prop/monoprop_hubbard1d_benchmark_results.jsonl +++ /dev/null @@ -1,15 +0,0 @@ -{"n_spinful_sites": 20, "n_layers": 10, "num_threads": 56, "affinity_cores": 112, "mu_gates": false, "runtime_seconds": 0.1330429679946974, "cpu_seconds": 7.082302, "busy_cores": 53.23319305596276, "final_overlap": 0.5540634920911529, "num_terms": 872870, "memory_MB": 46.74619007110596, "peak_rss_MB": 146.51171875, "library_version": "0.7.0a3.dev18+gea2a299e8.d20260730", "host": "lrdn4990.leonardo.local"} -{"n_spinful_sites": 20, "n_layers": 12, "num_threads": 56, "affinity_cores": 112, "mu_gates": false, "runtime_seconds": 0.22318566299509257, "cpu_seconds": 11.936139, "busy_cores": 53.48076054626526, "final_overlap": 0.6265985937915882, "num_terms": 2606766, "memory_MB": 163.49968814849854, "peak_rss_MB": 355.4765625, "library_version": "0.7.0a3.dev18+gea2a299e8.d20260730", "host": "lrdn4990.leonardo.local"} -{"n_spinful_sites": 20, "n_layers": 14, "num_threads": 56, "affinity_cores": 112, "mu_gates": false, "runtime_seconds": 0.450876449001953, "cpu_seconds": 24.17238, "busy_cores": 53.611981848923975, "final_overlap": 0.6468517510125221, "num_terms": 7276411, "memory_MB": 374.82527446746826, "peak_rss_MB": 763.83984375, "library_version": "0.7.0a3.dev18+gea2a299e8.d20260730", "host": "lrdn4990.leonardo.local"} -{"n_spinful_sites": 20, "n_layers": 16, "num_threads": 56, "affinity_cores": 112, "mu_gates": false, "runtime_seconds": 1.2240017639705911, "cpu_seconds": 65.882097, "busy_cores": 53.82516507679064, "final_overlap": 0.6237899786874563, "num_terms": 19078804, "memory_MB": 1049.5043535232544, "peak_rss_MB": 1528.4140625, "library_version": "0.7.0a3.dev18+gea2a299e8.d20260730", "host": "lrdn4990.leonardo.local"} -{"n_spinful_sites": 20, "n_layers": 18, "num_threads": 56, "affinity_cores": 112, "mu_gates": false, "runtime_seconds": 3.524797510006465, "cpu_seconds": 190.142814, "busy_cores": 53.944322605825725, "final_overlap": 0.5748905367524686, "num_terms": 47121859, "memory_MB": 2843.4690923690796, "peak_rss_MB": 3813.73046875, "library_version": "0.7.0a3.dev18+gea2a299e8.d20260730", "host": "lrdn4990.leonardo.local"} -{"n_spinful_sites": 40, "n_layers": 10, "num_threads": 56, "affinity_cores": 112, "mu_gates": false, "runtime_seconds": 0.1961011840030551, "cpu_seconds": 10.432621, "busy_cores": 53.20019383379892, "final_overlap": 0.5540634920911527, "num_terms": 872870, "memory_MB": 46.926506996154785, "peak_rss_MB": 147.79296875, "library_version": "0.7.0a3.dev18+gea2a299e8.d20260730", "host": "lrdn4990.leonardo.local"} -{"n_spinful_sites": 40, "n_layers": 12, "num_threads": 56, "affinity_cores": 112, "mu_gates": false, "runtime_seconds": 0.30405179306399077, "cpu_seconds": 16.175315, "busy_cores": 53.19920937481774, "final_overlap": 0.6265985937915881, "num_terms": 2606766, "memory_MB": 162.66238117218018, "peak_rss_MB": 370.16796875, "library_version": "0.7.0a3.dev18+gea2a299e8.d20260730", "host": "lrdn4990.leonardo.local"} -{"n_spinful_sites": 40, "n_layers": 14, "num_threads": 56, "affinity_cores": 112, "mu_gates": false, "runtime_seconds": 0.5518220340600237, "cpu_seconds": 29.564704999999996, "busy_cores": 53.57652136954019, "final_overlap": 0.646851751012522, "num_terms": 7276415, "memory_MB": 376.64658069610596, "peak_rss_MB": 770.14453125, "library_version": "0.7.0a3.dev18+gea2a299e8.d20260730", "host": "lrdn4990.leonardo.local"} -{"n_spinful_sites": 40, "n_layers": 16, "num_threads": 56, "affinity_cores": 112, "mu_gates": false, "runtime_seconds": 1.4028181759640574, "cpu_seconds": 75.353905, "busy_cores": 53.716088293634066, "final_overlap": 0.6237899786874558, "num_terms": 19078830, "memory_MB": 1042.8877954483032, "peak_rss_MB": 1562.70703125, "library_version": "0.7.0a3.dev18+gea2a299e8.d20260730", "host": "lrdn4990.leonardo.local"} -{"n_spinful_sites": 40, "n_layers": 18, "num_threads": 56, "affinity_cores": 112, "mu_gates": false, "runtime_seconds": 3.910900853923522, "cpu_seconds": 212.18199900000002, "busy_cores": 54.253995927085, "final_overlap": 0.5748905367524423, "num_terms": 47122113, "memory_MB": 2852.051636695862, "peak_rss_MB": 3901.05859375, "library_version": "0.7.0a3.dev18+gea2a299e8.d20260730", "host": "lrdn4990.leonardo.local"} -{"n_spinful_sites": 60, "n_layers": 10, "num_threads": 56, "affinity_cores": 112, "mu_gates": false, "runtime_seconds": 0.2506380130071193, "cpu_seconds": 13.362153, "busy_cores": 53.312555584377584, "final_overlap": 0.5540634920911527, "num_terms": 872870, "memory_MB": 46.64487934112549, "peak_rss_MB": 146.53515625, "library_version": "0.7.0a3.dev18+gea2a299e8.d20260730", "host": "lrdn4990.leonardo.local"} -{"n_spinful_sites": 60, "n_layers": 12, "num_threads": 56, "affinity_cores": 112, "mu_gates": false, "runtime_seconds": 0.37839777499902993, "cpu_seconds": 20.248353, "busy_cores": 53.51076126188086, "final_overlap": 0.6265985937915881, "num_terms": 2606766, "memory_MB": 164.04820346832275, "peak_rss_MB": 383.0859375, "library_version": "0.7.0a3.dev18+gea2a299e8.d20260730", "host": "lrdn4990.leonardo.local"} -{"n_spinful_sites": 60, "n_layers": 14, "num_threads": 56, "affinity_cores": 112, "mu_gates": false, "runtime_seconds": 0.6422348939813673, "cpu_seconds": 34.58507, "busy_cores": 53.851122578530266, "final_overlap": 0.646851751012522, "num_terms": 7276415, "memory_MB": 375.04934215545654, "peak_rss_MB": 768.890625, "library_version": "0.7.0a3.dev18+gea2a299e8.d20260730", "host": "lrdn4990.leonardo.local"} -{"n_spinful_sites": 60, "n_layers": 16, "num_threads": 56, "affinity_cores": 112, "mu_gates": false, "runtime_seconds": 1.546953356009908, "cpu_seconds": 83.49753199999999, "busy_cores": 53.97546841061005, "final_overlap": 0.6237899786874558, "num_terms": 19078830, "memory_MB": 1053.1319017410278, "peak_rss_MB": 1580.7265625, "library_version": "0.7.0a3.dev18+gea2a299e8.d20260730", "host": "lrdn4990.leonardo.local"} -{"n_spinful_sites": 60, "n_layers": 18, "num_threads": 56, "affinity_cores": 112, "mu_gates": false, "runtime_seconds": 4.179975813953206, "cpu_seconds": 227.321734, "busy_cores": 54.38350462248508, "final_overlap": 0.5748905367524423, "num_terms": 47122113, "memory_MB": 2834.194327354431, "peak_rss_MB": 3937.09375, "library_version": "0.7.0a3.dev18+gea2a299e8.d20260730", "host": "lrdn4990.leonardo.local"} diff --git a/benches/third_party/majorana_prop/plot_results.py b/benches/third_party/majorana_prop/plot_results.py index fd4db42c..b72f4f2e 100644 --- a/benches/third_party/majorana_prop/plot_results.py +++ b/benches/third_party/majorana_prop/plot_results.py @@ -15,74 +15,70 @@ from __future__ import annotations import argparse +import json from pathlib import Path import matplotlib.pyplot as plt -import pandas as pd - - -def load_benchmark(path: Path, source: str) -> pd.DataFrame: - """Load a benchmark JSONL file into a DataFrame of runtime/term-count rows.""" - df = pd.read_json(path, lines=True) - df = df.rename( - columns={ - "n_spinful_sites": "n_spin", - "n_layers": "layers", - "runtime_seconds": "seconds", - "memory_MB": "memory", - "final_overlap": "overlap", - } - ) - df["source"] = source - return df[ - ["n_spin", "layers", "num_terms", "seconds", "memory", "overlap", "source"] - ].sort_values(["n_spin", "layers"]) - - -def plot_metric(ax, data: pd.DataFrame, metric: str, ylabel: str) -> None: - """Plot ``metric`` vs. layers for each n_spin/source combination onto ``ax``.""" - styles = {"monoprop": "-o", "MajoranaPropagation.jl": "--x"} - colors = plt.cm.tab10.colors - for i, n_spin in enumerate(sorted(data["n_spin"].unique())): - color = colors[i % len(colors)] - for source, style in styles.items(): - subset = data[(data["n_spin"] == n_spin) & (data["source"] == source)] - if subset.empty: - continue + +STYLES = {"monoprop": "-o", "MajoranaPropagation.jl": "--x"} + + +def plot_metric( + ax, + step_range: list[int], + metric_dict: dict[str, list[float]], + ylabel: str, + secondary_dict: dict[str, list[float]] | None = None, +) -> None: + """Plot ``metric_dict[source]`` vs. ``step_range`` for each source onto ``ax``. + + ``secondary_dict``, where given, is drawn as a faint unlabeled line reusing each source's own + color and linestyle (only reduced alpha, no marker, distinguishes it) — a different + measurement of the same source, not a new series. + """ + colors_by_source = {} + for source, values in metric_dict.items(): + (line,) = ax.plot(step_range, values, STYLES.get(source, "-o"), label=source) + colors_by_source[source] = line.get_color() + if secondary_dict: + for source, values in secondary_dict.items(): + linestyle = "--" if STYLES.get(source, "-o").startswith("--") else "-" ax.plot( - subset["layers"], - subset[metric], - style, - color=color, - label=f"n={n_spin} ({source})", + step_range, + values, + linestyle=linestyle, + color=colors_by_source.get(source), + alpha=0.4, + linewidth=1, ) ax.set_xlabel("layers") ax.set_ylabel(ylabel) - ax.legend(fontsize="small", ncol=2) + ax.legend(fontsize="small") ax.grid(True, alpha=0.3) -def plot_runtime_figure(df: pd.DataFrame, out: Path) -> None: +def plot_runtime_figure( + step_range: list[int], runtimes: dict[str, list[float]], out: Path +) -> None: """Save the runtime on its own canvas, both axes linear. Runtime is the quantity that gets cited on its own, so it gets its own file rather than a quarter of the combined grid. Both axes are linear: seconds are read as seconds, so the - 600 s the Julia engine spends at the largest point is drawn 140x the height of monoprop's - 4 s instead of being compressed into a decade's width. That flattens monoprop's own curve - against the axis, which is the finding, not a defect of the axis. + time the Julia engine spends at the largest depth is drawn at its true multiple of + monoprop's instead of being compressed into a decade's width. That flattens monoprop's own + curve against the axis, which is the finding, not a defect of the axis. """ fig, ax = plt.subplots(figsize=(7.4, 5.4)) - plot_metric(ax, df, "seconds", "time (seconds)") + plot_metric(ax, step_range, runtimes, "time (seconds)") ax.set_ylim(bottom=0) ax.set_title("1D Hubbard runtime vs circuit depth", fontsize="medium") - # On this axis monoprop's three curves lie on top of each other along the bottom, where a - # reader cannot tell 4 s from 0 s. State the band's top so the flat lines read as small - # rather than as absent. - monoprop = df[df["source"] == "monoprop"]["seconds"] - if not monoprop.empty: + # On this axis monoprop's curve lies along the bottom, where a reader cannot tell 4 s from + # 0 s. State the band's top so the flat line reads as small rather than as absent. + monoprop = runtimes.get("monoprop") + if monoprop: ax.annotate( - f"monoprop: all points ≤ {monoprop.max():.1f} s", - xy=(df["layers"].max(), monoprop.max()), + f"monoprop: all points ≤ {max(monoprop):.1f} s", + xy=(step_range[-1], max(monoprop)), xytext=(-6, 14), textcoords="offset points", ha="right", @@ -98,16 +94,10 @@ def plot_runtime_figure(df: pd.DataFrame, out: Path) -> None: def main() -> None: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( - "--monoprop-results", + "--results", type=Path, - default=Path("monoprop_hubbard1d_benchmark_results.jsonl"), - help="Path to the monoprop benchmark results JSONL file.", - ) - parser.add_argument( - "--julia-results", - type=Path, - default=Path("julia_hubbard1d_benchmark_results.jsonl"), - help="Path to the julia benchmark results JSONL file.", + default=Path(__file__).with_name("results.json"), + help="Path to the shared benchmark results JSON file.", ) parser.add_argument( "--output-dir", @@ -122,29 +112,47 @@ def main() -> None: ) args = parser.parse_args() - monoprop_df = load_benchmark(args.monoprop_results, "monoprop") - julia_df = load_benchmark(args.julia_results, "MajoranaPropagation.jl") - df = pd.concat([monoprop_df, julia_df], ignore_index=True) + with args.results.open() as file: + data = json.load(file) + step_range = data["step_range"] args.output_dir.mkdir(parents=True, exist_ok=True) fig, axes = plt.subplots(2, 2, figsize=(14, 10)) - plot_metric(axes[0, 0], df, "seconds", "time (seconds)") - axes[0, 0].set_title("Runtime vs layers") + plot_metric(axes[0, 0], step_range, data["runtime_seconds"], "time (seconds)") + axes[0, 0].set_title(f"Runtime vs layers (n_spin={data['n_spinful_sites']})") - plot_metric(axes[0, 1], df, "num_terms", "number of terms") + plot_metric(axes[0, 1], step_range, data["num_terms"], "number of terms") axes[0, 1].set_title("Number of terms vs layers") - plot_metric(axes[1, 0], df, "memory", "memory (MB)") + native_memory_dict = data.get("native_memory_MB", {}) + plot_metric( + axes[1, 0], + step_range, + data["memory_MB"], + "memory (MB)", + secondary_dict=native_memory_dict, + ) axes[1, 0].set_title("Memory vs layers") - plot_metric(axes[1, 1], df, "overlap", "final overlap") - axes[1, 1].set_title("Final overlap vs layers") + plot_metric(axes[1, 1], step_range, data["expectation_value"], "expectation value") + axes[1, 1].set_title("Expectation value vs layers") fig.tight_layout() - fig.savefig(args.output_dir / "majorana_results.png") + if native_memory_dict: + fig.text( + 0.5, + 0.005, + "Faint lines: each engine's own native memory accounting (reference only, not the plotted peak)", + ha="center", + fontsize=8, + color="gray", + ) + fig.savefig(args.output_dir / "majorana_results.png", bbox_inches="tight") - plot_runtime_figure(df, args.output_dir / "majorana_runtime.png") + plot_runtime_figure( + step_range, data["runtime_seconds"], args.output_dir / "majorana_runtime.png" + ) if args.show: plt.show() diff --git a/benches/third_party/majorana_prop/results.json b/benches/third_party/majorana_prop/results.json new file mode 100644 index 00000000..8f53f37d --- /dev/null +++ b/benches/third_party/majorana_prop/results.json @@ -0,0 +1,271 @@ +{ + "runtime_seconds": { + "monoprop": [ + 0.0002520989983167965, + 0.008602955000242218, + 0.023853869999584276, + 0.03253005099759321, + 0.04263711499879719, + 0.054794172996480484, + 0.0691948399944522, + 0.08690634199228953, + 0.12709497199466568, + 0.16048369099371484, + 0.21669280699279625, + 0.30345294799190015, + 0.44508027899064473, + 0.7188304689916549, + 1.3468647859917837, + 2.438142914990749, + 4.438301431990112, + 7.850639691991091, + 13.51275252499181, + 23.016640379992168, + 37.62995160999344 + ], + "MajoranaPropagation.jl": [ + 2.4566e-5, + 0.06570643400000001, + 1.066572892, + 1.5099781319999999, + 1.9726161699999998, + 2.837843683, + 3.549592132, + 4.467205515, + 5.8747669469999995, + 7.879885148, + 11.006226405, + 16.263505637, + 25.217318649, + 40.227838588, + 64.894778695, + 103.74468536399999, + 165.595709796, + 265.030177113, + 425.03952550199995, + 680.1839463509999, + 1082.0488366109998 + ] + }, + "n_spinful_sites": 60, + "native_memory_MB": { + "monoprop": [ + 0.01990985870361328, + 0.020415306091308594, + 0.08391475677490234, + 0.30586719512939453, + 0.7510213851928711, + 1.8966608047485352, + 4.013327598571777, + 8.111088752746582, + 12.2099027633667, + 24.486376762390137, + 44.62830066680908, + 81.94080448150635, + 149.2356767654419, + 223.22717571258545, + 397.6318521499634, + 722.331093788147, + 1286.3422632217407, + 1910.7156629562378, + 3150.523093223572, + 5411.497309684753, + 7801.361777305603 + ], + "MajoranaPropagation.jl": [ + 0.00018310546875, + 0.00518035888671875, + 0.11733245849609375, + 0.5146713256835938, + 1.0477371215820312, + 2.1161575317382812, + 4.2556304931640625, + 8.202163696289062, + 15.763961791992188, + 28.70745849609375, + 52.974884033203125, + 92.66452026367188, + 164.24754333496094, + 304.7784729003906, + 462.87574005126953, + 760.7351608276367, + 1222.6302642822266, + 1976.4950866699219, + 3162.911178588867, + 5007.087661743164, + 7825.958740234375 + ] + }, + "memory_MB": { + "monoprop": [ + 38.58984375, + 38.58984375, + 38.58984375, + 38.58984375, + 38.58984375, + 43.62109375, + 44.28515625, + 52.13671875, + 54.19921875, + 64.51171875, + 86.52734375, + 122.26953125, + 176.3203125, + 285.68359375, + 433.17578125, + 726.55859375, + 1208.3515625, + 2059.21875, + 3048.30078125, + 4426.33984375, + 10636.85546875 + ], + "MajoranaPropagation.jl": [ + 534.9140625, + 551.109375, + 557.359375, + 557.3125, + 563.76171875, + 577.4765625, + 597.67578125, + 612.3359375, + 640.3359375, + 730.30078125, + 768.875, + 898.4765625, + 1081.39453125, + 1229.63671875, + 1765.05859375, + 2044.484375, + 2949.34375, + 4322.48046875, + 6477.3671875, + 10206.3359375, + 15694.859375 + ] + }, + "step_range": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20 + ], + "num_threads": { + "monoprop": "24", + "MajoranaPropagation.jl": 24 + }, + "expectation_value": { + "monoprop": [ + 0.0, + 0.009736140422307249, + 0.03827070394062437, + 0.08364455094749212, + 0.1427925477622065, + 0.21181820471470336, + 0.2863324191238933, + 0.36181749808823005, + 0.43398016578994325, + 0.49905876932925186, + 0.5540634920911527, + 0.5969321865128213, + 0.6265985937915881, + 0.6429730022355784, + 0.646851751012522, + 0.6397661922659064, + 0.6237899786874557, + 0.6013251208948387, + 0.5748905367524423, + 0.5469245051110052, + 0.5196156652467371 + ], + "MajoranaPropagation.jl": [ + 0.0, + 0.009736140422307242, + 0.03827070330991876, + 0.08364453430830862, + 0.1427924873717963, + 0.21181811458903516, + 0.2863323455315943, + 0.3618176213290471, + 0.4339803099823046, + 0.49905894198645734, + 0.5540635608915114, + 0.5969324107189943, + 0.6265984814098748, + 0.6429728162492787, + 0.6468519475075789, + 0.6397668861944836, + 0.6237907501255578, + 0.6013260020506138, + 0.5748921471542362, + 0.5469264564869873, + 0.51961452777913 + ] + }, + "n_layers": 20, + "num_terms": { + "monoprop": [ + 1, + 16, + 1344, + 5794, + 15637, + 35883, + 73597, + 141845, + 267386, + 488053, + 872870, + 1513509, + 2606766, + 4385183, + 7276415, + 11882982, + 19078830, + 30227167, + 47122113, + 72352237, + 109381056 + ], + "MajoranaPropagation.jl": [ + 2, + 17, + 1162, + 4351, + 11380, + 25649, + 52784, + 101198, + 187217, + 338317, + 597601, + 1025597, + 1754476, + 2933083, + 4865089, + 7957480, + 12859755, + 20582020, + 32625568, + 51103291, + 79406422 + ] + } +} diff --git a/benches/third_party/majorana_prop/run_benchmarks.sh b/benches/third_party/majorana_prop/run_benchmarks.sh index ea2724a3..041c8e3d 100755 --- a/benches/third_party/majorana_prop/run_benchmarks.sh +++ b/benches/third_party/majorana_prop/run_benchmarks.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # Run the Julia (MajoranaPropagation.jl) and monoprop 1D Hubbard benchmarks -# back to back, appending results to their respective JSONL files. +# back to back, merging both into the shared results.json, then plot them. set -euo pipefail @@ -14,15 +14,14 @@ export OMP_NUM_THREADS="${OMP_NUM_THREADS:-$monoprop_NUM_THREADS}" export MKL_NUM_THREADS="${MKL_NUM_THREADS:-$monoprop_NUM_THREADS}" export OPENBLAS_NUM_THREADS="${OPENBLAS_NUM_THREADS:-$monoprop_NUM_THREADS}" + +echo "Running monoprop benchmark (monoprop_NUM_THREADS=${monoprop_NUM_THREADS})" +uv run python monoprop_hubbard1d_benchmark.py + julia --project=@. -e 'using Pkg; Pkg.instantiate()' julia --project=@. -e 'using Pkg; Pkg.precompile()' -echo "Running Julia benchmark (cases 1-15, JULIA_NUM_THREADS=${JULIA_NUM_THREADS})" -for case in $(seq 1 15); do - julia --project=@. julia_hubbard1d_benchmark.jl --case "$case" -done +echo "Running Julia benchmark (JULIA_NUM_THREADS=${JULIA_NUM_THREADS})" +julia --project=@. julia_hubbard1d_benchmark.jl -echo "Running monoprop benchmark (cases 0-14, monoprop_NUM_THREADS=${monoprop_NUM_THREADS})" -for case in $(seq 0 14); do - uv run python monoprop_hubbard1d_benchmark.py --case "$case" -done +uv run python plot_results.py diff --git a/benches/third_party/pauli_prop/backends.py b/benches/third_party/pauli_prop/backends.py index 447fe5c7..fdac2f87 100644 --- a/benches/third_party/pauli_prop/backends.py +++ b/benches/third_party/pauli_prop/backends.py @@ -24,22 +24,30 @@ from __future__ import annotations +import sys import time from collections.abc import Callable, Sequence from dataclasses import dataclass, field +from pathlib import Path import numpy as np -import psutil - from model import Settings, grid_edges, observable, pauli_rotations, step_circuit -# What each backend's `memory` series actually measures. They are not the same -# quantity — say so wherever these numbers are plotted or tabulated. -MEMORY_METRICS = { +# The repository's own benchmark suite owns the memory instrumentation; this directory is a +# separate uv project, so reach it by path rather than by dependency. +sys.path.insert(0, str(Path(__file__).resolve().parents[2])) +from _memory import HighWaterMark # noqa: E402 + +# Every backend's `memory` series is this one quantity, measured the same way, so the +# curves may share an axis. +HOST_MEMORY_METRIC = "peak process RSS over the step (kernel VmHWM)" + +# What a backend reports about its *own* footprint, where it reports anything. Reference +# only: these are not commensurable with each other (one counts an operator, another an +# object graph, another a device pool), so they must never be compared across backends. +OPERATOR_MEMORY_METRICS = { "monoprop": "operator memory (reported by the library)", - "QuEra ppvm": "process RSS growth (no memory accounting exposed)", - "Qiskit pauli-prop": "process RSS growth (no memory accounting exposed)", - "cuPauliProp (GPU)": "GPU memory pool in use", + "cuPauliProp (GPU)": "GPU memory pool in use at end of step", "PauliPropagation.jl": "Base.summarysize of the Pauli sum", } @@ -52,36 +60,52 @@ class BackendResult: runtime: list[float] = field(default_factory=list) expvals: list[float] = field(default_factory=list) num_terms: list[int] = field(default_factory=list) + # Peak host RSS over each step; see HOST_MEMORY_METRIC. memory: list[float] = field(default_factory=list) - # Set when the library reports its own operator footprint, so the scaling - # summary can prefer it over a whole-process RSS proxy. - operator_memory_mb: float | None = None + # Empty for backends that expose no accounting of their own. + operator_memory: list[float] = field(default_factory=list) @property def memory_metric(self) -> str: - return MEMORY_METRICS.get(self.label, "unknown") + return HOST_MEMORY_METRIC + + @property + def operator_memory_metric(self) -> str: + return OPERATOR_MEMORY_METRICS.get(self.label, "none reported") + + @property + def operator_memory_mb(self) -> float | None: + """Return the library's own final footprint, where it reports one.""" + return self.operator_memory[-1] if self.operator_memory else None def _run_steps( settings: Settings, label: str, - step: Callable[[int], tuple[float, int, float]], + step: Callable[[int], tuple[float, int, float | None]], ) -> BackendResult: """Drive `step` once per Trotter point, timing it and recording what it returns. - `step(step_idx)` advances the backend by one benchmark step and returns - (expectation value, term count, memory MB). The timer brackets the propagation - *and* the expectation value, matching what every backend reports. + `step(step_idx)` advances the backend by one benchmark step and returns (expectation + value, term count, the library's own footprint in MB or None). The timer brackets the + propagation *and* the expectation value, matching what every backend reports. + + Peak memory is taken here rather than inside each backend so that every engine is + measured by the same instrument over exactly the timed region. Opening the window + settles the process, which is why it wraps the timer instead of the reverse. """ result = BackendResult(label=label) for step_idx, _ in enumerate(settings.step_range): - t1 = time.perf_counter() - expval, num_terms, memory_mb = step(step_idx) - t2 = time.perf_counter() + with HighWaterMark() as window: + t1 = time.perf_counter() + expval, num_terms, operator_mb = step(step_idx) + t2 = time.perf_counter() result.runtime.append(t2 - t1) result.expvals.append(expval) result.num_terms.append(num_terms) - result.memory.append(memory_mb) + result.memory.append(window.peak_mb) + if operator_mb is not None: + result.operator_memory.append(operator_mb) return result @@ -97,15 +121,13 @@ def run_monoprop(settings: Settings) -> BackendResult: lower_atol=settings.lower_atol, ) - def step(_step_idx: int) -> tuple[float, int, float]: + def step(_step_idx: int) -> tuple[float, int, float | None]: propagator.propagate(circ) expval = propagator.expectation_value() mem = propagator._simulator.operator_memory_bytes() / 1024**2 return expval, propagator.size(), mem - result = _run_steps(settings, "monoprop", step) - result.operator_memory_mb = result.memory[-1] - return result + return _run_steps(settings, "monoprop", step) def run_ppvm(settings: Settings) -> BackendResult: @@ -120,14 +142,9 @@ def run_ppvm(settings: Settings) -> BackendResult: max_pauli_weight=settings.max_pauli_weight, ) edges = grid_edges(settings.nx, settings.ny) - process = psutil.Process() - accumulated_bytes = 0 - - def step(_step_idx: int) -> tuple[float, int, float]: - # ppvm exposes no memory accounting: approximate it via RSS growth over its - # own step. The timer is already running, so keep this to two cheap reads. - nonlocal accumulated_bytes - before = process.memory_info().rss + + # ppvm exposes no memory accounting of its own; _run_steps measures it from outside. + def step(_step_idx: int) -> tuple[float, int, float | None]: for i, k in edges: pauli_sum.rzz(i, k, settings.theta_zz) for i in range(nq): @@ -135,8 +152,7 @@ def step(_step_idx: int) -> tuple[float, int, float]: for i in range(nq): pauli_sum.rx(i, settings.theta_x) expval = pauli_sum.overlap_with_zero() - accumulated_bytes += max(0, process.memory_info().rss - before) - return expval, len(pauli_sum), accumulated_bytes / 1024**2 + return expval, len(pauli_sum), None return _run_steps(settings, "QuEra ppvm", step) @@ -153,13 +169,11 @@ def run_qiskit(settings: Settings, max_terms: int | Sequence[int]) -> BackendRes operator = observable(settings) circ = step_circuit(settings) - process = psutil.Process() - accumulated_bytes = 0 - def step(step_idx: int) -> tuple[float, int, float]: - nonlocal operator, accumulated_bytes + # pauli-prop exposes no memory accounting of its own; _run_steps measures it from outside. + def step(step_idx: int) -> tuple[float, int, float | None]: + nonlocal operator budget = max_terms if isinstance(max_terms, int) else max_terms[step_idx] - before = process.memory_info().rss operator, _ = propagate_through_circuit( operator, circ, @@ -168,8 +182,7 @@ def step(step_idx: int) -> tuple[float, int, float]: frame="h", ) expval = float(operator.coeffs[~operator.paulis.x.any(axis=1)].sum()) - accumulated_bytes += max(0, process.memory_info().rss - before) - return expval, len(operator), accumulated_bytes / 1024**2 + return expval, len(operator), None return _run_steps(settings, "Qiskit pauli-prop", step) @@ -209,7 +222,7 @@ def run_cupauliprop(settings: Settings) -> BackendResult: for theta, paulis, qubits in pauli_rotations(settings) ] - def step(_step_idx: int) -> tuple[float, int, float]: + def step(_step_idx: int) -> tuple[float, int, float | None]: nonlocal expansion for gate in reversed(gates): expansion = expansion.apply_gate( @@ -221,12 +234,13 @@ def step(_step_idx: int) -> tuple[float, int, float]: ) significand, exponent = expansion.trace_with_zero_state() expval = float(significand * np.exp2(exponent)) + # End-of-step pool occupancy, not a peak: a transient freed before the step + # returns is invisible here. The device-side analogue of VmHWM is the pool's + # cudaMemPoolAttrUsedMemHigh, which is resettable; wiring it up needs a GPU. mem = cp.get_default_memory_pool().used_bytes() / 1024**2 return expval, expansion.num_terms, mem - result = _run_steps(settings, "cuPauliProp (GPU)", step) - result.operator_memory_mb = result.memory[-1] - return result + return _run_steps(settings, "cuPauliProp (GPU)", step) def _pack_pauli_string( diff --git a/benches/third_party/pauli_prop/plot_results.py b/benches/third_party/pauli_prop/plot_results.py index a1005392..266cc478 100644 --- a/benches/third_party/pauli_prop/plot_results.py +++ b/benches/third_party/pauli_prop/plot_results.py @@ -35,6 +35,7 @@ step_range = data["step_range"] runtime_dict = data["runtime"] memory_dict = data["memory"] +native_memory_dict = data.get("native_memory", {}) expvals_dict = data["expvals"] @@ -63,13 +64,25 @@ def _style_axes(ax: plt.Axes, ylabel: str) -> None: for label, runtime in runtime_dict.items(): steps, values = _filter_from_min_step(step_range, runtime) - ax1.plot(steps, values, color=colors[label], label=label) + ax1.plot(steps, values, color=colors[label], label=label, marker="o", markersize=4) _style_axes(ax1, "Time per step [s]") for label, memory in memory_dict.items(): steps, values = _filter_from_min_step(step_range, memory) - ax2.plot(steps, values, color=colors[label], label=label) + ax2.plot(steps, values, color=colors[label], label=label, marker="o", markersize=4) +for label, native_memory in native_memory_dict.items(): + steps, values = _filter_from_min_step(step_range, native_memory) + ax2.plot(steps, values, color=colors[label], linestyle="--", alpha=0.5) _style_axes(ax2, "Memory per step [MB]") fig.tight_layout() -fig.savefig(Path(__file__).parent / "pauli_results.png", dpi=150) +if native_memory_dict: + fig.text( + 0.5, + -0.03, + "Dashed: each engine's own native memory accounting (reference only, not the plotted peak)", + ha="center", + fontsize=8, + color="gray", + ) +fig.savefig(Path(__file__).parent / "pauli_results.png", dpi=150, bbox_inches="tight") diff --git a/benches/third_party/pauli_prop/plot_scaling.py b/benches/third_party/pauli_prop/plot_scaling.py index 6e6e1c18..9c6ef1c9 100644 --- a/benches/third_party/pauli_prop/plot_scaling.py +++ b/benches/third_party/pauli_prop/plot_scaling.py @@ -280,9 +280,10 @@ def main() -> None: "--memory-key", default="final_memory_MB", choices=["final_memory_MB", "operator_memory_MB", "peak_rss_MB"], - help="Which memory column to plot. The default is each library's own final " - "operator accounting, which is not the same quantity for every backend " - "(see MEMORY_METRICS in backends.py).", + help="Which memory column to plot. The default is the peak process RSS over the " + "final step, the same quantity for every backend. `operator_memory_MB` is each " + "library's own accounting and is not comparable across backends " + "(see OPERATOR_MEMORY_METRICS in backends.py).", ) args = parser.parse_args() diff --git a/benches/third_party/pauli_prop/results.json b/benches/third_party/pauli_prop/results.json index 6e8c97c8..cec07aae 100644 --- a/benches/third_party/pauli_prop/results.json +++ b/benches/third_party/pauli_prop/results.json @@ -1,231 +1,301 @@ { "expvals": { - "cuPauliProp (GPU)": [ - 0.997502082639013, - 0.9903354444404697, - 0.9794361767419918, - 0.9661809649164091, - 0.9522199951963057, - 0.9392112109453805, - 0.9286632604749514, - 0.9217215256190198, - 0.9192581471551303, - 0.9213746654172797, - 0.9275425648672762, - 0.9368014183084314, - 0.9479586047326162, - 0.959807462945837, - 0.9713666467866726, - 0.9817625295735538, - 0.9897022306685668, - 0.9940854516308357, - 0.9943906072586334, - 0.9904540465925349, - 0.9826744215990753 - ], "monoprop": [ 0.997502082639013, - 0.9903354444404695, - 0.9794361767419921, - 0.9661809649070634, - 0.9522199947834874, - 0.9392112074970561, - 0.9286632441788407, - 0.9217296258792056, - 0.9192734598537711, - 0.9213900001024592, - 0.9275541488151822, - 0.9368118309137169, - 0.9479633902932676, - 0.9598075713501336, - 0.9713646524559834, - 0.9817709140803824, - 0.9897243499921878, - 0.9941371201900285, - 0.9944276721765963, - 0.9904727245014008, - 0.9826714907890624 + 0.9903354444404696, + 0.9794363795243637, + 0.9661812273018453, + 0.9522201534337065, + 0.9392107973137432, + 0.928658478653306, + 0.9217007651974013, + 0.9191560692122138, + 0.9211446372117517, + 0.9271845753190968, + 0.936262879961247, + 0.9471923300179088, + 0.9587376552397145, + 0.969822741864718, + 0.9796969929662139, + 0.9872955993601161, + 0.9916867408908515, + 0.9923197727253984, + 0.9889797246327313, + 0.9820133275023492, + 0.9725123646210144, + 0.9615984971418958, + 0.9506665763222667, + 0.9410126650301269, + 0.9335833847204386, + 0.9293465294442301, + 0.9286933588139175 ], "PauliPropagation.jl": [ 0.997502082639013, 0.9903354444404696, - 0.9794361767419826, - 0.9661809649157846, - 0.9522199952275356, - 0.9392112114322317, - 0.9286632469647392, - 0.9217215611595609, - 0.9192583164556223, - 0.9213751727716332, - 0.9275437882758828, - 0.9368026677065466, - 0.947960191050668, - 0.959808776235663, - 0.9713620730260529, - 0.9817573708646599, - 0.9896923125323471, - 0.9940732143365996, - 0.9943759464284816, - 0.9904334566154421, - 0.9826514575855047 + 0.979436175612008, + 0.9661809422038069, + 0.9522198487388678, + 0.939210514948294, + 0.9286608751710758, + 0.9217112490651745, + 0.9191948496469634, + 0.9211910356841434, + 0.9271784314013242, + 0.93621958436388, + 0.9471566926984755, + 0.9587790697230953, + 0.9700810609412963, + 0.980162477442118, + 0.9878248525459249, + 0.9920739814443645, + 0.9924615969482351, + 0.988932945770035, + 0.9818791500670394, + 0.9724983537122938, + 0.9617580208511031, + 0.9510289865354113, + 0.9414094249138573, + 0.9339856557135312, + 0.9296925398484073, + 0.9290590727386197 ], "QuEra ppvm": [ 0.997502082639013, 0.9903354444404695, - 0.9794361709556907, - 0.9661809639597803, - 0.9522199930461324, - 0.939211173609548, - 0.9286630881228823, - 0.9217301572595826, - 0.9192646317194918, - 0.9213799216222511, - 0.9275449829326113, - 0.9368045165263471, - 0.947960505529967, - 0.9598101077658275, - 0.9713725242948469, - 0.9817783051928322, - 0.9897214879232376, - 0.9941040455760576, - 0.994408812093782, - 0.9904678303543354, - 0.9826766600224272 + 0.9794361695706763, + 0.9661809419183106, + 0.952219851108817, + 0.9392104831471718, + 0.9286607106018888, + 0.9217238529580906, + 0.9191949081786188, + 0.9211849242571408, + 0.9271566494169908, + 0.9361971175113651, + 0.9471358062576694, + 0.9587629296265585, + 0.9700533017012288, + 0.9801074408254876, + 0.9877817362610698, + 0.9920378901252231, + 0.9923792792918043, + 0.9888079988294428, + 0.9818048059131084, + 0.9724237538531765, + 0.9617113062712275, + 0.9509518686115404, + 0.9412854838029944, + 0.9338317435872012, + 0.9295056908756767, + 0.9288856203834183 ], "Qiskit pauli-prop": [ 0.997502082639013, - 0.9902986523563847, - 0.979410268438022, - 0.9662328011860434, - 0.95233216052613, - 0.9393290880734805, - 0.9286509501231658, - 0.921402878684645, - 0.918416768030144, - 0.9200459604215262, - 0.926371295132605, - 0.9364729500095121, - 0.9485473738161867, - 0.9609797327252398, - 0.9722174479178898, - 0.9808890727517166, - 0.987202501099926, - 0.9913511314008273, - 0.9925835343091947, - 0.990196575726684, - 0.9839317508567623 + 0.9902616416411947, + 0.9793735672305858, + 0.9662045535658592, + 0.9523044065600075, + 0.9393045616009582, + 0.9286284339439106, + 0.9213619296914645, + 0.9183461621464206, + 0.9197422831890821, + 0.9257819331883357, + 0.9355874815011092, + 0.9473519646429599, + 0.9595351288671997, + 0.9706431446987511, + 0.9793536253319963, + 0.985549673916859, + 0.989473983293285, + 0.9906124875227108, + 0.9885381901105181, + 0.9829097988824232, + 0.9739113012525142, + 0.9622651253609089, + 0.9500765782737872, + 0.9390899221282094, + 0.9307655543344951, + 0.9263461450046511, + 0.9258467634203383 + ], + "cuPauliProp (GPU)": [ + 0.997502082639013, + 0.9903354444404697, + 0.9794361756120173, + 0.966180942204426, + 0.9522198487076173, + 0.9392105144690244, + 0.9286608888777017, + 0.9217122128671503, + 0.9191946799798976, + 0.9211905093829498, + 0.9271772949331979, + 0.9362137063468555, + 0.9471504289968796, + 0.9587710132467064, + 0.9700774110665801, + 0.9801616099109822, + 0.9878274775819245, + 0.9920739099660344, + 0.9924591024470226, + 0.9889358100230531, + 0.9818742136637044, + 0.9724789876050834, + 0.9617425377637874, + 0.9510140541676235, + 0.9413815283280191, + 0.9339539276043874, + 0.9296816450847989, + 0.9290742115849888 ] }, "runtime": { - "cuPauliProp (GPU)": [ - 0.020176051184535027, - 0.019714422058314085, - 0.02136979578062892, - 0.021861208137124777, - 0.022774800192564726, - 0.02481368323788047, - 0.02854348300024867, - 0.029668635223060846, - 0.030517147853970528, - 0.033659269101917744, - 0.03531042719259858, - 0.03751846496015787, - 0.039435874205082655, - 0.04148514196276665, - 0.04600577801465988, - 0.05237875320017338, - 0.06031936779618263, - 0.1818047002889216, - 0.2003558687865734, - 0.24681029003113508 - ], "monoprop": [ - 0.0022588460706174374, - 0.002730349078774452, - 0.003645222634077072, - 0.004602230153977871, - 0.005807321984320879, - 0.007761758286505938, - 0.010728122666478157, - 0.015644437167793512, - 0.022155003156512976, - 0.030679493211209774, - 0.04855358274653554, - 0.07472044182941318, - 0.10821856698021293, - 0.1829305151477456, - 0.2592461039312184, - 0.34765971498563886, - 0.5380650600418448, - 0.753288147971034, - 1.133904397021979, - 1.6045584678649902 + 0.0097709740002756, + 0.009900220000417903, + 0.01062245899811387, + 0.010208111998508684, + 0.010301656999217812, + 0.011290021997410804, + 0.0119530299998587, + 0.012820420997741167, + 0.014460218000749592, + 0.01662767300149426, + 0.021685593004804105, + 0.025799656999879517, + 0.03668873199785594, + 0.049491324003611226, + 0.06257617600203957, + 0.08155569299560739, + 0.10271872900193557, + 0.15972542300005443, + 0.26723438299814006, + 0.4563296580017777, + 0.5873996790032834, + 0.8560777880047681, + 1.2323196020006435, + 1.717144444999576, + 2.43780124100158, + 3.313076704995183, + 4.471609448999516 ], "PauliPropagation.jl": [ - 0.000487617, - 0.001201636, - 0.002411776, - 0.005773478, - 0.010534435, - 0.02769105, - 0.049536102, - 0.088493142, - 0.15495462, - 0.280121239, - 0.528610672, - 0.762137845, - 1.214638218, - 2.272540931, - 3.176273919, - 5.286394518, - 7.245942986, - 11.462868432, - 15.114584049, - 24.351451212 + 0.937622339, + 0.632157357, + 0.703072464, + 0.627356295, + 0.796526843, + 0.659159013, + 0.701493095, + 0.833921075, + 0.917434561, + 1.408576301, + 1.80445128, + 2.311063839, + 2.965923955, + 4.003973926, + 5.832220713, + 8.279620075, + 11.971029624, + 16.877259221, + 24.415467044, + 32.886627487, + 42.673318014, + 57.788362696, + 77.248532745, + 103.575524758, + 139.279354003, + 185.886735186, + 246.66351665 ], "QuEra ppvm": [ - 0.00018265610560774803, - 0.000364821869879961, - 0.0006602783687412739, - 0.0016023130156099796, - 0.0034242477267980576, - 0.007562015671283007, - 0.0130642163567245, - 0.021341342013329268, - 0.04229155322536826, - 0.07402261719107628, - 0.128699810244143, - 0.23124448582530022, - 0.38588487124070525, - 0.6836787946522236, - 1.3010696759447455, - 2.5424343938939273, - 4.085273690987378, - 6.896651620976627, - 9.923423228785396, - 14.719230208080262 + 0.0009772880002856255, + 0.0019692240020958707, + 0.003414564002014231, + 0.006883168003696483, + 0.01336483699560631, + 0.026874936003878247, + 0.049350055000104476, + 0.08147332700173138, + 0.1398485649988288, + 0.24163390600006096, + 0.41795454300154233, + 0.6846707399963634, + 1.1364982200029772, + 2.14537202000065, + 4.182404636005231, + 6.776013484995929, + 10.121395098001813, + 14.356937502001529, + 20.45524097100133, + 29.319063470997207, + 41.06935781400534, + 58.04860244700103, + 84.5264299709961, + 119.09453202800069, + 166.27936962900276, + 240.10627245400246, + 323.1610831039943 ], "Qiskit pauli-prop": [ - 0.007913357112556696, - 0.008004344068467617, - 0.009662941563874483, - 0.012174050323665142, - 0.017478680703788996, - 0.027277078945189714, - 0.04581389995291829, - 0.07399411406368017, - 0.12357027316465974, - 0.21096500102430582, - 0.34313367400318384, - 0.5556078860536218, - 0.8972947858273983, - 1.4494283101521432, - 2.3114430508576334, - 3.8287112680263817, - 5.7641011090017855, - 8.530939413700253, - 12.57545457687229, - 18.853173348121345 + 0.0910057479995885, + 0.09350110300147207, + 0.09966104299383005, + 0.11175302699848544, + 0.1350917930030846, + 0.17215780800324865, + 0.23493830300139962, + 0.3418267029992421, + 0.5214368059969274, + 0.8170689979961026, + 1.2857159330014838, + 2.0053665779996663, + 3.158475438001915, + 4.8786017739985255, + 7.400178270996548, + 11.198802080994938, + 16.48318638800265, + 23.556144617999962, + 33.42640477000532, + 46.69200970899692, + 65.27846676199988, + 91.56253991199628, + 125.43558071399457, + 172.17275571200298, + 235.64995327400538, + 320.03587235399755, + 437.8359219260019 + ], + "cuPauliProp (GPU)": [ + 0.18526526999630732, + 0.18972675999975763, + 0.1926419649971649, + 0.19678318400110584, + 0.2010178460041061, + 0.20950794600503286, + 0.21115881600417197, + 0.21557678499812027, + 0.2187041910001426, + 0.22579057099937927, + 0.23606160299823387, + 0.25517424199642846, + 0.2714905280008679, + 0.30626787999790395, + 0.35749711799871875, + 0.419371971001965, + 0.49677398100175196, + 0.5983771039973362, + 0.890931271998852, + 1.250609467002505, + 1.8410984980000649, + 2.3888087309969706, + 3.464189799000451, + 4.2792873379949015, + 6.69145935200504, + 8.090263483994931, + 12.02248638200399 ] }, "step_range": [ @@ -249,240 +319,409 @@ 34, 36, 38, - 40 + 40, + 42, + 44, + 46, + 48, + 50, + 52, + 54 ], "memory": { - "cuPauliProp (GPU)": [ - 0.0068359375, - 0.02099609375, - 0.0390625, - 0.08251953125, - 0.162109375, - 0.2978515625, - 0.5458984375, - 0.958984375, - 1.68505859375, - 2.8603515625, - 4.74658203125, - 7.6689453125, - 12.1708984375, - 18.57080078125, - 28.3544921875, - 41.5859375, - 60.4814453125, - 86.59228515625, - 122.46875, - 173.025390625, - 242.6591796875 - ], "monoprop": [ - 0.02881336212158203, - 0.04080677032470703, - 0.06046581268310547, - 0.14527225494384766, - 0.2773103713989258, - 0.5006303787231445, - 0.8862504959106445, - 1.7319231033325195, - 3.103184700012207, - 5.1228837966918945, - 7.2314958572387695, - 13.951741218566895, - 24.81438159942627, - 28.878422737121582, - 50.83928394317627, - 82.69517993927002, - 113.43906116485596, - 167.34633350372314, - 228.9335069656372, - 400.28574085235596, - 458.79584217071533 + 99.24609375, + 99.24609375, + 99.24609375, + 99.24609375, + 99.24609375, + 99.24609375, + 99.24609375, + 99.24609375, + 104.984375, + 106.0, + 112.1875, + 117.6015625, + 129.71875, + 140.33203125, + 152.49609375, + 177.30078125, + 215.34375, + 269.90625, + 323.6875, + 408.66796875, + 545.92578125, + 695.8046875, + 939.47265625, + 1238.2265625, + 1702.04296875, + 2192.26953125, + 3068.6796875, + 3920.0234375 ], "PauliPropagation.jl": [ - 0.0062255859375, - 0.0245361328125, - 0.0977783203125, - 0.0977783203125, - 0.3907470703125, - 0.3907470703125, - 1.5626220703125, - 1.5626220703125, - 3.1251220703125, - 6.2501220703125, - 6.2501220703125, - 12.5001220703125, - 25.0001220703125, - 25.0001220703125, - 50.0001220703125, - 50.0001220703125, - 100.0001220703125, - 200.0001220703125, - 200.0001220703125, - 200.0001220703125, - 400.0001220703125 + 554.5078125, + 562.78125, + 563.6015625, + 563.6015625, + 563.6015625, + 565.1171875, + 559.171875, + 578.06640625, + 593.17578125, + 614.23828125, + 638.3359375, + 669.609375, + 714.609375, + 756.48046875, + 860.4921875, + 912.06640625, + 1006.12890625, + 1198.44921875, + 1331.4609375, + 1656.81640625, + 2051.0, + 2623.4375, + 3161.921875, + 4197.296875, + 5691.19921875, + 7193.875, + 8734.47265625, + 11237.98046875 ], "QuEra ppvm": [ - 0.99609375, - 1.09765625, - 1.09765625, - 1.1328125, - 1.3125, - 1.66015625, - 2.46875, - 2.9609375, - 4.42578125, - 7.16796875, - 12.328125, - 17.77734375, - 23.87109375, - 32.74609375, - 45.66015625, - 64.7578125, - 91.03125, - 129.25, - 180.68359375, - 250.69921875, - 348.30859375 + 150.1796875, + 150.1796875, + 150.1796875, + 150.1796875, + 151.12109375, + 151.37890625, + 151.63671875, + 153.69921875, + 158.2734375, + 161.2890625, + 168.16796875, + 174.16796875, + 183.9453125, + 196.8984375, + 217.1328125, + 250.6328125, + 299.640625, + 356.64453125, + 448.66015625, + 548.64453125, + 720.6640625, + 900.65625, + 1138.68359375, + 1530.67578125, + 1960.6875, + 2674.6875, + 3434.6953125, + 4710.6953125 ], "Qiskit pauli-prop": [ - 0.1875, - 0.45703125, - 0.6328125, - 3.01171875, - 3.546875, - 6.04296875, - 9.453125, - 14.8046875, - 20.88671875, - 35.22265625, - 46.765625, - 65.19921875, - 97.421875, - 153.75390625, - 242.32421875, - 398.3203125, - 508.140625, - 667.93359375, - 949.3125, - 1063.67578125, - 1388.71875 + 105.2734375, + 106.00390625, + 106.31640625, + 107.0859375, + 108.11328125, + 110.171875, + 116.05859375, + 127.546875, + 137.8125, + 158.859375, + 182.53515625, + 220.7109375, + 288.05859375, + 391.26171875, + 579.98828125, + 816.6015625, + 1063.75, + 1450.38671875, + 1951.3984375, + 2738.16015625, + 3688.72265625, + 5139.078125, + 6925.5, + 9614.63671875, + 12773.3515625, + 17074.453125, + 23311.59765625, + 30644.8203125 + ], + "cuPauliProp (GPU)": [ + 448.25, + 448.25, + 452.25, + 458.25, + 478.25, + 530.25, + 658.25, + 906.25, + 1230.25, + 1754.25, + 2652.25, + 4046.25, + 6220.25, + 9414.25, + 14098.25, + 21972.25, + 34032.25, + 51172.25, + 74560.25, + 80806.25, + 73070.25, + 80468.25, + 80306.25, + 80012.25, + 80066.25, + 80654.25, + 79984.25, + 80494.25 ] }, - "num_terms": { - "cuPauliProp (GPU)": [ - 120, - 430, - 832, - 1767, - 3507, - 6469, - 11892, - 20924, - 36784, - 62461, - 103654, - 167490, - 265849, - 405649, - 619391, - 908426, - 1321207, - 1891606, - 2675337, - 3779776, - 5300936 + "native_memory": { + "monoprop": [ + 0.021811485290527344, + 0.053984642028808594, + 0.09384822845458984, + 0.19596195220947266, + 0.36931705474853516, + 0.6798334121704102, + 1.1329317092895508, + 2.244696617126465, + 3.527798652648926, + 5.652615547180176, + 8.914322853088379, + 16.2201509475708, + 25.746647834777832, + 39.50025653839111, + 57.21512317657471, + 87.02963733673096, + 125.40126514434814, + 151.31500720977783, + 215.3877305984497, + 325.40620136260986, + 467.08703327178955, + 672.3922700881958, + 962.2750978469849, + 1153.1309022903442, + 1635.0377168655396, + 2350.2291383743286, + 3327.9866762161255, + 3630.8824434280396 + ], + "PauliPropagation.jl": [ + 0.02861785888671875, + 0.07315826416015625, + 0.1643218994140625, + 0.1643218994140625, + 0.3488922119140625, + 0.7205963134765625, + 1.466888427734375, + 2.9627304077148438, + 5.958045959472656, + 9.327789306640625, + 9.327789306640625, + 28.211692810058594, + 28.211692810058594, + 52.738128662109375, + 80.33037567138672, + 80.33037567138672, + 132.37162017822266, + 132.37162017822266, + 232.91806030273438, + 430.03279876708984, + 430.03279876708984, + 651.7868728637695, + 1069.2601928710938, + 1069.2601928710938, + 1538.9176712036133, + 2067.282341003418, + 2997.692581176758, + 2997.692581176758 ], + "cuPauliProp (GPU)": [ + 0.04736328125, + 0.1572265625, + 0.294921875, + 0.61767578125, + 1.21630859375, + 2.18701171875, + 3.9521484375, + 6.82568359375, + 11.75439453125, + 19.64892578125, + 31.93310546875, + 50.68310546875, + 79.15869140625, + 119.01025390625, + 178.7099609375, + 258.55224609375, + 370.36279296875, + 521.47412109375, + 727.70068359375, + 1012.0078125, + 1400.3779296875, + 1931.33984375, + 2657.013671875, + 3634.3740234375, + 4924.080078125, + 6620.19482421875, + 8834.146484375, + 11651.52783203125 + ] + }, + "num_terms": { "monoprop": [ 120, - 430, - 832, - 1768, - 3522, - 6512, - 11979, - 21078, - 36994, - 63049, - 105073, - 170823, - 274222, - 421161, - 649218, - 969166, - 1430033, - 2075465, - 2966901, - 4233793, - 6002213 + 414, + 768, + 1630, + 3255, + 5818, + 10454, + 17934, + 30953, + 51534, + 83801, + 133713, + 209965, + 319328, + 482566, + 710792, + 1032242, + 1465836, + 2056508, + 2871315, + 3982863, + 5509645, + 7585864, + 10386236, + 14099731, + 18948708, + 25241196, + 33309328 ], "PauliPropagation.jl": [ 120, - 430, - 831, - 1765, - 3506, - 6462, - 11886, - 20914, - 36810, - 62448, - 103589, - 167411, - 265753, - 405091, - 618061, - 906588, - 1318324, - 1887517, - 2670435, - 3774991, - 5295608 + 414, + 779, + 1641, + 3242, + 5837, + 10554, + 18235, + 31461, + 52522, + 85320, + 135506, + 211626, + 317843, + 476925, + 689973, + 988078, + 1390964, + 1942287, + 2701851, + 3740573, + 5160256, + 7101940, + 9718162, + 13169143, + 17708646, + 23635470, + 31177867 ], "QuEra ppvm": [ 4, 203, - 483, - 1035, - 2335, - 4582, - 8182, - 14694, - 26055, - 45190, - 76374, - 124092, - 203066, - 315571, - 479217, - 722068, - 1052355, - 1541676, - 2189107, - 3080583, - 4332743 + 469, + 1009, + 2219, + 4221, + 7488, + 13129, + 22826, + 38886, + 64629, + 103409, + 166019, + 253342, + 378146, + 560772, + 805621, + 1161443, + 1624827, + 2256599, + 3122764, + 4258841, + 5765752, + 7771081, + 10461746, + 14030516, + 18743094, + 24878659 ], "Qiskit pauli-prop": [ 120, - 430, - 832, - 1768, - 3522, - 6512, - 11979, - 21078, - 36994, - 63049, - 105073, - 170823, - 274222, - 421161, - 649218, - 969166, - 1430033, - 2075465, - 2966901, - 4233793, - 6002213 + 414, + 768, + 1630, + 3255, + 5818, + 10454, + 17934, + 30953, + 51534, + 83801, + 133713, + 209965, + 319328, + 482566, + 710792, + 1032242, + 1465836, + 2056508, + 2871315, + 3982863, + 5509645, + 7585864, + 10386236, + 14099731, + 18948708, + 25241196, + 33309328 + ], + "cuPauliProp (GPU)": [ + 120, + 414, + 780, + 1643, + 3244, + 5843, + 10562, + 18249, + 31430, + 52548, + 85406, + 135554, + 211720, + 318312, + 477990, + 691552, + 990613, + 1394792, + 1946394, + 2706837, + 3745619, + 5165796, + 7106774, + 9720944, + 13170557, + 17707199, + 23628909, + 31164633 ] } -} +} \ No newline at end of file diff --git a/benches/third_party/pauli_prop/run_model.jl b/benches/third_party/pauli_prop/run_model.jl index 2c9e20fe..9639bf77 100644 --- a/benches/third_party/pauli_prop/run_model.jl +++ b/benches/third_party/pauli_prop/run_model.jl @@ -16,6 +16,8 @@ using PauliPropagation using JSON using ProgressMeter +include(joinpath(@__DIR__, "..", "bench_common.jl")) + settings = JSON.parsefile(joinpath(@__DIR__, "settings.json")) nx, ny = settings["nx"], settings["ny"] @@ -41,15 +43,19 @@ append!(step_parameters, fill(theta_zz, length(topology))) append!(step_parameters, fill(theta_z, nq)) append!(step_parameters, fill(theta_x, nq)) -pauli_sum = PauliSum(nq) +pauli_sum = VectorPauliSum(PauliSum(nq)) add!(pauli_sum, [:Z, :Z], collect(obs_qubits), 1.0) num_terms = Int[] runtime = Float64[] memory = Float64[] +native_memory = Float64[] expvals = Float64[] +window = HighWaterMark() + @showprogress for (step_idx, num_steps) in enumerate(step_range) + start!(window) t1 = time_ns() global pauli_sum = propagate( step_circuit, pauli_sum, step_parameters; @@ -57,12 +63,14 @@ expvals = Float64[] ) expval = overlapwithzero(pauli_sum) t2 = time_ns() + stop!(window) if step_idx > 1 push!(runtime, (t2 - t1) / 1e9) end push!(num_terms, length(pauli_sum)) - push!(memory, Base.summarysize(pauli_sum) / 1024^2) + push!(memory, peak_mb(window)) + push!(native_memory, Base.summarysize(pauli_sum) / 1024^2) push!(expvals, expval) end @@ -71,6 +79,8 @@ data = JSON.parsefile(results_file) data["num_terms"]["PauliPropagation.jl"] = num_terms data["runtime"]["PauliPropagation.jl"] = runtime data["memory"]["PauliPropagation.jl"] = memory +haskey(data, "native_memory") || (data["native_memory"] = Dict()) +data["native_memory"]["PauliPropagation.jl"] = native_memory data["expvals"]["PauliPropagation.jl"] = expvals open(results_file, "w") do file diff --git a/benches/third_party/pauli_prop/run_model.py b/benches/third_party/pauli_prop/run_model.py index 4b8745c7..b96f67cf 100644 --- a/benches/third_party/pauli_prop/run_model.py +++ b/benches/third_party/pauli_prop/run_model.py @@ -83,6 +83,9 @@ def main() -> None: "num_terms": {r.label: r.num_terms for r in results.values()}, "runtime": {r.label: r.runtime[1:] for r in results.values()}, "memory": {r.label: r.memory for r in results.values()}, + "native_memory": { + r.label: r.operator_memory for r in results.values() if r.operator_memory + }, "expvals": {r.label: r.expvals for r in results.values()}, } # Preserve any backend already in the file (e.g. PauliPropagation.jl from an earlier diff --git a/benches/third_party/pauli_prop/run_one.py b/benches/third_party/pauli_prop/run_one.py index 25c948ef..f42d0471 100644 --- a/benches/third_party/pauli_prop/run_one.py +++ b/benches/third_party/pauli_prop/run_one.py @@ -87,6 +87,11 @@ def main() -> None: label = backend_mod.LABELS[args.backend] print(f"[{label}] {settings.describe()}", flush=True) + # Taken before the first measurement window opens, because opening one resets + # ru_maxrss too (both are the kernel's mm->hiwater_rss). This covers the import and + # setup phase; the run's own peaks come from the per-step windows. + setup_peak_mb = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1024 + if args.backend == "monoprop": result = backend_mod.run_monoprop(settings) elif args.backend == "ppvm": @@ -103,10 +108,9 @@ def main() -> None: else: # unreachable: argparse constrains the choices raise SystemExit(f"unknown backend {args.backend}") - # ru_maxrss is KiB on Linux. This is the whole process, interpreter and imported - # libraries included, so it is a ceiling on the operator footprint, not the - # operator footprint itself. - peak_rss_mb = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1024 + # Peak over the whole run: the largest per-step window, or the setup phase if the + # workload never exceeded it. ru_maxrss cannot be used here (see setup_peak_mb). + peak_rss_mb = max(setup_peak_mb, *result.memory) record = { "backend": args.backend, @@ -126,6 +130,7 @@ def main() -> None: "final_memory_MB": result.memory[-1], "memory_metric": result.memory_metric, "operator_memory_MB": result.operator_memory_mb, + "operator_memory_metric": result.operator_memory_metric, "peak_rss_MB": peak_rss_mb, "final_num_terms": int(result.num_terms[-1]), "final_expval": float(result.expvals[-1]), diff --git a/benches/third_party/pauli_prop/run_scaling.jl b/benches/third_party/pauli_prop/run_scaling.jl index 165f932f..182d0a33 100644 --- a/benches/third_party/pauli_prop/run_scaling.jl +++ b/benches/third_party/pauli_prop/run_scaling.jl @@ -19,6 +19,8 @@ using PauliPropagation using JSON +include(joinpath(@__DIR__, "..", "bench_common.jl")) + function parse_args(args) opts = Dict{String,Any}( "nx" => nothing, "ny" => nothing, "step-max" => nothing, @@ -76,6 +78,7 @@ step_range = settings["step_min"]:settings["step_size"]:step_max runtimes = Float64[] num_terms = Int[] memory = Float64[] +operator_memory = Float64[] expvals = Float64[] println("[PauliPropagation.jl] $(nx)x$(ny) ($nq qubits), dt=$dt, atol=$lower_atol, " * @@ -83,14 +86,24 @@ println("[PauliPropagation.jl] $(nx)x$(ny) ($nq qubits), dt=$dt, atol=$lower_ato "threads=$(Threads.nthreads()), v$(pkgversion(PauliPropagation))") flush(stdout) +# Same instrument as the Python arm (backends._run_steps), so the two are comparable. +window = HighWaterMark() + +# Taken before the first window opens: start! resets the kernel's mm->hiwater_rss, which is +# what Sys.maxrss() reports, so it stops being a whole-run ceiling from that point on. +setup_peak_mb = Sys.maxrss() / 1024^2 + for _ in step_range + start!(window) t1 = time_ns() global pauli_sum = advance!(pauli_sum) expval = overlapwithzero(pauli_sum) t2 = time_ns() + stop!(window) push!(runtimes, (t2 - t1) / 1e9) push!(num_terms, length(pauli_sum)) - push!(memory, Base.summarysize(pauli_sum) / 1024^2) + push!(memory, peak_mb(window)) + push!(operator_memory, Base.summarysize(pauli_sum) / 1024^2) push!(expvals, expval) end @@ -108,9 +121,10 @@ record = Dict( "total_runtime_excl_first_s" => sum(runtimes[2:end]), "final_step_s" => runtimes[end], "final_memory_MB" => memory[end], - "memory_metric" => "Base.summarysize of the Pauli sum", - "operator_memory_MB" => memory[end], - "peak_rss_MB" => Sys.maxrss() / 1024^2, + "memory_metric" => "peak process RSS over the step (kernel VmHWM)", + "operator_memory_MB" => operator_memory[end], + "operator_memory_metric" => "Base.summarysize of the Pauli sum", + "peak_rss_MB" => max(setup_peak_mb, maximum(memory)), "final_num_terms" => num_terms[end], "final_expval" => expvals[end], "max_terms_budget" => nothing, diff --git a/benches/third_party/pauli_prop/settings.json b/benches/third_party/pauli_prop/settings.json index b847f655..5e46f86d 100644 --- a/benches/third_party/pauli_prop/settings.json +++ b/benches/third_party/pauli_prop/settings.json @@ -1,12 +1,12 @@ { - "nx": 6, - "ny": 6, + "nx": 12, + "ny": 12, "hx": 1.0, "hz": 1.0, "j": 1.5, "dt": 0.05, "step_min": 0, - "step_max": 40, + "step_max": 55, "step_size": 2, "lower_atol": 1e-6, "cutoff": null, diff --git a/docs/content/docs/benchmarks.mdx b/docs/content/docs/benchmarks.mdx index 86184e5e..05899025 100644 --- a/docs/content/docs/benchmarks.mdx +++ b/docs/content/docs/benchmarks.mdx @@ -105,7 +105,8 @@ Each line of both JSONL files is one benchmark run (one system size / circuit de - `num_terms`: number of Majorana terms kept in the operator at the final layer. - `final_overlap`: expectation value at the final layer. - `runtime_seconds`: wall-clock time to run all `n_layers` layers. -- `memory_MB`: operator memory footprint at the final layer, in megabytes. +- `memory_MB`: peak resident set size over the layer, in megabytes, read from the kernel's + `VmHWM` high-water mark and reset per layer. - `num_threads`: thread count the run was given (`monoprop_NUM_THREADS` / `JULIA_NUM_THREADS`). - `cpu_seconds`: CPU time consumed over the timed loop, summed over all threads. - `busy_cores`: `cpu_seconds / runtime_seconds` — how many cores the run actually kept busy. @@ -261,11 +262,15 @@ For every engine, `results.json` collects, indexed by Trotter step: - `runtime`: wall-clock time per step, in seconds. The first step is excluded: it carries each engine's warm-up (JIT compilation, first-touch allocation), which is not what the per-step comparison is measuring. -- `memory`: the memory footprint of the evolving operator, in megabytes, for every step. Where an - engine exposes its own accounting this is exact (monoprop's C++ operator-memory accounting, - cuPauliProp's cupy device memory pool, `PauliPropagation.jl`'s `Base.summarysize` of the Pauli - sum); `QuEra ppvm` and `Qiskit pauli-prop` expose no such accounting, so their footprint is - reconstructed by accumulating this process's host-memory growth across each of their own steps. +- `memory`: peak resident set size over each step, in megabytes. Every engine is measured the + same way, by the kernel's `VmHWM` high-water mark reset at the start of each step, so these + curves are directly comparable. It is a whole-process figure, so it includes the interpreter + and the loaded libraries; read the growth across steps rather than the absolute value. +- `native_memory`: where an engine accounts for itself, its own figure (monoprop's C++ + operator-memory accounting, cuPauliProp's cupy device memory pool, `PauliPropagation.jl`'s + `Base.summarysize` of the Pauli sum). These are reference values only: they measure + different things and must not be compared across engines. `QuEra ppvm` and `Qiskit + pauli-prop` expose no such accounting and so report none. - `expvals`: the `ZZ` expectation value on `obs_qubits`, for every step. ### 5. Plot the results @@ -333,9 +338,9 @@ caveats belong with these ratios: `Performance` submodule. That path leaves duplicate Pauli strings unmerged, so it reports ~38% more terms than the exact `propagate` path and its expectation value differs by ~1e-4 — its term count is a storage count, not an operator size. -- **The memory figure is not one quantity.** `monoprop` and `cuPauliProp` report their own - operator accounting, `PauliPropagation.jl` its `Base.summarysize`, and ppvm and Qiskit only a - process-RSS proxy, because they expose nothing else. +- **The memory figure is the peak resident set size over each step**, taken from the kernel for + every engine alike, so the curves are comparable. Each engine's own accounting, where it has + one, is reported separately as `native_memory` and is *not* comparable across engines. #### SPECS The benchmark is run on [Leonardo (CINECA)](https://docs.hpc.cineca.it/hpc/leonardo.html): diff --git a/docs/public/benchmarks/majorana_results.png b/docs/public/benchmarks/majorana_results.png index 86d4d169..56ea268f 100644 Binary files a/docs/public/benchmarks/majorana_results.png and b/docs/public/benchmarks/majorana_results.png differ diff --git a/docs/public/benchmarks/pauli_results.png b/docs/public/benchmarks/pauli_results.png index 54f0e665..b78f7fae 100644 Binary files a/docs/public/benchmarks/pauli_results.png and b/docs/public/benchmarks/pauli_results.png differ diff --git a/tests/test_bench_memory.py b/tests/test_bench_memory.py index 694dfdc9..dc956be9 100644 --- a/tests/test_bench_memory.py +++ b/tests/test_bench_memory.py @@ -22,10 +22,14 @@ from __future__ import annotations +import gc +import resource + import pytest from _memory import ( HighWaterMark, PssSampler, + heap_trim, merge_peak_of_sum, peak_rss_bytes, pss_bytes, @@ -126,3 +130,26 @@ def test_peak_rss_never_below_current_rss() -> None: if peak_rss_bytes() == 0: pytest.skip("/proc/self/status VmHWM unavailable (non-Linux)") assert peak_rss_bytes() >= rss_bytes() + + +def test_reset_also_clears_ru_maxrss() -> None: + """``ru_maxrss`` shares ``mm->hiwater_rss`` with ``VmHWM``, so a window reset drops it. + + Benchmarks that record a whole-run ceiling alongside per-step windows have to take the + maximum over the windows instead; this pins the behaviour that forces that. + """ + if not reset_peak_rss(): + pytest.skip("/proc/self/clear_refs unavailable (non-Linux or kernel < 4.0)") + + blob = bytearray(80 * MIB) + for i in range(0, len(blob), 4096): + blob[i] = 1 + before = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss * 1024 + del blob + gc.collect() + heap_trim() + reset_peak_rss() + after = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss * 1024 + + assert before >= 70 * MIB + assert after < before