Skip to content

Commit

Permalink
Update graph images
Browse files Browse the repository at this point in the history
  • Loading branch information
@ArtiomTr
ArtiomTr committed May 30, 2024
1 parent 6dcb9ac commit 17e8437
Show file tree
Hide file tree
Showing 29 changed files with 301 additions and 94 deletions.
1 change: 1 addition & 0 deletions .gitignore
View file
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
.idea
.directory
target
benchmark_script/output
133 changes: 133 additions & 0 deletions benchmark_script/input/graphs/eip_graph_template.tex
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,133 @@
\documentclass[border=2pt]{standalone}
\usepackage{graphicx}
\usepackage{pgfplots}
\usepackage{subfig}
\usepackage{pdflscape}
\usepackage[export]{adjustbox}
\begin{document}
\hbox{
\adjustbox{valign=t}{
\begin{tikzpicture}
\begin{axis}[
legend style={at={(0.5, -0.3)}, anchor=north, legend cell align=left},
symbolic x coords={1 core, 2 cores, 4 cores, 8 cores, 16 cores},
xtick=data,
x tick label style={rotate=45, anchor=east},
ymin=0, ymax=${max_time}, %%% change this
ylabel near ticks,
yticklabel pos=right,
ylabel={time (${time_unit})}, %%% change this
grid = major,
grid style=dashed,
]
%%% rust-kzg with zkcrypto backend
\addlegendentry{rust-kzg with zkcrypto backend}
\addplot[mark=*, mark options={scale=1.5}, thick, black] coordinates {
(1 core, ${1>rust-kzg with zkcrypto backend (parallel)})
(2 cores, ${2>rust-kzg with zkcrypto backend (parallel)})
(4 cores, ${4>rust-kzg with zkcrypto backend (parallel)})
(8 cores, ${8>rust-kzg with zkcrypto backend (parallel)})
(16 cores, ${16>rust-kzg with zkcrypto backend (parallel)}) };
%%% rust-kzg with blst backend
\addlegendentry{rust-kzg with blst backend (BGMW)}
\addplot[mark=*, mark options={scale=1.5}, thick, green] coordinates {
(1 core, ${1>rust-kzg with blst backend (parallel, bgmw)})
(2 cores, ${2>rust-kzg with blst backend (parallel, bgmw)})
(4 cores, ${4>rust-kzg with blst backend (parallel, bgmw)})
(8 cores, ${8>rust-kzg with blst backend (parallel, bgmw)})
(16 cores, ${16>rust-kzg with blst backend (parallel, bgmw)}) };
\addlegendentry{rust-kzg with constantine backend (BGMW)}
\addplot[mark=*, mark options={scale=1.5}, thick, orange] coordinates {
(1 core, ${1>rust-kzg with constantine backend (parallel)})
(2 cores, ${2>rust-kzg with constantine backend (parallel)})
(4 cores, ${4>rust-kzg with constantine backend (parallel)})
(8 cores, ${8>rust-kzg with constantine backend (parallel)})
(16 cores, ${16>rust-kzg with constantine backend (parallel)}) };
\addlegendentry{rust-kzg with arkworks backend (BGMW)}
\addplot[mark=*, mark options={scale=1.5}, thick, red] coordinates {
(1 core, ${1>rust-kzg with arkworks backend (parallel, bgmw)})
(2 cores, ${2>rust-kzg with arkworks backend (parallel, bgmw)})
(4 cores, ${4>rust-kzg with arkworks backend (parallel, bgmw)})
(8 cores, ${8>rust-kzg with arkworks backend (parallel, bgmw)})
(16 cores, ${16>rust-kzg with arkworks backend (parallel, bgmw)}) };
%%% rust-kzg with mcl backend
%\addlegendentry{rust-kzg with mcl backend}
%\addplot[mark=*, mark options={scale=1.5}, thick, black] coordinates {
% (1 core, 236.06)
% (2 cores, 113.92)
% (4 cores, 65.031)
% (8 cores, 48.163)
% (16 cores, 33.903) };
\end{axis}
\end{tikzpicture}
}%
\adjustbox{valign=t}{
\begin{tikzpicture}
\begin{axis}[
legend style={at={(0.5, -0.3)}, anchor=north, legend cell align=left},
symbolic x coords={1 core, 2 cores, 4 cores, 8 cores, 16 cores},
xtick=data,
x tick label style={rotate=45, anchor=east},
ymin=0, ymax=${max_time_2},
ylabel near ticks,
yticklabel pos=right,
ylabel={time (${time_unit})},
grid = major,
grid style=dashed,
]
%%% rust-kzg with blst backend
\addlegendentry{rust-kzg with blst backend (BGMW)}
\addplot[mark=*, mark options={scale=1.5}, thick, green] coordinates {
(1 core, ${1>rust-kzg with blst backend (parallel, bgmw)})
(2 cores, ${2>rust-kzg with blst backend (parallel, bgmw)})
(4 cores, ${4>rust-kzg with blst backend (parallel, bgmw)})
(8 cores, ${8>rust-kzg with blst backend (parallel, bgmw)})
(16 cores, ${16>rust-kzg with blst backend (parallel, bgmw)}) };
\addlegendentry{rust-kzg with constantine backend (BGMW)}
\addplot[mark=*, mark options={scale=1.5}, thick, orange] coordinates {
(1 core, ${1>rust-kzg with constantine backend (parallel, bgmw)})
(2 cores, ${2>rust-kzg with constantine backend (parallel, bgmw)})
(4 cores, ${4>rust-kzg with constantine backend (parallel, bgmw)})
(8 cores, ${8>rust-kzg with constantine backend (parallel, bgmw)})
(16 cores, ${16>rust-kzg with constantine backend (parallel, bgmw)}) };
%%% go-kzg-4844 (VerifyBlobKZGProofBatch)
\addlegendentry{go-kzg-4844} % TODO: pick new name
\addplot[mark=*, mark options={scale=1.5}, thick, cyan] coordinates {
(1 core, ${1>go-kzg-4844})
(2 cores, ${2>go-kzg-4844})
(4 cores, ${4>go-kzg-4844})
(8 cores, ${8>go-kzg-4844})
(16 cores, ${16>go-kzg-4844}) };
%%% rust binding (c-kzg-4844)
\addlegendentry{rust binding (c-kzg-4844)}
\addplot[mark=*, mark options={scale=1.5}, thick, purple] coordinates {
(1 core, ${1>rust binding (c-kzg-4844)})
(2 cores, ${2>rust binding (c-kzg-4844)})
(4 cores, ${4>rust binding (c-kzg-4844)})
(8 cores, ${8>rust binding (c-kzg-4844)})
(16 cores, ${16>rust binding (c-kzg-4844)}) };
%%% rust binding (rust-kzg with blst backend)
\addlegendentry{rust binding (rust-kzg with blst backend)}
\addplot[mark=*, mark options={scale=1.5}, thick, pink] coordinates {
(1 core, ${1>rust binding (rust-kzg with blst backend)})
(2 cores, ${2>rust binding (rust-kzg with blst backend)})
(4 cores, ${4>rust binding (rust-kzg with blst backend)})
(8 cores, ${8>rust binding (rust-kzg with blst backend)})
(16 cores, ${16>rust binding (rust-kzg with blst backend)}) };
%%% go binding (rust-kzg with blst backend)
\addlegendentry{go binding (rust-kzg with blst backend)}
\addplot[mark=*, mark options={scale=1.5}, thick, blue] coordinates {
(1 core, ${1>go binding (rust-kzg with blst backend)})
(2 cores, ${2>go binding (rust-kzg with blst backend)})
(4 cores, ${4>go binding (rust-kzg with blst backend)})
(8 cores, ${8>go binding (rust-kzg with blst backend)})
(16 cores, ${16>go binding (rust-kzg with blst backend)}) };
\end{axis}
\end{tikzpicture}
}
}
\end{document}
156 changes: 155 additions & 1 deletion benchmark_script/main.py
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import re
import pandas as pd
import subprocess

normalize_benchmark_name = {
'blob_to_kzg_commitment': 'blob_to_kzg_commitment',
Expand Down Expand Up @@ -141,19 +142,172 @@ def parse_thread_group(file, threads):

return group

def replace_path(match, resolve_var):
path = match.group(1)

result = resolve_var(path.split(">"))

return str(result)


def expand_template(in_filename, out_filename, resolve_var):
with open(in_filename, "r") as in_file, open(out_filename, "w") as out_file:
for line in in_file:
out_line = re.sub(r'\$\{([^}]+)\}', lambda match: replace_path(match, resolve_var), line)
out_file.write(out_line)

def generate_eip_graph(out_filename, data, criteria, time_unit):
max_time = None
max_time_2 = None
for cores in data:
for backend in ['rust-kzg with blst backend (parallel, bgmw)', 'rust-kzg with zkcrypto backend (parallel)']:
result = None
for i in criteria:
if i in data[cores][backend]:
result = data[cores][backend][i]
break;
if max_time is None and not (result is None):
max_time = result
if not (max_time is None) and not (result is None) and max_time < result:
max_time = result
for backend in ['rust-kzg with blst backend (parallel, bgmw)', 'go-kzg-4844', 'rust binding (c-kzg-4844)', 'rust binding (rust-kzg with blst backend)', 'go binding (rust-kzg with blst backend)']:
result = None
for i in criteria:
if i in data[cores][backend]:
result = data[cores][backend][i]
break;
if max_time_2 is None and not (result is None):
max_time_2 = result
if not (max_time_2 is None) and not (result is None) and max_time_2 < result:
max_time_2 = result

time_scale = None
if time_unit == 'ms':
time_scale = 1 / 1000000
elif time_unit == 's':
time_scale = 1 / 1000000000

max_time *= time_scale
max_time *= 1.1
max_time_2 *= time_scale
max_time_2 *= 1.1

def resolve_var(path):
if len(path) == 1:
if path[0] == 'max_time_2':
return max_time_2
if path[0] == 'max_time':
return max_time
if path[0] == 'time_unit':
return time_unit

obj = data
for segment in path:
obj = obj[segment]

result = None
for i in criteria:
if i in obj:
result = obj[i]
break
return result * time_scale

expand_template("./input/graphs/eip_graph_template.tex", f"./output/{out_filename}.tex", resolve_var)
status = subprocess.call(f"wsl xelatex -output-directory=./output ./output/{out_filename}.tex", shell=True)
if status != 0:
print("Failed to generate graph pdf")
return
status = subprocess.call(f"wsl convert -verbose -density 300 -trim ./output/{out_filename}.pdf -quality 100 -flatten ./output/{out_filename}.jpg")
if status != 0:
print("Failed to convert graph pdf to image")
return


def main():
with open("./input/rust-kzg-benchmarks.txt", "r") as bench_results, pd.ExcelWriter("./output/results.ods", mode="w", engine="odf") as output_writer:
with open("./input/rust-kzg-benchmarks2.txt", "r") as bench_results, pd.ExcelWriter("./output/results.ods", mode="w", engine="odf") as output_writer:
groups = {}

line = bench_results.readline()
while len(line) > 0:
result = re.match(r"^\*+ BENCHMARKING ON (\d+) CORES \*+$", line)
if not (result is None):
print("bench", result.group(1))
group = parse_thread_group(bench_results, result.group(1))
groups[result.group(1)] = group
df = pd.DataFrame(data=group)
df.to_excel(output_writer, sheet_name=f"{result.group(1)} cores")

line = bench_results.readline()

# additional aggregation sheets
aggr_groups = [
['blob_to_kzg_commitment'],
['compute_kzg_proof'],
['compute_blob_kzg_proof'],
['verify_kzg_proof'],
['verify_blob_kzg_proof'],
['verify_blob_kzg_proof_batch/1', 'verify_blob_kzg_proof_batch/1 (sequential)'], # fallback to sequential version
['verify_blob_kzg_proof_batch/2', 'verify_blob_kzg_proof_batch/2 (sequential)'],
['verify_blob_kzg_proof_batch/4', 'verify_blob_kzg_proof_batch/4 (sequential)'],
['verify_blob_kzg_proof_batch/8', 'verify_blob_kzg_proof_batch/8 (sequential)'],
['verify_blob_kzg_proof_batch/16', 'verify_blob_kzg_proof_batch/16 (sequential)'],
['verify_blob_kzg_proof_batch/32', 'verify_blob_kzg_proof_batch/32 (sequential)'],
['verify_blob_kzg_proof_batch/64', 'verify_blob_kzg_proof_batch/64 (sequential)'],
["bench_DAS_extension scale: '15'"],
["bench_fft_fr scale: '15'"],
["bench_fft_g1 scale: '15'"],
["bench_fk_single_da scale: '14'"],
["bench_fk_multi_da scale: '14'"],
["bench_commit_to_poly scale: '15'"],
["bench_compute_proof_single scale: '15'"],
["bench_g1_lincomb with precomputation points: '4096'", "bench_g1_lincomb points: '4096'"], # fallback to lincomb without precomputation
["bench_new_poly_div scale: '15'"],
["bench_recover scale: '15'"],
["bench_zero_poly scale: '15'"],
]

aggregated_data = {}

for aggrs in aggr_groups:
aggregated_data[aggrs[0]] = {}
for group in groups:
for backend in groups[group]:
result = None
for aggr in aggrs:
if aggr in groups[group][backend]:
result = groups[group][backend][aggr]
break
if result is None:
print("Skipping \"" + aggr + "\" for " + backend + "backend")
continue
if not (backend in aggregated_data[aggrs[0]]):
aggregated_data[aggrs[0]][backend] = {}
aggregated_data[aggrs[0]][backend][group] = result
for aggregate in aggregated_data:
df = pd.DataFrame(data=aggregated_data[aggregate])
df.to_excel(output_writer, sheet_name=aggregate)

eip_graphs = [
('blob_to_kzg_commitment', ['blob_to_kzg_commitment']),
('blob_to_kzg_commitment', ['blob_to_kzg_commitment']),
('compute_kzg_proof', ['compute_kzg_proof']),
('compute_blob_kzg_proof', ['compute_blob_kzg_proof']),
('verify_kzg_proof', ['verify_kzg_proof']),
('verify_blob_kzg_proof', ['verify_blob_kzg_proof']),
('verify_blob_kzg_proof_batch_1', ['verify_blob_kzg_proof_batch/1', 'verify_blob_kzg_proof_batch/1 (sequential)']), # fallback to sequential version
('verify_blob_kzg_proof_batch_2', ['verify_blob_kzg_proof_batch/2', 'verify_blob_kzg_proof_batch/2 (sequential)']),
('verify_blob_kzg_proof_batch_4', ['verify_blob_kzg_proof_batch/4', 'verify_blob_kzg_proof_batch/4 (sequential)']),
('verify_blob_kzg_proof_batch_8', ['verify_blob_kzg_proof_batch/8', 'verify_blob_kzg_proof_batch/8 (sequential)']),
('verify_blob_kzg_proof_batch_16', ['verify_blob_kzg_proof_batch/16', 'verify_blob_kzg_proof_batch/16 (sequential)']),
('verify_blob_kzg_proof_batch_32', ['verify_blob_kzg_proof_batch/32', 'verify_blob_kzg_proof_batch/32 (sequential)']),
('verify_blob_kzg_proof_batch_64', ['verify_blob_kzg_proof_batch/64', 'verify_blob_kzg_proof_batch/64 (sequential)']),
]

for (graph_name, criteria) in eip_graphs:
generate_eip_graph(graph_name, groups, criteria, 'ms')




if __name__ == '__main__':
main()
Loading

0 comments on commit 17e8437

Please sign in to comment.