dib-lab · ctb · Aug 20, 2023 · Aug 20, 2023 · Aug 21, 2023 · Aug 21, 2023
diff --git a/01.basic-benchmarks.ipynb b/01.basic-benchmarks.ipynb
diff --git a/02.basic-benchmarks-seaborn.ipynb b/02.basic-benchmarks-seaborn.ipynb
diff --git a/README.md b/README.md
@@ -1,4 +1,26 @@
-## command to get file sizes
+# Benchmarks for branchwater paper
+
+This repo contains the benchmarking code for
+
+[Sourmash Branchwater Enables Lightweight Petabyte-Scale Sequence Search](https://dib-lab.github.io/2022-paper-branchwater-software/) ([github repo](https://github.com/dib-lab/2022-paper-branchwater-software/)), Irber et al., 2022.
+
+[doi: 10.1101/2022.11.02.514947](https://www.biorxiv.org/content/10.1101/2022.11.02.514947v1.full)
+
+## Running
+
+Allocate 64 threads and 64 GB:
+```!
+srun -p high2 --time=72:00:00 --nodes=1 --cpus-per-task 64 --mem 80GB --pty bash
+```
+
+then run the snakemake workflow:
+```
+snakemake -c 64 -p -s benchmarking.snakefile  --resources only_one_job=1 -k
+```
+
+## appendix
+
+### command to get file sizes
 
 ```
 find . -type f -printf "%k %p\n" | tee ~/scratch/magsearch/ls-wort.txt

diff --git a/benchmarking.snakefile b/benchmarking.snakefile
@@ -1,4 +1,5 @@
-sra_search_bin="/home/ctbrown/scratch/magsearch/bin/searcher"
+#sra_search_bin="/home/ctbrown/scratch/magsearch/bin/searcher"
+manysearch_cmd = "sourmash scripts manysearch"
 
 rule all:
     input:
@@ -10,163 +11,168 @@ rule all:
         expand("benchmarks/a_vs_a_1000_t{t}.txt", t=[4,8,16]),
         expand("benchmarks/a_vs_catalog.txt"),
 
+rule threads:
+    input:
+        expand("benchmarks/a_vs_a_1000_t{t}.txt", t=[4,8,16,24,32,40,48,56,64]),
+
+
 rule a_vs_a:
     input:
-        bin=sra_search_bin,
         queries="data/gtdb-list-a.sigs.txt",
         against="data/wort-list-a.txt",
     output:
         csv="outputs/output_a_vs_a.csv",
+    resources:
+        only_one_job=1,
     benchmark:
         "benchmarks/a_vs_a.txt"
     threads: 32
     shell: """
-        export RAYON_NUM_THREADS={threads}
-        {input.bin} -k 31 --scaled=1000 -o {output.csv} \
-            {input.queries} {input.against}
+        {manysearch_cmd} -k 31 --scaled=1000 -o {output.csv} \
+            {input.queries} {input.against} -c {threads}
     """
 
 rule a_sub_vs_a:
     input:
-        bin=sra_search_bin,
         queries="data/gtdb-list-a-{n}.sigs.txt",
         against="data/wort-list-a.txt",
     output:
         csv="outputs/output_a_{n}_vs_a.csv",
+    resources:
+        only_one_job=1,
     benchmark:
         "benchmarks/a_{n}_vs_a.txt"
     threads: 32
     shell: """
-        export RAYON_NUM_THREADS={threads}
-        {input.bin} -k 31 --scaled=1000 -o {output.csv} \
-            {input.queries} {input.against}
+        {manysearch_cmd} -k 31 --scaled=1000 -o {output.csv} \
+            {input.queries} {input.against} -c {threads}
     """
 
 rule a_vs_a_sub:
     input:
-        bin=sra_search_bin,
         queries="data/gtdb-list-a.sigs.txt",
         against="data/wort-list-a-{n}.txt",
     output:
         csv="outputs/output_a_vs_a_{n}.csv",
+    resources:
+        only_one_job=1,
     benchmark:
         "benchmarks/a_vs_a_{n}.txt"
     threads: 32
     shell: """
-        export RAYON_NUM_THREADS={threads}
-        {input.bin} -k 31 --scaled=1000 -o {output.csv} \
-            {input.queries} {input.against}
+        {manysearch_cmd} -k 31 --scaled=1000 -o {output.csv} \
+            {input.queries} {input.against} -c {threads}
     """
 
 rule a_vs_b:
     input:
-        bin=sra_search_bin,
         queries="data/gtdb-list-a.sigs.txt",
         against="data/wort-list-b.txt",
     output:
         csv="outputs/output_a_vs_b.csv",
+    resources:
+        only_one_job=1,
     benchmark:
         "benchmarks/a_vs_b.txt"
     threads: 32
     shell: """
-        export RAYON_NUM_THREADS={threads}
-        {input.bin} -k 31 --scaled=1000 -o {output.csv} \
-            {input.queries} {input.against}
+        {manysearch_cmd} -k 31 --scaled=1000 -o {output.csv} \
+            {input.queries} {input.against} -c {threads}
     """
 
 rule a_vs_c:
     input:
-        bin=sra_search_bin,
         queries="data/gtdb-list-a.sigs.txt",
         against="data/wort-list-c.txt",
     output:
         csv="outputs/output_a_vs_c.csv",
+    resources:
+        only_one_job=1,
     benchmark:
         "benchmarks/a_vs_c.txt"
     threads: 32
     shell: """
-        export RAYON_NUM_THREADS={threads}
-        {input.bin} -k 31 --scaled=1000 -o {output.csv} \
-            {input.queries} {input.against}
+        {manysearch_cmd} -k 31 --scaled=1000 -o {output.csv} \
+            {input.queries} {input.against} -c {threads}
     """
 
 rule a_vs_d:
     input:
-        bin=sra_search_bin,
         queries="data/gtdb-list-a.sigs.txt",
         against="data/wort-list-d.txt",
     output:
         csv="outputs/output_a_vs_d.csv",
+    resources:
+        only_one_job=1,
     benchmark:
         "benchmarks/a_vs_d.txt"
     threads: 32
     shell: """
-        export RAYON_NUM_THREADS={threads}
-        {input.bin} -k 31 --scaled=1000 -o {output.csv} \
-            {input.queries} {input.against}
+        {manysearch_cmd} -k 31 --scaled=1000 -o {output.csv} \
+            {input.queries} {input.against} -c {threads}
     """
 
 rule a_vs_e:
     input:
-        bin=sra_search_bin,
         queries="data/gtdb-list-a.sigs.txt",
         against="data/wort-list-e.txt",
     output:
         csv="outputs/output_a_vs_e.csv",
+    resources:
+        only_one_job=1,
     benchmark:
         "benchmarks/a_vs_e.txt"
     threads: 32
     shell: """
-        export RAYON_NUM_THREADS={threads}
-        {input.bin} -k 31 --scaled=1000 -o {output.csv} \
-            {input.queries} {input.against}
+        {manysearch_cmd} -k 31 --scaled=1000 -o {output.csv} \
+            {input.queries} {input.against} -c {threads}
     """
 
 rule a_vs_largest:
     input:
-        bin=sra_search_bin,
         queries="data/gtdb-list-a.sigs.txt",
         against="data/wort-largest-10k.txt",
     output:
         csv="outputs/output_a_vs_largest_10k.csv",
+    resources:
+        only_one_job=1,
     benchmark:
         "benchmarks/a_vs_largest_10k.txt"
     threads: 32
     shell: """
-        export RAYON_NUM_THREADS={threads}
-        {input.bin} -k 31 --scaled=1000 -o {output.csv} \
-            {input.queries} {input.against}
+        {manysearch_cmd} -k 31 --scaled=1000 -o {output.csv} \
+            {input.queries} {input.against} -c {threads}
     """
 
 rule a_vs_a_sub_threads:
     input:
-        bin=sra_search_bin,
         queries="data/gtdb-list-a.sigs.txt",
         against="data/wort-list-a-{n}.txt",
     output:
         csv="outputs/output_a_vs_a_{n}_t{thr}.csv",
+    resources:
+        only_one_job=1,
     benchmark:
         "benchmarks/a_vs_a_{n}_t{thr}.txt"
-    threads: 64
+    threads: 64                 # max out threads so nothing else running
     shell: """
-        export RAYON_NUM_THREADS={wildcards.thr}
-        {input.bin} -k 31 --scaled=1000 -o {output.csv} \
-            {input.queries} {input.against}
+        {manysearch_cmd} -k 31 --scaled=1000 -o {output.csv} \
+            {input.queries} {input.against} -c {wildcards.thr}
     """
 
 
 rule a_vs_catalog:
     input:
-        bin=sra_search_bin,
         queries="data/gtdb-list-a.sigs.txt",
         against="data/metagenomes-catalog.txt",
     output:
         csv="outputs/output_a_vs_catalog.csv",
+    resources:
+        only_one_job=1,
     benchmark:
         "benchmarks/a_vs_catalog.txt"
     threads: 32
     shell: """
-        export RAYON_NUM_THREADS={threads}
-        {input.bin} -k 31 --scaled=1000 -o {output.csv} \
-            {input.queries} {input.against} || true
+        {manysearch_cmd} -k 31 --scaled=1000 -o {output.csv} \
+            {input.queries} {input.against} -c {threads} || true
     """
diff --git a/benchmarks/a_100_vs_a.txt b/benchmarks/a_100_vs_a.txt
@@ -1,2 +1,2 @@
 s	h:m:s	max_rss	max_vms	max_uss	max_pss	io_in	io_out	mean_load	cpu_time
-1347.0233	0:22:27	12910.21	15135.63	13073.72	13073.72	0.01	0.14	2762.70	37225.36
+600.8063	0:10:00	14732.45	21267.59	14615.98	14620.31	0.01	8.24	2379.52	14299.89
diff --git a/benchmarks/a_200_vs_a.txt b/benchmarks/a_200_vs_a.txt
@@ -1,2 +1,2 @@
 s	h:m:s	max_rss	max_vms	max_uss	max_pss	io_in	io_out	mean_load	cpu_time
-1291.4875	0:21:31	14541.96	18260.40	14601.04	14601.05	0.01	0.24	3044.57	39323.17
+652.8024	0:10:52	15158.36	22479.65	14919.01	14923.30	0.00	16.54	2315.08	15116.74
diff --git a/benchmarks/a_300_vs_a.txt b/benchmarks/a_300_vs_a.txt
@@ -1,2 +1,2 @@
 s	h:m:s	max_rss	max_vms	max_uss	max_pss	io_in	io_out	mean_load	cpu_time
-1454.4474	0:24:14	16348.15	18954.26	16429.44	16429.44	0.02	0.31	2828.24	41136.09
+688.3763	0:11:28	14350.04	21521.45	14501.56	14505.92	0.00	23.97	2324.88	16007.57
diff --git a/benchmarks/a_400_vs_a.txt b/benchmarks/a_400_vs_a.txt
@@ -1,2 +1,2 @@
 s	h:m:s	max_rss	max_vms	max_uss	max_pss	io_in	io_out	mean_load	cpu_time
-1551.7362	0:25:51	14438.77	16864.46	14666.07	14666.08	0.02	0.40	2751.87	42702.26
+657.4765	0:10:57	18378.81	26965.45	17962.34	17966.62	0.00	31.45	2575.65	16934.31
diff --git a/benchmarks/a_500_vs_a.txt b/benchmarks/a_500_vs_a.txt
@@ -1,2 +1,2 @@
 s	h:m:s	max_rss	max_vms	max_uss	max_pss	io_in	io_out	mean_load	cpu_time
-1603.6622	0:26:43	18893.50	21000.83	18968.33	18968.33	0.03	0.48	2802.76	44947.46
+675.8257	0:11:15	15105.68	20776.05	15626.83	15631.12	0.01	37.62	2647.93	17899.05
diff --git a/benchmarks/a_600_vs_a.txt b/benchmarks/a_600_vs_a.txt
@@ -1,2 +1,2 @@
 s	h:m:s	max_rss	max_vms	max_uss	max_pss	io_in	io_out	mean_load	cpu_time
-1696.6006	0:28:16	17783.73	19868.45	17888.35	17888.35	0.03	0.59	2755.67	46753.81
+775.0072	0:12:55	14575.46	20334.43	14950.77	14955.17	0.00	45.79	2440.03	18914.15
diff --git a/benchmarks/a_700_vs_a.txt b/benchmarks/a_700_vs_a.txt
@@ -1,2 +1,2 @@
 s	h:m:s	max_rss	max_vms	max_uss	max_pss	io_in	io_out	mean_load	cpu_time
-1610.4366	0:26:50	16855.73	19042.42	16989.04	16989.04	0.04	0.70	3032.36	48835.22
+751.2011	0:12:31	18513.38	24394.66	18375.63	18379.91	0.00	56.44	2616.67	19660.27
diff --git a/benchmarks/a_800_vs_a.txt b/benchmarks/a_800_vs_a.txt
@@ -1,2 +1,2 @@
 s	h:m:s	max_rss	max_vms	max_uss	max_pss	io_in	io_out	mean_load	cpu_time
-1770.2948	0:29:30	16316.45	19000.11	16440.92	16440.92	0.04	0.85	2863.60	50694.71
+783.5165	0:13:03	14754.23	22361.55	14833.44	14837.79	0.00	65.54	2639.75	20686.77
diff --git a/benchmarks/a_900_vs_a.txt b/benchmarks/a_900_vs_a.txt
@@ -1,2 +1,2 @@
 s	h:m:s	max_rss	max_vms	max_uss	max_pss	io_in	io_out	mean_load	cpu_time
-1806.3763	0:30:06	16779.61	19569.78	16286.82	16286.82	95967.58	0.97	2889.70	52198.87
+807.1810	0:13:27	18012.00	24211.34	17882.16	17886.44	0.01	72.65	2673.99	21584.14
diff --git a/benchmarks/a_vs_a.txt b/benchmarks/a_vs_a.txt
@@ -1,2 +1,2 @@
 s	h:m:s	max_rss	max_vms	max_uss	max_pss	io_in	io_out	mean_load	cpu_time
-1394.1251	0:23:14	16106.77	18410.19	16167.91	16167.94	95974.41	1.14	2914.11	40626.79
+875.6504	0:14:35	18036.04	24671.35	17774.06	17778.36	0.00	82.84	2567.74	22488.25
diff --git a/benchmarks/a_vs_a_1000.txt b/benchmarks/a_vs_a_1000.txt
@@ -1,2 +1,2 @@
 s	h:m:s	max_rss	max_vms	max_uss	max_pss	io_in	io_out	mean_load	cpu_time
-154.1568	0:02:34	8026.29	8707.47	8042.18	8042.25	0.07	0.11	2112.52	3256.75
+128.7141	0:02:08	7857.28	13580.94	7753.30	7757.58	0.00	7.86	1526.84	1968.99
diff --git a/benchmarks/a_vs_a_1000_t16.txt b/benchmarks/a_vs_a_1000_t16.txt
@@ -1,2 +1,2 @@
 s	h:m:s	max_rss	max_vms	max_uss	max_pss	io_in	io_out	mean_load	cpu_time
-329.2121	0:05:29	6144.40	7038.87	6169.71	6170.21	7404.52	0.10	1137.80	3746.52
+191.4212	0:03:11	6044.67	14880.95	6076.89	6090.70	0.00	7.91	1018.62	1956.75
diff --git a/benchmarks/a_vs_a_1000_t36.txt b/benchmarks/a_vs_a_1000_t36.txt
diff --git a/benchmarks/a_vs_a_1000_t4.txt b/benchmarks/a_vs_a_1000_t4.txt
@@ -1,2 +1,2 @@
 s	h:m:s	max_rss	max_vms	max_uss	max_pss	io_in	io_out	mean_load	cpu_time
-814.4291	0:13:34	2246.58	2380.33	2243.58	2244.08	89.14	0.11	341.55	2782.05
+560.6648	0:09:20	2803.79	11262.21	2790.16	2795.00	0.00	8.13	347.01	1952.88
diff --git a/benchmarks/a_vs_a_1000_t40.txt b/benchmarks/a_vs_a_1000_t40.txt
diff --git a/benchmarks/a_vs_a_1000_t44.txt b/benchmarks/a_vs_a_1000_t44.txt
diff --git a/benchmarks/a_vs_a_1000_t48.txt b/benchmarks/a_vs_a_1000_t48.txt
diff --git a/benchmarks/a_vs_a_1000_t52.txt b/benchmarks/a_vs_a_1000_t52.txt
diff --git a/benchmarks/a_vs_a_1000_t56.txt b/benchmarks/a_vs_a_1000_t56.txt
diff --git a/benchmarks/a_vs_a_1000_t60.txt b/benchmarks/a_vs_a_1000_t60.txt
diff --git a/benchmarks/a_vs_a_1000_t64.txt b/benchmarks/a_vs_a_1000_t64.txt
diff --git a/benchmarks/a_vs_a_1000_t8.txt b/benchmarks/a_vs_a_1000_t8.txt
@@ -1,2 +1,2 @@
 s	h:m:s	max_rss	max_vms	max_uss	max_pss	io_in	io_out	mean_load	cpu_time
-484.0369	0:08:04	2661.74	3352.14	2661.75	2662.25	1189.09	0.11	649.08	3142.19
+291.7664	0:04:51	3481.41	11885.39	3448.68	3462.75	0.00	8.00	679.32	1988.99
diff --git a/benchmarks/a_vs_a_2000.txt b/benchmarks/a_vs_a_2000.txt
@@ -1,2 +1,2 @@
 s	h:m:s	max_rss	max_vms	max_uss	max_pss	io_in	io_out	mean_load	cpu_time
-269.0769	0:04:29	8977.02	9920.71	8961.81	8961.88	0.13	0.25	2450.90	6594.63
+182.4287	0:03:02	10165.68	17257.38	10114.44	10118.73	0.00	16.05	2143.11	3913.24
diff --git a/benchmarks/a_vs_a_3000.txt b/benchmarks/a_vs_a_3000.txt
@@ -1,2 +1,2 @@
 s	h:m:s	max_rss	max_vms	max_uss	max_pss	io_in	io_out	mean_load	cpu_time
-354.2737	0:05:54	11786.54	13038.87	11863.02	11863.11	0.20	0.36	2943.38	10428.58
+255.9341	0:04:15	10565.33	16437.94	10245.39	10249.78	0.01	24.18	2371.27	6072.81
diff --git a/benchmarks/a_vs_a_4000.txt b/benchmarks/a_vs_a_4000.txt
@@ -1,2 +1,2 @@
 s	h:m:s	max_rss	max_vms	max_uss	max_pss	io_in	io_out	mean_load	cpu_time
-478.0915	0:07:58	13964.38	15485.38	14074.45	14074.53	0.26	0.48	3003.24	14358.91
+338.4716	0:05:38	12050.80	16843.22	11950.77	11955.05	0.00	32.11	2468.33	8358.54
diff --git a/benchmarks/a_vs_a_5000.txt b/benchmarks/a_vs_a_5000.txt
@@ -1,2 +1,2 @@
 s	h:m:s	max_rss	max_vms	max_uss	max_pss	io_in	io_out	mean_load	cpu_time
-704.6193	0:11:44	15294.20	18001.37	15028.22	15028.30	0.33	0.64	2722.32	19182.34
+523.5476	0:08:43	14703.90	21026.22	14647.26	14651.61	0.00	40.57	2116.40	11084.23
diff --git a/benchmarks/a_vs_a_6000.txt b/benchmarks/a_vs_a_6000.txt
@@ -1,2 +1,2 @@
 s	h:m:s	max_rss	max_vms	max_uss	max_pss	io_in	io_out	mean_load	cpu_time
-1073.2904	0:17:53	12150.98	14048.74	12201.99	12201.99	0.39	0.73	2635.62	28288.41
+480.6425	0:08:00	17173.78	22992.17	16840.85	16845.15	0.01	48.84	2719.14	13071.86
diff --git a/benchmarks/a_vs_a_7000.txt b/benchmarks/a_vs_a_7000.txt
@@ -1,2 +1,2 @@
 s	h:m:s	max_rss	max_vms	max_uss	max_pss	io_in	io_out	mean_load	cpu_time
-1269.1121	0:21:09	15887.25	17586.37	15507.43	15507.43	0.46	0.79	2923.52	37104.07
+546.2705	0:09:06	15370.99	21318.24	15225.82	15230.17	0.00	56.76	2780.64	15193.29
diff --git a/benchmarks/a_vs_a_8000.txt b/benchmarks/a_vs_a_8000.txt
@@ -1,2 +1,2 @@
 s	h:m:s	max_rss	max_vms	max_uss	max_pss	io_in	io_out	mean_load	cpu_time
-1419.2742	0:23:39	16026.34	18511.11	16068.09	16068.09	0.52	0.89	3018.64	42844.40
+618.4300	0:10:18	21471.61	27889.91	20961.72	20966.02	0.00	65.14	2820.35	17445.65
diff --git a/benchmarks/a_vs_a_9000.txt b/benchmarks/a_vs_a_9000.txt
@@ -1,2 +1,2 @@
 s	h:m:s	max_rss	max_vms	max_uss	max_pss	io_in	io_out	mean_load	cpu_time
-1576.3652	0:26:16	16715.54	19374.40	16759.83	16759.83	7.45	1.00	3052.70	48122.82
+665.4772	0:11:05	17767.80	24126.63	17617.10	17621.45	0.01	74.00	2964.38	19731.08
diff --git a/benchmarks/a_vs_b.txt b/benchmarks/a_vs_b.txt
@@ -1,2 +1,2 @@
 s	h:m:s	max_rss	max_vms	max_uss	max_pss	io_in	io_out	mean_load	cpu_time
-1394.1750	0:23:14	17553.49	20057.43	17685.95	17685.99	94003.57	1.16	2771.80	38643.81
+827.8913	0:13:47	16330.66	22992.92	16727.98	16732.33	0.01	79.96	2671.40	22116.21
diff --git a/benchmarks/a_vs_c.txt b/benchmarks/a_vs_c.txt
@@ -1,2 +1,2 @@
 s	h:m:s	max_rss	max_vms	max_uss	max_pss	io_in	io_out	mean_load	cpu_time
-1357.7237	0:22:37	18257.87	20041.38	18277.77	18277.80	91280.15	1.01	2794.07	37936.37
+759.2550	0:12:39	17084.92	22578.64	16849.08	16853.45	0.01	82.22	2845.20	21602.36
diff --git a/benchmarks/a_vs_catalog.txt b/benchmarks/a_vs_catalog.txt
@@ -1,2 +1,2 @@
 s	h:m:s	max_rss	max_vms	max_uss	max_pss	io_in	io_out	mean_load	cpu_time
-125090.2216	1 day, 10:44:50	38624.68	42192.40	38807.58	38808.14	7265572.11	87.13	2961.77	3704886.43
+56864.2508	15:47:44	35820.05	44126.49	35512.32	35517.34	78.45	6359.39	3016.27	1715185.69
diff --git a/benchmarks/a_vs_d.txt b/benchmarks/a_vs_d.txt
@@ -1,2 +1,2 @@
 s	h:m:s	max_rss	max_vms	max_uss	max_pss	io_in	io_out	mean_load	cpu_time
-1505.0091	0:25:05	16107.48	19187.17	16137.28	16137.32	93738.74	1.02	2605.24	39209.39
+807.2910	0:13:27	15190.04	20914.19	15028.30	15032.68	0.00	82.21	2735.69	22088.67
diff --git a/benchmarks/a_vs_e.txt b/benchmarks/a_vs_e.txt
@@ -1,2 +1,2 @@
 s	h:m:s	max_rss	max_vms	max_uss	max_pss	io_in	io_out	mean_load	cpu_time
-1596.4828	0:26:36	14046.44	15868.38	14061.43	14061.46	91948.11	0.96	2367.13	37791.57
+743.7042	0:12:23	17108.77	22594.65	17374.48	17378.82	0.00	84.39	2892.58	21516.02
diff --git a/benchmarks/a_vs_largest_10k.txt b/benchmarks/a_vs_largest_10k.txt
@@ -1,2 +1,2 @@
 s	h:m:s	max_rss	max_vms	max_uss	max_pss	io_in	io_out	mean_load	cpu_time
-35660.1640	9:54:20	59988.88	70891.44	59716.36	59716.39	2225534.90	1.58	2867.89	1022695.90
+19046.7232	5:17:26	60354.86	76403.62	61956.32	61960.61	111.21	204.75	2979.81	567559.73
diff --git a/environment.yml b/environment.yml
@@ -0,0 +1,13 @@
+name: branchwater-bench
+channels:
+    - conda-forge
+    - bioconda
+    - defaults
+dependencies:
+    - sourmash>=4.8.3,<5
+    - snakemake-minimal>=7,<8
+    - pip
+    - rust
+    - maturin>=1,<2
+    - pip:
+      - pyo3_branchwater==0.6.1