Skip to content

Commit b349ffe

Browse files
GjjvdBurgGjjvdBurg
committed
Merge branch 'update'
2 parents 692632b + e0f485f commit b349ffe

File tree

184 files changed

+15407
-11430
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

184 files changed

+15407
-11430
lines changed

‎CHANGELOG.md

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,14 @@
11
# Changelog
22

3-
## Version 1.0.0
3+
## Version 2.0
4+
5+
* Added the "zero" baseline method
6+
* Added a script to compute summary statistics
7+
* Added rank plots for multivariate datasets
8+
* Corrected an error in the computation of the F1 score and updated the
9+
results. This correction had no major effect on the conclusions of the
10+
paper.
11+
12+
## Version 1.0
413

514
* Initial release

‎Makefile

Lines changed: 94 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -78,7 +78,9 @@ best_tables: \
7878
$(TABLE_DIR)/best_cover_uni_avg.json \
7979
$(TABLE_DIR)/best_cover_multi_avg.json \
8080
$(TABLE_DIR)/best_f1_uni_full.json \
81-
$(TABLE_DIR)/best_cover_uni_full.json
81+
$(TABLE_DIR)/best_cover_uni_full.json \
82+
$(TABLE_DIR)/best_f1_multi_full.json \
83+
$(TABLE_DIR)/best_cover_multi_full.json
8284

8385
$(TABLE_DIR)/best_f1_combined_full.tex: $(SCRIPT_DIR)/make_table.py summaries | table-dir
8486
python $< -s $(SUMMARY_DIR) -e best -m f1 -d combined -f tex -t full > $@
@@ -104,6 +106,12 @@ $(TABLE_DIR)/best_cover_uni_full.json: $(SCRIPT_DIR)/make_table.py summaries | t
104106
$(TABLE_DIR)/best_f1_uni_full.json: $(SCRIPT_DIR)/make_table.py summaries | table-dir
105107
python $< -s $(SUMMARY_DIR) -e best -m f1 -d uni -f json -t full > $@
106108

109+
$(TABLE_DIR)/best_f1_multi_full.json: $(SCRIPT_DIR)/make_table.py summaries
110+
python $< -s $(SUMMARY_DIR) -e best -m f1 -d multi -f json -t full > $@
111+
112+
$(TABLE_DIR)/best_cover_multi_full.json: $(SCRIPT_DIR)/make_table.py summaries
113+
python $< -s $(SUMMARY_DIR) -e best -m cover -d multi -f json -t full > $@
114+
107115
default_tables: \
108116
$(TABLE_DIR)/default_f1_combined_full.tex \
109117
$(TABLE_DIR)/default_cover_combined_full.tex \
@@ -112,7 +120,9 @@ default_tables: \
112120
$(TABLE_DIR)/default_cover_uni_avg.json \
113121
$(TABLE_DIR)/default_cover_multi_avg.json \
114122
$(TABLE_DIR)/default_cover_uni_full.json \
115-
$(TABLE_DIR)/default_f1_uni_full.json
123+
$(TABLE_DIR)/default_f1_uni_full.json \
124+
$(TABLE_DIR)/default_cover_multi_full.json \
125+
$(TABLE_DIR)/default_f1_multi_full.json
116126

117127
$(TABLE_DIR)/default_f1_combined_full.tex: $(SCRIPT_DIR)/make_table.py summaries | table-dir
118128
python $< -s $(SUMMARY_DIR) -e default -m f1 -d combined -f tex -t full > $@
@@ -138,6 +148,11 @@ $(TABLE_DIR)/default_cover_uni_full.json: $(SCRIPT_DIR)/make_table.py summaries
138148
$(TABLE_DIR)/default_f1_uni_full.json: $(SCRIPT_DIR)/make_table.py summaries | table-dir
139149
python $< -s $(SUMMARY_DIR) -e default -m f1 -d uni -f json -t full > $@
140150

151+
$(TABLE_DIR)/default_cover_multi_full.json: $(SCRIPT_DIR)/make_table.py summaries | table-dir
152+
python $< -s $(SUMMARY_DIR) -e default -m cover -d multi -f json -t full > $@
153+
154+
$(TABLE_DIR)/default_f1_multi_full.json: $(SCRIPT_DIR)/make_table.py summaries | table-dir
155+
python $< -s $(SUMMARY_DIR) -e default -m f1 -d multi -f json -t full > $@
141156

142157
aggregate_wide: $(TABLE_DIR)/aggregate_table_wide.tex
143158

@@ -193,27 +208,56 @@ rank-dir:
193208
mkdir -p $(RANK_DIR)
194209

195210
rankplots: \
196-
$(RANK_DIR)/rankplot_best_cover_uni.tex \
197-
$(RANK_DIR)/rankplot_best_f1_uni.tex \
198-
$(RANK_DIR)/rankplot_default_cover_uni.tex \
199-
$(RANK_DIR)/rankplot_default_f1_uni.tex \
200211
$(RANK_DIR)/rankplot_best_cover_uni.pdf \
201212
$(RANK_DIR)/rankplot_best_f1_uni.pdf \
202213
$(RANK_DIR)/rankplot_default_cover_uni.pdf \
203-
$(RANK_DIR)/rankplot_default_f1_uni.pdf
214+
$(RANK_DIR)/rankplot_default_f1_uni.pdf \
215+
$(RANK_DIR)/rankplot_best_cover_multi.pdf \
216+
$(RANK_DIR)/rankplot_best_f1_multi.pdf \
217+
$(RANK_DIR)/rankplot_default_cover_multi.pdf \
218+
$(RANK_DIR)/rankplot_default_f1_multi.pdf
219+
220+
#######
221+
# UNI #
222+
#######
223+
224+
$(RANK_DIR)/rankplot_best_cover_uni.tex: $(TABLE_DIR)/best_cover_uni_full.json \
225+
$(SCRIPT_DIR)/rank_plots.py | rank-dir
226+
python $(SCRIPT_DIR)/rank_plots.py -i $< -o $@ -b max --type best
227+
228+
$(RANK_DIR)/rankplot_best_f1_uni.tex: $(TABLE_DIR)/best_f1_uni_full.json \
229+
$(SCRIPT_DIR)/rank_plots.py | rank-dir
230+
python $(SCRIPT_DIR)/rank_plots.py -i $< -o $@ -b max --type best
231+
232+
$(RANK_DIR)/rankplot_default_cover_uni.tex: $(TABLE_DIR)/default_cover_uni_full.json \
233+
$(SCRIPT_DIR)/rank_plots.py | rank-dir
234+
python $(SCRIPT_DIR)/rank_plots.py -i $< -o $@ -b max --type default
235+
236+
$(RANK_DIR)/rankplot_default_f1_uni.tex: $(TABLE_DIR)/default_f1_uni_full.json \
237+
$(SCRIPT_DIR)/rank_plots.py | rank-dir
238+
python $(SCRIPT_DIR)/rank_plots.py -i $< -o $@ -b max --type default
204239

205-
$(RANK_DIR)/rankplot_best_cover_uni.tex: $(TABLE_DIR)/best_cover_uni_full.json $(SCRIPT_DIR)/rank_plots.py | rank-dir
240+
#########
241+
# MULTI #
242+
#########
243+
244+
$(RANK_DIR)/rankplot_best_cover_multi.tex: $(TABLE_DIR)/best_cover_multi_full.json \
245+
$(SCRIPT_DIR)/rank_plots.py | rank-dir
206246
python $(SCRIPT_DIR)/rank_plots.py -i $< -o $@ -b max --type best
207247

208-
$(RANK_DIR)/rankplot_best_f1_uni.tex: $(TABLE_DIR)/best_f1_uni_full.json $(SCRIPT_DIR)/rank_plots.py | rank-dir
248+
$(RANK_DIR)/rankplot_best_f1_multi.tex: $(TABLE_DIR)/best_f1_multi_full.json \
249+
$(SCRIPT_DIR)/rank_plots.py | rank-dir
209250
python $(SCRIPT_DIR)/rank_plots.py -i $< -o $@ -b max --type best
210251

211-
$(RANK_DIR)/rankplot_default_cover_uni.tex: $(TABLE_DIR)/default_cover_uni_full.json $(SCRIPT_DIR)/rank_plots.py | rank-dir
252+
$(RANK_DIR)/rankplot_default_cover_multi.tex: $(TABLE_DIR)/default_cover_multi_full.json \
253+
$(SCRIPT_DIR)/rank_plots.py | rank-dir
212254
python $(SCRIPT_DIR)/rank_plots.py -i $< -o $@ -b max --type default
213255

214-
$(RANK_DIR)/rankplot_default_f1_uni.tex: $(TABLE_DIR)/default_f1_uni_full.json $(SCRIPT_DIR)/rank_plots.py | rank-dir
256+
$(RANK_DIR)/rankplot_default_f1_multi.tex: $(TABLE_DIR)/default_f1_multi_full.json \
257+
$(SCRIPT_DIR)/rank_plots.py | rank-dir
215258
python $(SCRIPT_DIR)/rank_plots.py -i $< -o $@ -b max --type default
216259

260+
217261
$(RANK_DIR)/rankplot_%.pdf: $(RANK_DIR)/rankplot_%.tex | rank-dir
218262
latexmk -pdf -pdflatex="pdflatex -interaction=nonstopmode --shell-escape" \
219263
-outdir=$(RANK_DIR) $<
@@ -237,7 +281,14 @@ clean_rankplots:
237281
CONSTANT_TARGETS = $(CONST_DIR)/sigtest_global_best_cover_uni.tex \
238282
$(CONST_DIR)/sigtest_global_best_f1_uni.tex \
239283
$(CONST_DIR)/sigtest_global_default_cover_uni.tex \
240-
$(CONST_DIR)/sigtest_global_default_f1_uni.tex
284+
$(CONST_DIR)/sigtest_global_default_f1_uni.tex \
285+
$(CONST_DIR)/SeriesLengthMin.tex \
286+
$(CONST_DIR)/SeriesLengthMax.tex \
287+
$(CONST_DIR)/SeriesLengthMean.tex \
288+
$(CONST_DIR)/UniqueAnnotationsMin.tex \
289+
$(CONST_DIR)/UniqueAnnotationsMax.tex \
290+
$(CONST_DIR)/UniqueAnnotationsMean.tex \
291+
$(CONST_DIR)/UniqueAnnotationsStd.tex
241292

242293
const-dir:
243294
mkdir -p $(CONST_DIR)
@@ -260,6 +311,34 @@ $(CONST_DIR)/sigtest_global_default_f1_uni.tex: $(TABLE_DIR)/default_f1_uni_full
260311
$(SCRIPT_DIR)/significance.py | const-dir
261312
python $(SCRIPT_DIR)/significance.py -i $< -o $@ --type best --mode global
262313

314+
$(CONST_DIR)/SeriesLengthMin.tex: $(SCRIPT_DIR)/descriptive_length.py \
315+
$(DATASET_SUMMARIES) | const-dir
316+
python $< -s $(SUMMARY_DIR) -t min > $@
317+
318+
$(CONST_DIR)/SeriesLengthMax.tex: $(SCRIPT_DIR)/descriptive_length.py \
319+
$(DATASET_SUMMARIES) | const-dir
320+
python $< -s $(SUMMARY_DIR) -t max > $@
321+
322+
$(CONST_DIR)/SeriesLengthMean.tex: $(SCRIPT_DIR)/descriptive_length.py \
323+
$(DATASET_SUMMARIES) | const-dir
324+
python $< -s $(SUMMARY_DIR) -t mean > $@
325+
326+
$(CONST_DIR)/UniqueAnnotationsMin.tex: $(SCRIPT_DIR)/descriptive_annotations.py \
327+
$(DATASET_SUMMARIES) | const-dir
328+
python $< -s $(SUMMARY_DIR) -t min > $@
329+
330+
$(CONST_DIR)/UniqueAnnotationsMax.tex: $(SCRIPT_DIR)/descriptive_annotations.py \
331+
$(DATASET_SUMMARIES) | const-dir
332+
python $< -s $(SUMMARY_DIR) -t max > $@
333+
334+
$(CONST_DIR)/UniqueAnnotationsMean.tex: $(SCRIPT_DIR)/descriptive_annotations.py \
335+
$(DATASET_SUMMARIES) | const-dir
336+
python $< -s $(SUMMARY_DIR) -t mean > $@
337+
338+
$(CONST_DIR)/UniqueAnnotationsStd.tex: $(SCRIPT_DIR)/descriptive_annotations.py \
339+
$(DATASET_SUMMARIES) | const-dir
340+
python $< -s $(SUMMARY_DIR) -t std > $@
341+
263342
clean_constants:
264343
rm -f $(CONSTANT_TARGETS)
265344

@@ -312,4 +391,6 @@ validate: ./utils/validate_schema.py ./schema.json
312391
# #
313392
###########
314393

315-
clean: clean_summaries clean_tables clean_rankplots clean_venvs
394+
clean: clean_results clean_venvs
395+
396+
clean_results: clean_summaries clean_tables clean_rankplots clean_constants

‎abed_conf.py

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -125,6 +125,7 @@
125125
"best_kcpa",
126126
"best_wbs",
127127
"best_prophet",
128+
"best_zero",
128129
"default_bocpd",
129130
"default_bocpdms",
130131
"default_rbocpdms",
@@ -138,6 +139,7 @@
138139
"default_kcpa",
139140
"default_wbs",
140141
"default_prophet",
142+
"default_zero",
141143
]
142144

143145
# many of these combinations will be invalid for the changepoint package, but
@@ -210,6 +212,7 @@
210212
"integrated": ["true", "false"],
211213
},
212214
"best_prophet": {"Nmax": ["max", "default"]},
215+
"best_zero": {"no_param": [0]},
213216
"default_bocpd": {"no_param": [0]},
214217
"default_bocpdms": {"no_param": [0]},
215218
"default_rbocpdms": {"no_param": [0]},
@@ -223,6 +226,7 @@
223226
"default_kcpa": {"no_param": [0]},
224227
"default_wbs": {"no_param": [0]},
225228
"default_prophet": {"no_param": [0]},
229+
"default_zero": {"no_param": [0]}
226230
}
227231

228232
COMMANDS = {
@@ -243,6 +247,7 @@
243247
"best_rbocpdms": (
244248
"source {execdir}/python/rbocpdms/venv/bin/activate && python {execdir}/python/cpdbench_rbocpdms.py -i {datadir}/{dataset}.json --intensity {intensity} --prior-a {prior_a} --prior-b {prior_b} --threshold 100 --alpha-param {alpha_param} --alpha-rld {alpha_rld} --use-timeout"
245249
),
250+
"best_zero": "python {execdir}/python/cpdbench_zero.py -i {datadir}/{dataset}.json",
246251
"default_amoc": "Rscript --no-save --slave {execdir}/R/cpdbench_changepoint.R -i {datadir}/{dataset}.json -p MBIC -f mean -t Normal -m AMOC",
247252
"default_binseg": "Rscript --no-save --slave {execdir}/R/cpdbench_changepoint.R -i {datadir}/{dataset}.json -p MBIC -f mean -t Normal -m BinSeg -Q default",
248253
"default_cpnp": "Rscript --no-save --slave {execdir}/R/cpdbench_changepointnp.R -i {datadir}/{dataset}.json -p MBIC -q 10",
@@ -256,6 +261,7 @@
256261
"default_bocpd": "Rscript --no-save --slave {execdir}/R/cpdbench_ocp.R -i {datadir}/{dataset}.json -l 100 --prior-a 1.0 --prior-b 1.0 --prior-k 1.0",
257262
"default_bocpdms": "source {execdir}/python/bocpdms/venv/bin/activate && python {execdir}/python/cpdbench_bocpdms.py -i {datadir}/{dataset}.json --intensity 100 --prior-a 1.0 --prior-b 1.0 --threshold 0",
258263
"default_rbocpdms": "source {execdir}/python/rbocpdms/venv/bin/activate && python {execdir}/python/cpdbench_rbocpdms.py -i {datadir}/{dataset}.json --intensity 100 --prior-a 1.0 --prior-b 1.0 --threshold 100 --alpha-param 0.5 --alpha-rld 0.5 --timeout 240",
264+
"default_zero": "python {execdir}/python/cpdbench_zero.py -i {datadir}/{dataset}.json",
259265
}
260266

261267
METRICS = {}

‎abed_results/apple/best_zero/ce6f534a7a0632dd.json

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
{
2+
"command": "/home/gertjan/share/work/turing/aida/research/projects/changepoint/experiments/cpdbench/execs/python/cpdbench_zero.py -i /home/gertjan/share/work/turing/aida/research/projects/changepoint/experiments/cpdbench/datasets/apple.json",
3+
"dataset": "apple",
4+
"dataset_md5": "22edb48471bd3711f7a6e15de6413643",
5+
"error": null,
6+
"hostname": "nyx",
7+
"parameters": {},
8+
"result": {
9+
"cplocations": [],
10+
"runtime": 2.384185791015625e-07
11+
},
12+
"script": "/home/gertjan/share/work/turing/aida/research/projects/changepoint/experiments/cpdbench/execs/python/cpdbench_zero.py",
13+
"script_md5": "95b65ddd5669b41385966a4aad387118",
14+
"status": "SUCCESS"
15+
}

‎abed_results/apple/default_zero/6ce2dbdc33d50375.json

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
{
2+
"command": "/home/gertjan/share/work/turing/aida/research/projects/changepoint/experiments/cpdbench/execs/python/cpdbench_zero.py -i /home/gertjan/share/work/turing/aida/research/projects/changepoint/experiments/cpdbench/datasets/apple.json",
3+
"dataset": "apple",
4+
"dataset_md5": "22edb48471bd3711f7a6e15de6413643",
5+
"error": null,
6+
"hostname": "nyx",
7+
"parameters": {},
8+
"result": {
9+
"cplocations": [],
10+
"runtime": 2.384185791015625e-07
11+
},
12+
"script": "/home/gertjan/share/work/turing/aida/research/projects/changepoint/experiments/cpdbench/execs/python/cpdbench_zero.py",
13+
"script_md5": "95b65ddd5669b41385966a4aad387118",
14+
"status": "SUCCESS"
15+
}

‎abed_results/bank/best_zero/6ced213582b692f2.json

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
{
2+
"command": "/home/gertjan/share/work/turing/aida/research/projects/changepoint/experiments/cpdbench/execs/python/cpdbench_zero.py -i /home/gertjan/share/work/turing/aida/research/projects/changepoint/experiments/cpdbench/datasets/bank.json",
3+
"dataset": "bank",
4+
"dataset_md5": "5207135ea53fc6fa2a8119908da73abf",
5+
"error": null,
6+
"hostname": "nyx",
7+
"parameters": {},
8+
"result": {
9+
"cplocations": [],
10+
"runtime": 2.384185791015625e-07
11+
},
12+
"script": "/home/gertjan/share/work/turing/aida/research/projects/changepoint/experiments/cpdbench/execs/python/cpdbench_zero.py",
13+
"script_md5": "95b65ddd5669b41385966a4aad387118",
14+
"status": "SUCCESS"
15+
}

‎abed_results/bank/default_zero/bab9216b61d1d3c0.json

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
{
2+
"command": "/home/gertjan/share/work/turing/aida/research/projects/changepoint/experiments/cpdbench/execs/python/cpdbench_zero.py -i /home/gertjan/share/work/turing/aida/research/projects/changepoint/experiments/cpdbench/datasets/bank.json",
3+
"dataset": "bank",
4+
"dataset_md5": "5207135ea53fc6fa2a8119908da73abf",
5+
"error": null,
6+
"hostname": "nyx",
7+
"parameters": {},
8+
"result": {
9+
"cplocations": [],
10+
"runtime": 4.76837158203125e-07
11+
},
12+
"script": "/home/gertjan/share/work/turing/aida/research/projects/changepoint/experiments/cpdbench/execs/python/cpdbench_zero.py",
13+
"script_md5": "95b65ddd5669b41385966a4aad387118",
14+
"status": "SUCCESS"
15+
}

‎abed_results/bee_waggle_6/best_zero/6fec83c31881a45d.json

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
{
2+
"command": "/home/gertjan/share/work/turing/aida/research/projects/changepoint/experiments/cpdbench/execs/python/cpdbench_zero.py -i /home/gertjan/share/work/turing/aida/research/projects/changepoint/experiments/cpdbench/datasets/bee_waggle_6.json",
3+
"dataset": "bee_waggle_6",
4+
"dataset_md5": "4f03feafecb3be0b069b3cb0d6b17d4f",
5+
"error": null,
6+
"hostname": "nyx",
7+
"parameters": {},
8+
"result": {
9+
"cplocations": [],
10+
"runtime": 4.76837158203125e-07
11+
},
12+
"script": "/home/gertjan/share/work/turing/aida/research/projects/changepoint/experiments/cpdbench/execs/python/cpdbench_zero.py",
13+
"script_md5": "95b65ddd5669b41385966a4aad387118",
14+
"status": "SUCCESS"
15+
}

‎abed_results/bee_waggle_6/default_zero/ae18dda4acb23ea7.json

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
{
2+
"command": "/home/gertjan/share/work/turing/aida/research/projects/changepoint/experiments/cpdbench/execs/python/cpdbench_zero.py -i /home/gertjan/share/work/turing/aida/research/projects/changepoint/experiments/cpdbench/datasets/bee_waggle_6.json",
3+
"dataset": "bee_waggle_6",
4+
"dataset_md5": "4f03feafecb3be0b069b3cb0d6b17d4f",
5+
"error": null,
6+
"hostname": "nyx",
7+
"parameters": {},
8+
"result": {
9+
"cplocations": [],
10+
"runtime": 2.384185791015625e-07
11+
},
12+
"script": "/home/gertjan/share/work/turing/aida/research/projects/changepoint/experiments/cpdbench/execs/python/cpdbench_zero.py",
13+
"script_md5": "95b65ddd5669b41385966a4aad387118",
14+
"status": "SUCCESS"
15+
}

‎abed_results/bitcoin/best_zero/c0865595ee43be76.json

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
{
2+
"command": "/home/gertjan/share/work/turing/aida/research/projects/changepoint/experiments/cpdbench/execs/python/cpdbench_zero.py -i /home/gertjan/share/work/turing/aida/research/projects/changepoint/experiments/cpdbench/datasets/bitcoin.json",
3+
"dataset": "bitcoin",
4+
"dataset_md5": "f90ff14ed1fc0c3d47d4394d25cbce93",
5+
"error": null,
6+
"hostname": "nyx",
7+
"parameters": {},
8+
"result": {
9+
"cplocations": [],
10+
"runtime": 0.0
11+
},
12+
"script": "/home/gertjan/share/work/turing/aida/research/projects/changepoint/experiments/cpdbench/execs/python/cpdbench_zero.py",
13+
"script_md5": "95b65ddd5669b41385966a4aad387118",
14+
"status": "SUCCESS"
15+
}

0 commit comments

Comments
 (0)