FusingHeterogeneousData/Makefile at main · xsedla1o/FusingHeterogeneousData · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
BUILD := data/generated
SHELL := /bin/bash

all: report
experiments: experiment-1 experiment-2 experiment-3
experiments-long: experiment-1 experiment-2 experiment-3 experiment-5

######## Dataset Manager commands ########
status:
	python3 dataset_manager.py $@

######## Dataset Report Building ########
report: latex/report.pdf
latex/report.pdf: latex/report.tex data/source/dataset.csv $(BUILD)/report_requirements.stamp
	python3 report_plots.py
	cd latex && xelatex -8bit -shell-escape report && xelatex -8bit -shell-escape report

report_requirements: $(BUILD)/report_requirements.stamp
$(BUILD)/report_requirements.stamp: $(BUILD)/strictly-filtered-n1.stamp $(BUILD)/strictly-filtered-n2.stamp $(BUILD)/strictly-filtered-n3.stamp \
								  	experiments-long $(BUILD)/rule-training.stamp
	touch $@

test_distortion_resilience: $(BUILD)/kfold-validation-ds1-dr.stamp $(BUILD)/kfold-validation-ds2-dr.stamp $(BUILD)/function-tester-dr.stamp

######## Generating Synthetic Datasets ########
synthetic_dataset_current:
	python3 dataset_synthesizer.py config/experiments.yml DATASET_GENERATOR,CURRENT 2000

fully_synthetic_dataset:
	python3 dataset_synthesizer.py config/experiments.yml DATASET_GENERATOR,CURRENT 2000 --full-synth

######## Experiments ########
# Experiment 1
experiment-1: $(BUILD)/experiment-1.stamp
$(BUILD)/experiment-1.stamp: $(BUILD)/basic.stamp $(BUILD)/reference.stamp $(BUILD)/balanced.stamp
	touch $@

# Experiment 2
experiment-2: $(BUILD)/experiment-2.stamp
$(BUILD)/experiment-2.stamp: $(BUILD)/combiners.stamp
	touch $@

# Experiment 3
experiment-3: $(BUILD)/experiment-3.stamp
$(BUILD)/experiment-3.stamp: $(BUILD)/kfold-validation-ds1.stamp $(BUILD)/kfold-validation-ds2.stamp
	touch $@

# Experiment 5
experiment-5: $(BUILD)/experiment-5.stamp
$(BUILD)/experiment-5.stamp: $(BUILD)/function-tester.stamp
	touch $@


######## Experiment module details ########
$(BUILD)/basic.stamp: data/source/dataset.csv
	python3 basic_eval.py config/experiments.yml BASIC,INFORMED
	touch $@

$(BUILD)/reference.stamp: $(BUILD)/basic.stamp
	python3 reference.py config/experiments.yml REFERENCE
	touch $@

$(BUILD)/balanced.stamp: $(BUILD)/basic.stamp $(BUILD)/reference.stamp
	python3 sanitizer.py config/experiments.yml SANITIZER,INFORMED -b
	touch $@

$(BUILD)/strictly-filtered-n%.stamp: $(BUILD)/balanced.stamp
	python3 filter.py config/experiments.yml FILTER,N$* $* -s
	touch $@

$(BUILD)/combiners.stamp: $(BUILD)/balanced.stamp
	python3 combiners.py config/experiments.yml COMBINERS
	touch $@

$(BUILD)/rule-training.stamp: $(BUILD)/function-tester.stamp $(BUILD)/kfold-validation-ds1.stamp $(BUILD)/kfold-validation-ds2.stamp
	python3 rule_training.py config/experiments.yml RULE_TRAINING --discard-untrained --detect-best
	touch $@

$(BUILD)/kfold-validation-ds1.stamp: $(BUILD)/balanced.stamp
	python3 rule_optimization_kfold.py config/experiments.yml KFOLD_VALIDATION --method=distribute-confusion --also-run priority majority oracle dt rf ab --out=ds1
	touch $@

$(BUILD)/kfold-validation-ds2.stamp: $(BUILD)/balanced.stamp
	python3 rule_optimization_kfold.py config/experiments.yml KFOLD_VALIDATION --method=two-focal --out=ds2
	touch $@

$(BUILD)/param-optimization-complete.stamp: $(BUILD)/balanced.stamp
	python3 param_optimization.py config/experiments.yml PARAM_OPTIMIZATION --reduce 2000
	touch $@

$(BUILD)/param-optimization.stamp:  $(BUILD)/param-optimization-complete.stamp
	touch $@

$(BUILD)/function-tester.stamp: $(BUILD)/param-optimization.stamp $(BUILD)/balanced.stamp
	python3 function_param_tester.py config/experiments.yml FUNCTION_TESTER
	touch $@

######## Distortion Resilience Testing ########

$(BUILD)/kfold-validation-ds1-dr.stamp:
	python3 rule_optimization_kfold.py config/experiments.yml KFOLD_DISTORTION_RESILIENCE --distortion-resilience --method=distribute-confusion --also-run priority majority oracle dt rf ab --out=ds1
	touch $@

$(BUILD)/kfold-validation-ds2-dr.stamp:
	python3 rule_optimization_kfold.py config/experiments.yml KFOLD_DISTORTION_RESILIENCE --distortion-resilience --method=two-focal --out=ds2
	touch $@

$(BUILD)/function-tester-dr.stamp:
	python3 function_param_tester.py config/experiments.yml FUNCTION_TESTER_DISTORTION_RESILIENCE --distortion-resilience
	touch $@

######## Phony targets ########

.PHONY: clean clean-all all experiments experiments-long report_requirements \
		experiment-1 experiment-2 experiment-3 experiment-4 experiment-5 \
		status init init-synth

init:
	mkdir -p data/stash
	rm -f config/experiments.yml config/attr_conf
	cd config && ln -s attr_conf_real attr_conf
	cd config && ln -s experiments_classic.yml experiments.yml

init-synth:
	rm -f config/experiments.yml config/attr_conf
	cd config && ln -s attr_conf_synth attr_conf
	cd config && ln -s experiments_synth.yml experiments.yml

clean:
	rm -rf $(BUILD)/*
	rm -f latex/report.aux latex/report.log latex/report.out latex/report.toc

clean-all: clean
	rm -f latex/generated/* latex/report.pdf