-
Notifications
You must be signed in to change notification settings - Fork 5
/
meta_test.py
208 lines (162 loc) · 7.08 KB
/
meta_test.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
import argparse
import re
import os
import scipy.stats as scipy_stats
from src.output_results import *
parser = argparse.ArgumentParser()
parser.add_argument("--name",required=True)
parser.add_argument("--file",required=True,help="file with names of test_results stats.txt files to read")
args = parser.parse_args()
# echo args
for k,v in args.__dict__.items():
print(" " + k + ":", v, type(v))
os.makedirs("meta_results/" + args.name, exist_ok=True)
with open(args.file, "r") as f:
filenames = f.readlines()
filenames = [i.strip() for i in filenames if i.strip() != ""]
filenames = [i.split(maxsplit=1) for i in filenames]
def allin(snippets, line):
return all([i in line for i in snippets])
class TestData:
def __init__(self, name, filename):
self.filename = filename
self.name = name
self.num_seeds = 0
with open(filename, "r") as f:
lines = f.readlines()
datalines = []
in_files = True
for i in lines:
if not in_files:
datalines.append(i)
elif allin(["Min", "Avg", "Max", "Med", "Std"], i):
in_files = False
elif i.strip() != "":
self.num_seeds += 1
datalines = [i.strip().lower() for i in datalines]
data = {}
step_arr = []
keys = ["min", "avg", "max", "med", "std"]
for line in datalines:
if line.startswith("step "):
_, num = line.split()
step_arr.append(int(num))
else:
line = line.split()
try:
data[line[0]]
except KeyError:
data[line[0]] = {k:[] for k in keys}
for i in range(len(line[1:])):
data[line[0]][keys[i]].append( float(line[i+1]) )
# data is dict of form: data[metric_name][min/avg/max/...] = list(values)
self.data = data
self.step_arr = step_arr
all_tests = []
for fname in filenames:
all_tests.append(TestData(*fname))
def conf_interval_90(means, stds, n):
"""
90 percent confidence interval
X_bar +/- Z (std / sqrt(n))
"""
Z = 1.645
n = n.reshape(-1, 1)
err = Z * stds / np.sqrt(n)
return means - err, means + err, err
def get_mark(data):
data = [i[-1] for i in data]
return data.index(min(data))
def met_name(met):
met = met.lower()
if met == "mse":
return "MSE"
if met == "mae":
return "MAE"
if met == "relpred":
return "Relative Error"
if met == "celcius":
return "Celcius MAE"
return met.title()
metrics = all_tests[0].data.keys()
step_arr = all_tests[0].step_arr
dnames = [i.name for i in all_tests]
# PLOTTING
for met in metrics:
for aggreg in ["min", "avg", "max", "med"]:
plotdata = np.array([i.data[met][aggreg] for i in all_tests])
mark = get_mark(plotdata)
name = args.name + "." + aggreg + "_" + met
make_plot(xrange=step_arr, data=tuple(plotdata), dnames=dnames, title="Prediction " + aggreg.title() + " " + met_name(met),
mark=mark, axlabels=("days", met_name(met)), legendloc="upper left",
marker_step=30, fillbetweens=None,
fillbetween_desc="", ylim=None, ymin=0,
directory="meta_results/"+args.name, filename=name+".png")
if aggreg in ["avg", "med"]:
stds = np.array([i.data[met]["std"] for i in all_tests])
n = np.array([i.num_seeds for i in all_tests])
low, high, error = conf_interval_90(plotdata, stds, n)
fillbetweens = [(low[i], high[i]) for i in range(len(low))]
name = name + ".w_90per_conf"
make_plot(xrange=step_arr, data=tuple(plotdata), dnames=dnames, title="Prediction " + aggreg.title() + " " + met_name(met),
mark=mark, axlabels=("days", met_name(met)), legendloc="upper left",
marker_step=30, fillbetweens=fillbetweens,
fillbetween_desc="with 90% confidence interval", ylim=None, ymin=0,
directory="meta_results/"+args.name, filename=name+".png")
if aggreg == "avg":
# confidence interval bar graph
plotdata = [(i.data[met][aggreg][-1], i.name) for i in all_tests]
plotdata = [plotdata[i] + (error[i][-1],) for i in range(len(plotdata))]
plotdata.sort(key=lambda x: x[0])
data = [i[0] for i in plotdata]
names = [i[1] for i in plotdata]
err = [i[2] for i in plotdata]
for i in range(len(data)):
color = "C" + str(i)
bar = plt.bar(names[i], data[i], alpha=0.2, ec="black",
ecolor=color,
error_kw={"capthick": 1.5},
yerr=err[i], capsize=3000)
bar[0].set_color(color)
plt.title("Step 180 " + aggreg.title() + " " + met_name(met))
plt.xticks(rotation=10)
plt.tight_layout()
# plt.show()
plt.savefig("meta_results/" + args.name + "/" + args.name + "." + met + "_confidence.png")
plt.clf()
# write confidences
with open("meta_results/" + args.name + "/" + args.name + "." + met + "_confidence.txt", "w") as f:
# conf interval on difference
f.write("\t\t")
for t in all_tests:
f.write(t.name + "\t")
for test in all_tests:
f.write("\n" + test.name + "\t")
for othertest in all_tests:
if othertest != test:
mean1 = test.data[met][aggreg][-1]
mean2 = othertest.data[met][aggreg][-1]
std1 = test.data[met]["std"][-1]
std2 = othertest.data[met]["std"][-1]
n1 = test.num_seeds
n2 = othertest.num_seeds
mean_diff = abs(mean1 - mean2)
error_thing = np.sqrt( (std1**2)/n1 + (std2**2)/n2 )
t = mean_diff / error_thing
df_num = ( (std1**2)/n1 + (std2**2)/n2 )**2
den1 = np.maximum(n1*n1*(n1-1), 1)
den2 = np.maximum(n2*n2*(n2-1), 1)
df_denom = (std1**4)/den1 + (std2**4)/den2
df = df_num / df_denom
# p = scipy_stats.norm.cdf(Z)
# p = (2 * p) - 1 # convert to two-sided p value
# pct = p * 100
# p1 = scipy_stats.t.sf(t, df) * 2
p = scipy_stats.t.cdf(t, df)
p = (2 * p) - 1 # two sided
pct = p * 100
# print(test.name, "\tvs\t", othertest.name, "\t", pct)
f.write("{:5.3f}\t".format(pct))
else:
f.write("x \t")
print("\n")