-
Notifications
You must be signed in to change notification settings - Fork 0
/
main.py
117 lines (91 loc) · 3.12 KB
/
main.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
import os
from pprint import pprint
import numpy as np
import argparse
import re
from src.reddit_interface import *
from src.keras_model import *
from src.data_gathering import *
# np.random.seed(0)
# tf.random.set_seed(0)
os.makedirs("models", exist_ok=True)
os.makedirs("data", exist_ok=True)
parser = argparse.ArgumentParser()
parser.add_argument("--name",required=True,type=str,help="name to save this model under")
parser.add_argument("--datasize",type=int,default=2000,help="numbers of examples for each class to use")
parser.add_argument("--epochs",type=int,default=100)
parser.add_argument("--batchsize",type=int,default=128)
parser.add_argument("--test",action="store_true",default=False,help="only test model")
parser.add_argument("--modelname",default="original")
args = parser.parse_args()
### Load Data
# positive
if not os.path.exists("data/pos_data.json"):
p = request_format_save(size=10000, pos=True)
else:
p = load_json_data("pos_data")
# negative
if not os.path.exists("data/neg_data.json"):
n = request_format_save(size=10000, pos=False)
else:
n = load_json_data("neg_data")
pse = load_json_data("pos_sent_emot")
nse = load_json_data("neg_sent_emot")
print("Positive examples:", len(p), "Negative examples:", len(n))
# trim to 1000 words max, remove words with numbers
p = [i.split()[:1000] for i in p]
n = [i.split()[:1000] for i in n]
p = [[j for j in i if not re.match(r"[0-9]", j)] for i in p]
n = [[j for j in i if not re.match(r"[0-9]", j)] for i in n]
p = [" ".join(i) for i in p]
n = [" ".join(i) for i in n]
p_rest = p[args.datasize:]
n_rest = n[args.datasize:]
p = p[:args.datasize]
n = n[:args.datasize]
pse_rest = pse[args.datasize:]
nse_rest = nse[args.datasize:]
pse = pse[:args.datasize]
nse = nse[:args.datasize]
# combine and shuffle
data = np.array(p + n)
sent_emot = np.array(pse + nse)
labels = np.array([1 for i in p] + [0 for i in n])
data_rest = np.array(p_rest + n_rest)
sent_emot = np.array(pse_rest + nse_rest)
labels_rest = np.array([1 for i in p_rest] + [0 for i in n_rest])
indices = np.arange(len(data))
np.random.shuffle(indices)
slabels = labels[indices]
sdata = data[indices]
ssent_emot = sent_emot[indices]
# calculate number of words
d = {}
for i in p+n:
for j in i.split():
d[j] = 0
print(len(d), "unique words")
### Run Model
model = OurModel(sdata, slabels, sent_emot=ssent_emot, args=args)
if not args.test:
model.train()
rest_len = len(data_rest)
data_rest = make_hash_words(data_rest)
def run_tests(model):
model.test(model.x, model.y, name="training set")
model.test(model.valx, model.valy, name="validation set")
if rest_len > 0:
if model.uses_sentiment:
model.test([data_rest, sent_emot], labels_rest, "rest set")
else:
model.test(data_rest, labels_rest, "rest set")
else:
print("No rest set")
print("\n--- Final Model ---")
modelname = "models/" + args.name + "_final.hdf5"
model.model = keras.models.load_model(modelname)
run_tests(model)
print("\n--- Best Validation Loss Model ---")
modelname = "models/" + args.name + "_best_val_loss.hdf5"
model.model = keras.models.load_model(modelname)
run_tests(model)