-
Notifications
You must be signed in to change notification settings - Fork 0
/
3_remove_long_lines.py
executable file
·52 lines (37 loc) · 1.79 KB
/
3_remove_long_lines.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
import argparse
from os.path import join
from tqdm import tqdm
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument('--data-dir', default='data_dir')
args = parser.parse_args()
for dataset_split in ['train', 'test']:
with open(join(args.data_dir, 'ar.org.%s' % dataset_split), 'r') as file:
ar_org_lines = file.readlines()
with open(join(args.data_dir, 'ar.bpe.%s' % dataset_split), 'r') as file:
ar_bpe_lines = file.readlines()
with open(join(args.data_dir, 'en.org.%s' % dataset_split), 'r') as file:
en_org_lines = file.readlines()
with open(join(args.data_dir, 'en.bpe.%s' % dataset_split), 'r') as file:
en_bpe_lines = file.readlines()
lines = zip(ar_org_lines, ar_bpe_lines, en_org_lines, en_bpe_lines)
ar_org_lines = list()
ar_bpe_lines = list()
en_org_lines = list()
en_bpe_lines = list()
for line in lines:
if len(line[1].split()) > 50 or len(line[3].split()) > 50:
continue
ar_org_lines.append(line[0].strip())
ar_bpe_lines.append(line[1].strip())
en_org_lines.append(line[2].strip())
en_bpe_lines.append(line[3].strip())
with open(join(args.data_dir, 'ar.org.%s' % dataset_split), 'w') as file:
file.write('\n'.join(ar_org_lines))
with open(join(args.data_dir, 'ar.bpe.%s' % dataset_split), 'w') as file:
file.write('\n'.join(ar_bpe_lines))
with open(join(args.data_dir, 'en.org.%s' % dataset_split), 'w') as file:
file.write('\n'.join(en_org_lines))
with open(join(args.data_dir, 'en.bpe.%s' % dataset_split), 'w') as file:
file.write('\n'.join(en_bpe_lines))
print("Done!")