diff --git a/DataLoader.py b/DataLoader.py index 195102f..1b2208b 100644 --- a/DataLoader.py +++ b/DataLoader.py @@ -8,14 +8,13 @@ import numpy as np from PIL import Image, ImageDraw import cv2 -import matplotlib.pyplot as plt import time import utils -import matplotlib.pyplot as plt class myJAAD(torch.utils.data.Dataset): def __init__(self, args): + print('Loading', args.dtype, 'data ...') if(args.from_file): sequence_centric = pd.read_csv(args.file) @@ -30,16 +29,12 @@ def __init__(self, args): else: #read data + print('Reading data files ...') df = pd.DataFrame() new_index=0 for file in glob.glob(os.path.join(args.jaad_dataset,args.dtype,"*")): temp = pd.read_csv(file) if not temp.empty: - #drop unnecessary columns - temp = temp.drop(columns=['type', 'occlusion', 'nod', 'slow_down', 'speed_up', 'WALKING', 'walking', - 'standing', 'looking', 'handwave', 'clear_path', 'CLEAR_PATH','STANDING', - 'standing_pred', 'looking_pred', 'walking_pred','keypoints', 'crossing_pred']) - temp['file'] = [file for t in range(temp.shape[0])] #assign unique ID to each @@ -51,8 +46,8 @@ def __init__(self, args): temp = temp.sort_values(['ID', 'frame'], axis=0) df = df.append(temp, ignore_index=True) - print('reading files complete') + print('Processing data ...') #create sequence column df.insert(0, 'sequence', df.ID) @@ -119,18 +114,12 @@ def __init__(self, args): sequence_centric = data.copy() - if args.sample: - if args.trainOrVal == 'train': - self.data = sequence_centric.loc[:args.n_train_sequences].copy().reset_index(drop=True) - elif args.trainOrVal == 'val': - self.data = sequence_centric.loc[args.n_train_sequences:].copy().reset_index(drop=True) - - else: - self.data = sequence_centric.copy().reset_index(drop=True) + + self.data = sequence_centric.copy().reset_index(drop=True) self.args = args self.dtype = args.dtype - print(self.dtype, " set loaded") + print(args.dtype, "set loaded") print('*'*30) @@ -195,32 +184,9 @@ def scene_transforms(self, scene): def data_loader(args): - if args.dtype == 'train': - train_set = myJAAD(args) - train_loader = torch.utils.data.DataLoader( - train_set, batch_size=args.batch_size, shuffle=args.loader_shuffle, - pin_memory=args.pin_memory, num_workers=args.loader_workers, drop_last=True) - - args.trainOrVal = 'val' - - val_set = myJAAD(args) - val_loader = torch.utils.data.DataLoader( - val_set, batch_size=args.batch_size, shuffle=args.loader_shuffle, - pin_memory=args.pin_memory, num_workers=args.loader_workers, drop_last=True) - - return train_loader, val_loader - - elif args.dtype == 'val': - - #rgs.file = args.val_file - #rgs.dtype = 'val' - #rgs.trainOrVal = 'test' - #rgs.sample = False - - test_set = myJAAD(args) - - test_loader = torch.utils.data.DataLoader( - test_set, batch_size=args.batch_size, shuffle=args.loader_shuffle, - pin_memory=args.pin_memory, num_workers=args.loader_workers, drop_last=True) + dataset = myJAAD(args) + dataloader = torch.utils.data.DataLoader( + dataset, batch_size=args.batch_size, shuffle=args.loader_shuffle, + pin_memory=args.pin_memory, num_workers=args.loader_workers, drop_last=True) - return test_loader + return dataloader diff --git a/__pycache__/DataLoader.cpython-36.pyc b/__pycache__/DataLoader.cpython-36.pyc new file mode 100644 index 0000000..69b71ad Binary files /dev/null and b/__pycache__/DataLoader.cpython-36.pyc differ diff --git a/__pycache__/network.cpython-36.pyc b/__pycache__/network.cpython-36.pyc new file mode 100644 index 0000000..c4d2cf1 Binary files /dev/null and b/__pycache__/network.cpython-36.pyc differ diff --git a/__pycache__/utils.cpython-36.pyc b/__pycache__/utils.cpython-36.pyc new file mode 100644 index 0000000..777db5b Binary files /dev/null and b/__pycache__/utils.cpython-36.pyc differ diff --git a/prepare_data.py b/prepare_data.py new file mode 100644 index 0000000..5abb1e5 --- /dev/null +++ b/prepare_data.py @@ -0,0 +1,92 @@ +import os +import sys +import argparse +import numpy as np +import pandas as pd + +parser = argparse.ArgumentParser() +parser.add_argument('jaad_path', type=str, help='Path to zhr cloned JAAD repository') +parser.add_argument('train_ratio', type=float, help='Ratio of train video') +parser.add_argument('val_ratio', type=float, help='Ratio of val video') +parser.add_argument('test_ratio', type=float, help='Ratio of test video') + +args = parser.parse_args() + +data_path = args.jaad_path +sys.path.insert(1, data_path+'/') + +import jaad_data + +if not os.path.isdir(os.path.join(data_path, 'processed_annotations')): + os.mkdir(os.path.join(data_path, 'processed_annotations')) + +if not os.path.isdir(os.path.join(data_path, 'processed_annotations', 'train')): + os.mkdir(os.path.join(data_path, 'processed_annotations', 'train')) + +if not os.path.isdir(os.path.join(data_path, 'processed_annotations', 'val')): + os.mkdir(os.path.join(data_path, 'processed_annotations', 'val')) + +if not os.path.isdir(os.path.join(data_path, 'processed_annotations', 'test')): + os.mkdir(os.path.join(data_path, 'processed_annotations', 'test')) + +jaad = jaad_data.JAAD(data_path=data_path) +dataset = jaad.generate_database() + +n_train_video = int(args.train_ratio * 346) +n_val_video = int(args.val_ratio * 346) +n_test_video = int(args.test_ratio * 346) + +videos = list(dataset.keys()) +train_videos = videos[:n_train_video] +val_videos = videos[n_train_video:n_train_video+n_val_video] +test_videos = videos[n_train_video+n_val_video:] + + +for video in dataset: + print('Processing', video, '...') + vid = dataset[video] + data = np.empty((0,8)) + for ped in vid['ped_annotations']: + if vid['ped_annotations'][ped]['behavior']: + frames = np.array(vid['ped_annotations'][ped]['frames']).reshape(-1,1) + ids = np.repeat(vid['ped_annotations'][ped]['old_id'], frames.shape[0]).reshape(-1,1) + bbox = np.array(vid['ped_annotations'][ped]['bbox']) + x = bbox[:,0].reshape(-1,1) + y = bbox[:,1].reshape(-1,1) + w = np.abs(bbox[:,0] - bbox[:,2]).reshape(-1,1) + h = np.abs(bbox[:,1] - bbox[:,3]).reshape(-1,1) + scenefolderpath = np.repeat(os.path.join(data_path, 'scene', video.replace('video_', '')), frames.shape[0]).reshape(-1,1) + + cross = np.array(vid['ped_annotations'][ped]['behavior']['cross']).reshape(-1,1) + + ped_data = np.hstack((frames, ids, x, y, w, h, scenefolderpath, cross)) + data = np.vstack((data, ped_data)) + data_to_write = pd.DataFrame({'frame': data[:,0].reshape(-1), + 'ID': data[:,1].reshape(-1), + 'x': data[:,2].reshape(-1), + 'y': data[:,3].reshape(-1), + 'w': data[:,4].reshape(-1), + 'h': data[:,5].reshape(-1), + 'scenefolderpath': data[:,6].reshape(-1), + 'crossing_true': data[:,7].reshape(-1)}) + data_to_write['filename'] = data_to_write.frame + data_to_write.filename = data_to_write.filename.apply(lambda x: '%04d'%int(x)+'.png') + + if video in train_videos: + data_to_write.to_csv(os.path.join(data_path, 'processed_annotations', 'train', video+'.csv'), index=False) + elif video in val_videos: + data_to_write.to_csv(os.path.join(data_path, 'processed_annotations', 'val', video+'.csv'), index=False) + elif video in test_videos: + data_to_write.to_csv(os.path.join(data_path, 'processed_annotations', 'test', video+'.csv'), index=False) + + + + + + + + + + + + \ No newline at end of file diff --git a/test.py b/test.py index bc0a9de..3790234 100644 --- a/test.py +++ b/test.py @@ -7,7 +7,6 @@ import torchvision import torchvision.transforms as transforms -import matplotlib.pyplot as plt import numpy as np from sklearn.metrics import recall_score, accuracy_score, average_precision_score, precision_score @@ -18,12 +17,13 @@ class args(): def __init__(self): - self.jaad_dataset = '/data/haziq-data/jaad/annotations' #folder containing parsed jaad annotations (used when first time loading data) - self.dtype = 'val' + self.jaad_dataset = '/data/smailait-data/JAAD/processed_annotations' #folder containing parsed jaad annotations (used when first time loading data) + self.dtype = 'test' self.from_file = False #read dataset from csv file or reprocess data - self.file = '/data/smail-data/jaad_val_16_16.csv' - self.save_path = '/data/smail-data/jaad_val_16_16.csv' - self.model_path = '/data/smail-data/multitask_pv_lstm_trained.pkl' + self.save = True + self.file = '/data/smailait-data/jaad_test_16_16.csv' + self.save_path = '/data/smailait-data/jaad_test_16_16.csv' + self.model_path = '/data/smailait-data/models/multitask_pv_lstm_trained.pkl' self.loader_workers = 10 self.loader_shuffle = True self.pin_memory = False @@ -33,12 +33,9 @@ def __init__(self): self.n_epochs = 100 self.hidden_size = 512 self.hardtanh_limit = 100 - self.sample = False - self.n_train_sequences = 40000 - self.trainOrVal = 'test' - self.citywalks = False self.input = 16 self.output = 16 + self.stride = 16 self.skip = 1 self.task = 'bounding_box-intention' self.use_scenes = False diff --git a/train.py b/train.py index cfa2b26..fbf90e0 100644 --- a/train.py +++ b/train.py @@ -6,8 +6,7 @@ import torchvision import torchvision.transforms as transforms - -import matplotlib.pyplot as plt + import numpy as np from sklearn.metrics import recall_score, accuracy_score, average_precision_score, precision_score @@ -17,12 +16,13 @@ class args(): def __init__(self): - self.jaad_dataset = '../../../../data/haziq-data/jaad/annotations' #folder containing parsed jaad annotations (used when first time loading data) + self.jaad_dataset = '/data/smailait-data/JAAD/processed_annotations' #folder containing parsed jaad annotations (used when first time loading data) self.dtype = 'train' self.from_file = False #read dataset from csv file or reprocess data - self.file = '/data/smail-data/jaad_train_16_16.csv' - self.save_path = '/data/smail-data/jaad_train_16_16.csv' - self.model_path = '/data/smail-data/multitask_pv_lstm_trained.pkl' + self.save = True + self.file = '/data/smailait-data/jaad_train_16_16.csv' + self.save_path = '/data/smailait-data/jaad_train_16_16.csv' + self.model_path = '/data/smailait-data/models/multitask_pv_lstm_trained.pkl' self.loader_workers = 10 self.loader_shuffle = True self.pin_memory = False @@ -32,12 +32,9 @@ def __init__(self): self.n_epochs = 100 self.hidden_size = 512 self.hardtanh_limit = 100 - self.sample = False - self.n_train_sequences = 40000 - self.trainOrVal = 'train' - self.citywalks = False self.input = 16 self.output = 16 + self.stride = 16 self.skip = 1 self.task = 'bounding_box-intention' self.use_scenes = False @@ -46,7 +43,11 @@ def __init__(self): args = args() net = network.PV_LSTM(args).to(args.device) -train, val = DataLoader.data_loader(args) +train = DataLoader.data_loader(args) +args.dtype = 'val' +args.save_path = args.save_path.replace('train', 'val') +args.file = args.file.replace('train', 'val') +val = DataLoader.data_loader(args) optimizer = optim.Adam(net.parameters(), lr=args.lr) scheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer, factor=0.5, patience=15, @@ -178,17 +179,7 @@ def __init__(self): '| fde: %.4f'% fde, '| aiou: %.4f'% aiou, '| fiou: %.4f'% fiou, '| state_acc: %.4f'% avg_acc, '| rec: %.4f'% avg_rec, '| pre: %.4f'% avg_pre, '| intention_acc: %.4f'% intent_acc, '| t:%.4f'%(time.time()-start)) - -print('='*100) -plt.figure(figsize=(10,8)) -plt.plot(list(range(len(train_s_scores))), train_s_scores, label = 'BB Training loss') -plt.plot(list(range(len(val_s_scores))), val_s_scores, label = 'BB Validation loss') -plt.plot(list(range(len(train_c_scores))), train_c_scores, label = 'Intention Training loss') -plt.plot(list(range(len(val_c_scores))), val_c_scores, label = 'Intention Validation loss') -plt.xlabel('epoch') -plt.ylabel('Mean square error loss') -plt.legend() -plt.show() + print('='*100) print('Saving ...') torch.save(net.state_dict(), args.model_path)