Bug fix

DataLoader.py

@@ -8,14 +8,13 @@
 import numpy as np
 from PIL import Image, ImageDraw
 import cv2
-import matplotlib.pyplot as plt
 import time
 import utils
-import matplotlib.pyplot as plt
 
 
 class myJAAD(torch.utils.data.Dataset):
     def __init__(self, args):
+        print('Loading', args.dtype, 'data ...')
 
         if(args.from_file):
             sequence_centric = pd.read_csv(args.file)
@@ -30,16 +29,12 @@ def __init__(self, args):
 
         else:
             #read data
+            print('Reading data files ...')
             df = pd.DataFrame()
             new_index=0
             for file in glob.glob(os.path.join(args.jaad_dataset,args.dtype,"*")):
                 temp = pd.read_csv(file)
                 if not temp.empty:
-                    #drop unnecessary columns
-                    temp = temp.drop(columns=['type', 'occlusion', 'nod', 'slow_down', 'speed_up', 'WALKING', 'walking',
-                   'standing', 'looking', 'handwave', 'clear_path', 'CLEAR_PATH','STANDING', 
-                   'standing_pred', 'looking_pred', 'walking_pred','keypoints', 'crossing_pred'])
-
                     temp['file'] = [file for t in range(temp.shape[0])]
 
                     #assign unique ID to each 
@@ -51,8 +46,8 @@ def __init__(self, args):
                     temp = temp.sort_values(['ID', 'frame'], axis=0)
 
                     df = df.append(temp, ignore_index=True)
-            print('reading files complete')
 
+            print('Processing data ...')
             #create sequence column
             df.insert(0, 'sequence', df.ID)
 
@@ -119,18 +114,12 @@ def __init__(self, args):
 
             sequence_centric = data.copy()
 
-        if args.sample:
-            if args.trainOrVal == 'train':
-                self.data = sequence_centric.loc[:args.n_train_sequences].copy().reset_index(drop=True)
-            elif args.trainOrVal == 'val':
-                self.data = sequence_centric.loc[args.n_train_sequences:].copy().reset_index(drop=True)
-
-        else:
-            self.data = sequence_centric.copy().reset_index(drop=True)
+
+        self.data = sequence_centric.copy().reset_index(drop=True)
 
         self.args = args
         self.dtype = args.dtype
-        print(self.dtype, " set loaded")
+        print(args.dtype, "set loaded")
         print('*'*30)
 
 
@@ -195,32 +184,9 @@ def scene_transforms(self, scene):
 
 
 def data_loader(args):
-    if args.dtype == 'train':
-        train_set = myJAAD(args)
-        train_loader = torch.utils.data.DataLoader(
-            train_set, batch_size=args.batch_size, shuffle=args.loader_shuffle,
-            pin_memory=args.pin_memory, num_workers=args.loader_workers, drop_last=True)
-
-        args.trainOrVal = 'val'
-
-        val_set = myJAAD(args)
-        val_loader = torch.utils.data.DataLoader(
-            val_set, batch_size=args.batch_size, shuffle=args.loader_shuffle,
-            pin_memory=args.pin_memory, num_workers=args.loader_workers, drop_last=True)
-
-        return train_loader, val_loader
-
-    elif args.dtype == 'val':
-
-        #rgs.file = args.val_file
-        #rgs.dtype = 'val'
-        #rgs.trainOrVal = 'test'
-        #rgs.sample = False
-
-        test_set = myJAAD(args)
-
-        test_loader = torch.utils.data.DataLoader(
-            test_set, batch_size=args.batch_size, shuffle=args.loader_shuffle,
-            pin_memory=args.pin_memory, num_workers=args.loader_workers, drop_last=True)
+    dataset = myJAAD(args)
+    dataloader = torch.utils.data.DataLoader(
+        dataset, batch_size=args.batch_size, shuffle=args.loader_shuffle,
+        pin_memory=args.pin_memory, num_workers=args.loader_workers, drop_last=True)
 
-        return test_loader
+    return dataloader

prepare_data.py

@@ -0,0 +1,92 @@
+import os
+import sys
+import argparse
+import numpy as np
+import pandas as pd
+
+parser = argparse.ArgumentParser()
+parser.add_argument('jaad_path', type=str, help='Path to zhr cloned JAAD repository')
+parser.add_argument('train_ratio', type=float, help='Ratio of train video')
+parser.add_argument('val_ratio', type=float, help='Ratio of val video')
+parser.add_argument('test_ratio', type=float, help='Ratio of test video')
+
+args = parser.parse_args()
+
+data_path = args.jaad_path
+sys.path.insert(1, data_path+'/')
+
+import jaad_data
+
+if not os.path.isdir(os.path.join(data_path, 'processed_annotations')):
+    os.mkdir(os.path.join(data_path, 'processed_annotations'))
+
+if not os.path.isdir(os.path.join(data_path, 'processed_annotations', 'train')):    
+    os.mkdir(os.path.join(data_path, 'processed_annotations', 'train'))
+
+if not os.path.isdir(os.path.join(data_path, 'processed_annotations', 'val')):
+    os.mkdir(os.path.join(data_path, 'processed_annotations', 'val'))
+
+if not os.path.isdir(os.path.join(data_path, 'processed_annotations', 'test')):
+    os.mkdir(os.path.join(data_path, 'processed_annotations', 'test'))
+
+jaad = jaad_data.JAAD(data_path=data_path)
+dataset = jaad.generate_database()
+
+n_train_video = int(args.train_ratio * 346)
+n_val_video = int(args.val_ratio * 346)
+n_test_video = int(args.test_ratio * 346)
+
+videos = list(dataset.keys())
+train_videos = videos[:n_train_video]
+val_videos = videos[n_train_video:n_train_video+n_val_video]
+test_videos = videos[n_train_video+n_val_video:]
+
+
+for video in dataset:
+    print('Processing', video, '...')
+    vid = dataset[video]
+    data = np.empty((0,8))
+    for ped in vid['ped_annotations']:
+        if vid['ped_annotations'][ped]['behavior']:
+            frames = np.array(vid['ped_annotations'][ped]['frames']).reshape(-1,1)
+            ids = np.repeat(vid['ped_annotations'][ped]['old_id'], frames.shape[0]).reshape(-1,1)
+            bbox = np.array(vid['ped_annotations'][ped]['bbox'])
+            x = bbox[:,0].reshape(-1,1)
+            y = bbox[:,1].reshape(-1,1)
+            w = np.abs(bbox[:,0] - bbox[:,2]).reshape(-1,1)
+            h = np.abs(bbox[:,1] - bbox[:,3]).reshape(-1,1)
+            scenefolderpath = np.repeat(os.path.join(data_path, 'scene', video.replace('video_', '')), frames.shape[0]).reshape(-1,1)
+
+            cross = np.array(vid['ped_annotations'][ped]['behavior']['cross']).reshape(-1,1)
+
+            ped_data = np.hstack((frames, ids, x, y, w, h, scenefolderpath, cross))
+            data = np.vstack((data, ped_data))
+    data_to_write = pd.DataFrame({'frame': data[:,0].reshape(-1), 
+                                  'ID': data[:,1].reshape(-1), 
+                                  'x': data[:,2].reshape(-1), 
+                                  'y': data[:,3].reshape(-1), 
+                                  'w': data[:,4].reshape(-1), 
+                                  'h': data[:,5].reshape(-1), 
+                                  'scenefolderpath': data[:,6].reshape(-1), 
+                                  'crossing_true': data[:,7].reshape(-1)})
+    data_to_write['filename'] = data_to_write.frame
+    data_to_write.filename = data_to_write.filename.apply(lambda x: '%04d'%int(x)+'.png')
+
+    if video in train_videos:
+        data_to_write.to_csv(os.path.join(data_path, 'processed_annotations', 'train', video+'.csv'), index=False)
+    elif video in val_videos:
+        data_to_write.to_csv(os.path.join(data_path, 'processed_annotations', 'val', video+'.csv'), index=False)
+    elif video in test_videos:
+        data_to_write.to_csv(os.path.join(data_path, 'processed_annotations', 'test', video+'.csv'), index=False)
+
+
+
+
+
+
+
+
+
+
+
+

test.py

@@ -7,7 +7,6 @@
 import torchvision
 import torchvision.transforms as transforms
 
-import matplotlib.pyplot as plt
 import numpy as np
 from sklearn.metrics import recall_score, accuracy_score, average_precision_score, precision_score
 
@@ -18,12 +17,13 @@
 
 class args():
     def __init__(self):
-        self.jaad_dataset = '/data/haziq-data/jaad/annotations' #folder containing parsed jaad annotations (used when first time loading data)
-        self.dtype        = 'val'
+        self.jaad_dataset = '/data/smailait-data/JAAD/processed_annotations' #folder containing parsed jaad annotations (used when first time loading data)
+        self.dtype        = 'test'
         self.from_file    = False #read dataset from csv file or reprocess data
-        self.file         = '/data/smail-data/jaad_val_16_16.csv'
-        self.save_path    = '/data/smail-data/jaad_val_16_16.csv'
-        self.model_path    = '/data/smail-data/multitask_pv_lstm_trained.pkl'
+        self.save         = True
+        self.file         = '/data/smailait-data/jaad_test_16_16.csv'
+        self.save_path    = '/data/smailait-data/jaad_test_16_16.csv'
+        self.model_path    = '/data/smailait-data/models/multitask_pv_lstm_trained.pkl'
         self.loader_workers = 10
         self.loader_shuffle = True
         self.pin_memory     = False
@@ -33,12 +33,9 @@ def __init__(self):
         self.n_epochs       = 100
         self.hidden_size    = 512
         self.hardtanh_limit = 100
-        self.sample         = False
-        self.n_train_sequences = 40000
-        self.trainOrVal = 'test'
-        self.citywalks  = False
         self.input  = 16
         self.output = 16
+        self.stride = 16
         self.skip   = 1
         self.task   = 'bounding_box-intention'
         self.use_scenes = False       

train.py

@@ -6,8 +6,7 @@
 
 import torchvision
 import torchvision.transforms as transforms
-
-import matplotlib.pyplot as plt
+
 import numpy as np
 from sklearn.metrics import recall_score, accuracy_score, average_precision_score, precision_score
 
@@ -17,12 +16,13 @@
 
 class args():
     def __init__(self):
-        self.jaad_dataset = '../../../../data/haziq-data/jaad/annotations' #folder containing parsed jaad annotations (used when first time loading data)
+        self.jaad_dataset = '/data/smailait-data/JAAD/processed_annotations' #folder containing parsed jaad annotations (used when first time loading data)
         self.dtype        = 'train'
         self.from_file    = False #read dataset from csv file or reprocess data
-        self.file         = '/data/smail-data/jaad_train_16_16.csv'
-        self.save_path    = '/data/smail-data/jaad_train_16_16.csv'
-        self.model_path    = '/data/smail-data/multitask_pv_lstm_trained.pkl'
+        self.save         = True
+        self.file         = '/data/smailait-data/jaad_train_16_16.csv'
+        self.save_path    = '/data/smailait-data/jaad_train_16_16.csv'
+        self.model_path    = '/data/smailait-data/models/multitask_pv_lstm_trained.pkl'
         self.loader_workers = 10
         self.loader_shuffle = True
         self.pin_memory     = False
@@ -32,12 +32,9 @@ def __init__(self):
         self.n_epochs       = 100
         self.hidden_size    = 512
         self.hardtanh_limit = 100
-        self.sample         = False
-        self.n_train_sequences = 40000
-        self.trainOrVal = 'train'
-        self.citywalks  = False
         self.input  = 16
         self.output = 16
+        self.stride = 16
         self.skip   = 1
         self.task   = 'bounding_box-intention'
         self.use_scenes = False       
@@ -46,7 +43,11 @@ def __init__(self):
 args = args()
 
 net = network.PV_LSTM(args).to(args.device)
-train, val = DataLoader.data_loader(args)
+train = DataLoader.data_loader(args)
+args.dtype = 'val'
+args.save_path = args.save_path.replace('train', 'val')
+args.file = args.file.replace('train', 'val')
+val = DataLoader.data_loader(args)
 
 optimizer = optim.Adam(net.parameters(), lr=args.lr)
 scheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer, factor=0.5, patience=15, 
@@ -178,17 +179,7 @@ def __init__(self):
           '| fde: %.4f'% fde, '| aiou: %.4f'% aiou, '| fiou: %.4f'% fiou, '| state_acc: %.4f'% avg_acc, 
           '| rec: %.4f'% avg_rec, '| pre: %.4f'% avg_pre, '| intention_acc: %.4f'% intent_acc, 
           '| t:%.4f'%(time.time()-start))
-
-print('='*100)
-plt.figure(figsize=(10,8))
-plt.plot(list(range(len(train_s_scores))), train_s_scores, label = 'BB Training loss')
-plt.plot(list(range(len(val_s_scores))), val_s_scores, label = 'BB Validation loss')
-plt.plot(list(range(len(train_c_scores))), train_c_scores, label = 'Intention Training loss')
-plt.plot(list(range(len(val_c_scores))), val_c_scores, label = 'Intention Validation loss')
-plt.xlabel('epoch')
-plt.ylabel('Mean square error loss')
-plt.legend()
-plt.show()
+
 print('='*100) 
 print('Saving ...')
 torch.save(net.state_dict(), args.model_path)