✨ feat: Add CF - RR Ensemble main2 (#24)

.gitignore

@@ -1,4 +1,5 @@
 .DS_Store
+*.pyc
 */*/*.pyc
 */*.pyc
 */*/*/*.pyc

src/data/dl_data.py

@@ -5,7 +5,7 @@
 import torch.nn as nn
 from torch.utils.data import TensorDataset, DataLoader, Dataset
 from src.utils import EarlyStopping
-
+from copy import deepcopy
 
 def age_map(x: int) -> int:
     x = int(x)
@@ -166,28 +166,77 @@ def dl_data_split(args, data):
                                                         shuffle=True
                                                         )
     data['X_train'], data['X_valid'], data['y_train'], data['y_valid'] = X_train, X_valid, y_train, y_valid
-    print(f"X_train: \n {data['X_train'].sample(5)}")
+    print(f"X_train: \n {data['X_train'].sample(10)}, len: {len(data['X_train'])}")
     return data
 
-def dl_data_loader(args, data):
-#     print(torch.LongTensor(data['X_train'].values))
-#     print('e')
-#     print(torch.FloatTensor(data['y_train'].values) / 10.0)
-#     raise 'ok'
 
-    if args.ZEROONE:
-        train_dataset = TensorDataset(torch.LongTensor(data['X_train'].values), torch.FloatTensor(data['y_train'].values) / 10.0)
-        valid_dataset = TensorDataset(torch.LongTensor(data['X_valid'].values), torch.FloatTensor(data['y_valid'].values) / 10.0)
-    else:
-        train_dataset = TensorDataset(torch.LongTensor(data['X_train'].values), torch.FloatTensor(data['y_train'].values))
-        valid_dataset = TensorDataset(torch.LongTensor(data['X_valid'].values), torch.FloatTensor(data['y_valid'].values))
+def dl_data_loader(args, data, cf=False, range_str='09', last_valid=False, is_boosting=False):
+    new_data = deepcopy(data) 
+
+    if last_valid:
+        # last_valid 는 전체를 그냥 통과한다.
+        range_str='09'
 
+    if cf:        
+        def convert_rating_to_label(x):
+            for i, r in enumerate(data['ranges']):
+                if int(r[0])+1 <= x <= int(r[1])+1:
+                    return i
+
+        # 클래시피케이션이면 라벨로 변경한다.
+        #print(new_data['y_train'])
+        new_data['y_train'] = new_data['y_train'].apply(convert_rating_to_label)
+        new_data['y_valid'] = new_data['y_valid'].apply(convert_rating_to_label)
+
+        train_dataset = TensorDataset(torch.LongTensor(new_data['X_train'].values), torch.LongTensor(new_data['y_train'].values))
+        valid_dataset = TensorDataset(torch.LongTensor(new_data['X_valid'].values), torch.LongTensor(new_data['y_valid'].values))
+    else:
+        # 리그레션이면 range_str에 따라 잘라서 전달한다.
+        live_indices_train =\
+            new_data['y_train'][(new_data['y_train'] >= int(range_str[0])+1) & (new_data['y_train'] <= int(range_str[1])+1)].index
+
+        live_indices_valid =\
+            new_data['y_valid'][(new_data['y_valid'] >= int(range_str[0])+1) & (new_data['y_valid'] <= int(range_str[1])+1)].index
+
+
+        new_data['X_train'] = new_data['X_train'].loc[live_indices_train]
+        new_data['y_train'] = new_data['y_train'].loc[live_indices_train]
+
+        new_data['X_valid'] = new_data['X_valid'].loc[live_indices_valid]
+        new_data['y_valid'] = new_data['y_valid'].loc[live_indices_valid]
+
+        # print(new_data['X_train'])
+        # print(new_data['y_train'])
+
+        if args.ZEROONE:
+            new_data['y_train'] = new_data['y_train'].astype(np.float64) / 10.0
+            new_data['y_valid'] = new_data['y_valid'].astype(np.float64) / 10.0
+
+        if args.RR_MODEL in ('XGB', 'LGBM', 'CATB'):
+            new_data['train_dataloader'], new_data['valid_dataloader'], new_data['test_dataloader'] = \
+                (new_data['X_train'], new_data['y_train']), (new_data['X_valid'], new_data['y_valid']), (new_data['test'], None)
+
+        train_dataset = TensorDataset(torch.LongTensor(new_data['X_train'].values), torch.FloatTensor(new_data['y_train'].values))
+        valid_dataset = TensorDataset(torch.LongTensor(new_data['X_valid'].values), torch.FloatTensor(new_data['y_valid'].values))
+        # print(new_data['y_valid'])
+
     test_dataset = TensorDataset(torch.LongTensor(data['test'].values))
 
-    train_dataloader = DataLoader(train_dataset, batch_size=args.BATCH_SIZE, shuffle=args.DATA_SHUFFLE, num_workers = 4)
-    valid_dataloader = DataLoader(valid_dataset, batch_size=args.BATCH_SIZE, shuffle=args.DATA_SHUFFLE, num_workers = 4)
-    test_dataloader = DataLoader(test_dataset, batch_size=args.BATCH_SIZE, shuffle=False, num_workers = 4)
+    if cf:
+        train_dataloader = DataLoader(train_dataset, batch_size=args.CF_BATCH_SIZE, shuffle=args.DATA_SHUFFLE, num_workers = 4)
+        valid_dataloader = DataLoader(valid_dataset, batch_size=args.CF_BATCH_SIZE, shuffle=args.DATA_SHUFFLE, num_workers = 4)
+        test_dataloader = DataLoader(test_dataset, batch_size=args.CF_BATCH_SIZE, shuffle=False, num_workers = 4)
 
-    data['train_dataloader'], data['valid_dataloader'], data['test_dataloader'] = train_dataloader, valid_dataloader, test_dataloader
+    else:
+        train_dataloader = DataLoader(train_dataset, batch_size=args.RR_BATCH_SIZE, shuffle=args.DATA_SHUFFLE, num_workers = 4)
+        valid_dataloader = DataLoader(valid_dataset, batch_size=args.RR_BATCH_SIZE, shuffle=args.DATA_SHUFFLE, num_workers = 4)
+        test_dataloader = DataLoader(test_dataset, batch_size=args.RR_BATCH_SIZE, shuffle=False, num_workers = 4)
 
-    return data
+    if is_boosting:
+        new_data['train_dataloader'], new_data['valid_dataloader'], new_data['test_dataloader'] = \
+            (new_data['X_train'], new_data['y_train']), (new_data['X_valid'], new_data['y_valid']), (new_data['test'], None)
+
+    else:
+        new_data['train_dataloader'], new_data['valid_dataloader'], new_data['test_dataloader'] = train_dataloader, valid_dataloader, test_dataloader
+
+    return new_data

src/models/_models.py

@@ -3,9 +3,18 @@
 import torch
 import torch.nn as nn
 
+from sklearn.metrics import accuracy_score
+from sklearn.metrics import confusion_matrix
+
 def rmse(real: list, predict: list) -> float:
     pred = np.array(predict)
     return np.sqrt(np.mean((real-pred) ** 2))
+
+def acc(real: list, predict: list) -> float:
+    return accuracy_score(real, predict)
+
+def confusion_mat(real: list, predict: list):
+    return confusion_matrix(real, predict, normalize='true')
 
 
 class RMSELoss(torch.nn.Module):
@@ -30,6 +39,17 @@ def forward(self, x, y):
         return loss
 
 
+class CrossEntropyLoss(torch.nn.Module):
+    def __init__(self, weight):
+        self.weight = weight
+        super(CrossEntropyLoss,self).__init__()
+
+    def forward(self, x, y):
+        criterion = nn.CrossEntropyLoss(weight=self.weight)
+        loss = criterion(x, y)
+        return loss
+
+
 class FactorizationMachine(nn.Module):
 
     def __init__(self, reduce_sum:bool=True):
@@ -96,18 +116,27 @@ def forward(self, x: torch.Tensor):
 
 class _FactorizationMachineModel(nn.Module):
 
-    def __init__(self, field_dims: np.ndarray, embed_dim: int):
+    def __init__(self, field_dims: np.ndarray, embed_dim: int, last_dim=1):
         super().__init__()
         self.embedding = FeaturesEmbedding(field_dims, embed_dim)
         self.linear = FeaturesLinear(field_dims)
         self.fm = FactorizationMachine(reduce_sum=True)
+
+        # 클래시파이어 수정 부분
+        self.last_dim = last_dim
+        if last_dim != 1:
+            self.last_classifier = nn.Linear(1, last_dim)
+
 
     def forward(self, x: torch.Tensor):
         """
         :param x: Long tensor of size ``(batch_size, num_fields)``
         """
         x = self.linear(x) + self.fm(self.embedding(x))
         # return torch.sigmoid(x.squeeze(1))
+        # 클래시파이어 수정 부분
+        if self.last_dim != 1:
+            x = self.last_classifier(x)
         return x.squeeze(1)
 
 class FieldAwareFactorizationMachine(nn.Module):
@@ -174,7 +203,7 @@ def forward(self, x):
 
 class _NeuralCollaborativeFiltering(nn.Module):
 
-    def __init__(self, field_dims, user_field_idx, item_field_idx, embed_dim, mlp_dims, dropout):
+    def __init__(self, field_dims, user_field_idx, item_field_idx, embed_dim, mlp_dims, dropout, last_dim=1):
         super().__init__()
         self.user_field_idx = user_field_idx
         self.item_field_idx = item_field_idx
@@ -183,6 +212,11 @@ def __init__(self, field_dims, user_field_idx, item_field_idx, embed_dim, mlp_di
         self.mlp = MultiLayerPerceptron(self.embed_output_dim, mlp_dims, dropout, output_layer=False)
         self.fc = torch.nn.Linear(mlp_dims[-1] + 2 * embed_dim, 1)
 
+        # 클래시파이어 수정 부분
+        self.last_dim = last_dim
+        if last_dim != 1:
+            self.last_classifier = nn.Linear(1, last_dim)
+
     def forward(self, x):
         """
         :param x: Long tensor of size ``(batch_size, num_user_fields)``
@@ -196,8 +230,13 @@ def forward(self, x):
         x = self.mlp(x.view(-1, self.embed_output_dim))
         x = torch.cat([gmf, x, context_x], dim=1)
 
-        x = self.fc(x).squeeze(1)
-        return x
+        # 클래시파이어 수정 부분        
+        x = self.fc(x)
+
+        if self.last_dim != 1:
+            x = self.last_classifier(x)
+
+        return x.squeeze(1)
 
 class _WideAndDeepModel(nn.Module):
 

src/models/context_models.py

@@ -7,29 +7,47 @@
 import torch.optim as optim
 
 from ._models import _FactorizationMachineModel, _FieldAwareFactorizationMachineModel
-from ._models import rmse, RMSELoss
+from ._models import rmse, acc, confusion_mat, RMSELoss, SmoothL1Loss, CrossEntropyLoss
 from src.utils import EarlyStopping
 
 class FactorizationMachineModel:
 
-    def __init__(self, args, data):
+    def __init__(self, args, data, cf=False):
         super().__init__()
         self.args = args
-        self.criterion = RMSELoss()
+        ## 클래시파이어로 변환하는 과정 및 로스 교체 코드
+        self.cf = cf
+        if cf:
+            last_dim = len(data['ranges'])
+            weight = torch.FloatTensor([0.43, 0.27, 0.35]).to(args.DEVICE)
+            print('Loss: CrossEntropyLoss')
+            self.criterion = CrossEntropyLoss(weight=weight)
+        else:
+            last_dim = 1
+            if args.LOSS == 'sl1':
+                print('Loss: sl1')
+                self.criterion = SmoothL1Loss(self.args.BETA)
+            elif args.LOSS == 'rmse':
+                print('Loss: rmse')
+                self.criterion = RMSELoss()
 
         self.train_dataloader = data['train_dataloader']
         self.valid_dataloader = data['valid_dataloader']
         self.field_dims = data['field_dims']
 
         self.embed_dim = args.FM_EMBED_DIM
         self.epochs = args.EPOCHS
-        self.learning_rate = args.LR
+        ## 리그레션 일 시 클래시파이어 일시 달라짐
+        if cf:
+            self.learning_rate = args.CF_LR
+        else:
+            self.learning_rate = args.RR_LR
         self.weight_decay = args.WEIGHT_DECAY
         self.log_interval = 100
 
         self.device = args.DEVICE
 
-        self.model = _FactorizationMachineModel(self.field_dims, self.embed_dim).to(self.device)
+        self.model = _FactorizationMachineModel(self.field_dims, self.embed_dim, last_dim=last_dim).to(self.device)
         self.optimizer = torch.optim.Adam(params=self.model.parameters(), lr=self.learning_rate, amsgrad=True, weight_decay=self.weight_decay)
 
 
@@ -45,7 +63,12 @@ def train(self, fold_num):
                 fields, target = fields.to(self.device), target.to(self.device)
 
                 y = self.model(fields)
-                loss = self.criterion(y, target.float())
+
+                # 클래시파이어 부분
+                if self.cf:
+                    loss = self.criterion(y, target.long())
+                else:
+                    loss = self.criterion(y, target.float())
 
                 loss.backward()
                 self.optimizer.step()
@@ -64,7 +87,7 @@ def train(self, fold_num):
         formatted_user_num = format(self.args.USER_NUM, '02')
         formatted_book_num = format(self.args.BOOK_NUM, '02')
         ppath = os.path.join(self.args.SAVE_PATH,
-            self.args.MODEL,
+            self.args.CF_MODEL, '+', self.args.RR_MODEL, ## 클래시파이어 수정 부분
             f"u{formatted_user_num}_b{formatted_book_num}",
             f"fold{fold_num}",
             'checkpoint.pt')
@@ -85,6 +108,18 @@ def predict_train(self):
                 y = self.model(fields)
                 targets.extend(target.tolist())
                 predicts.extend(y.tolist())
+
+        if self.cf:
+            # print(np.argmax(predicts, axis=1), targets)
+            t = np.get_printoptions()
+            np.set_printoptions(precision=2)
+
+            print('[confusion matrix] row: real, col: pred\n', confusion_mat(targets, np.argmax(predicts, axis=1)) * 100)
+            print('[classification acc]:', f'{acc(targets, np.argmax(predicts, axis=1)) * 100:.3f}%')
+            np.set_printoptions(precision=t['precision'])
+
+            return rmse(targets, np.argmax(predicts, axis=1))
+
         if self.args.ZEROONE:
             return rmse([t * 10.0 for t in targets], [p * 10.0 for p in predicts])
         else:
@@ -99,6 +134,10 @@ def predict(self, dataloader):
                 fields = fields[0].to(self.device)
                 y = self.model(fields)
                 predicts.extend(y.tolist())
+
+        # 클래시파이어 부분
+        if self.cf:
+            return np.argmax(predicts, axis=1)
         return predicts