Merge pull request #27 from Zihang-Xu-2002/ltsm-stack

anomaly detection & json hyperparam

ltsm/data_pipeline/anormly_pipeline.py

@@ -12,11 +12,14 @@
 import random
 import ipdb
 from torch import nn
+import json
 
 from ltsm.data_provider.data_factory import get_datasets
 from ltsm.data_provider.data_loader import HF_Dataset
 from ltsm.data_pipeline.model_manager import ModelManager
 
+from sklearn.metrics import precision_score, recall_score, f1_score
+
 import logging
 from transformers import (
     Trainer,
@@ -48,6 +51,22 @@ def compute_loss(self, model, inputs, return_outputs=False):
         loss = nn.functional.cross_entropy(outputs, labels)
         #loss = nn.functional.cross_entropy(outputs.reshape(B*L,-1), inputs["labels"][:,1:].long().reshape(B*L))
         return (loss, outputs) if return_outputs else loss
+    def compute_metrics(self, p):
+        preds = p.predictions[0] if isinstance(p.predictions, tuple) else p.predictions
+        print(preds.shape, p.label_ids.shape)
+        preds = np.squeeze(preds)
+        if preds.shape != p.label_ids.shape:
+            label_ids = np.squeeze(p.label_ids)
+        else:
+            label_ids = p.label_ids
+        print(preds.shape, label_ids.shape)
+        preds_class = (preds > 0.5).astype(int)
+
+        return {
+                "precision": precision_score(label_ids, preds_class, average="micro"),
+                "recall": recall_score(label_ids, preds_class, average="micro"),
+                "f1": f1_score(label_ids, preds_class, average="micro")              
+        }
 
 class AnomalyTrainingPipeline():
     """
@@ -140,64 +159,28 @@ def run(self):
 def anomaly_get_args():
     parser = argparse.ArgumentParser(description='LTSM')
 
-    # Basic Config
-    parser.add_argument('--model_id', type=str, default='test_run', help='model id')
-    parser.add_argument('--model_name_or_path', type=str, default="gpt2-medium", help='model name')
-    parser.add_argument('--seed', type=int, default=2024, help='random seed')
-    parser.add_argument('--device', type=str, default="cuda:0")
-    parser.add_argument('--checkpoints', type=str, default='./checkpoints/')
-
-    # Data Settings
-    parser.add_argument('--data_path', nargs='+', default='dataset/weather.csv', help='data files')
-    parser.add_argument('--test_data_path_list', nargs='+', required=True, help='test data file')
-    parser.add_argument('--prompt_data_path', type=str, default='./weather.csv', help='prompt data file')
-    parser.add_argument('--data_processing', type=str, default="standard_scaler", help='data processing method')
-    parser.add_argument('--train_ratio', type=float, default=0.7, help='train data ratio')
-    parser.add_argument('--val_ratio', type=float, default=0.1, help='validation data ratio')
-    parser.add_argument('--do_anomaly', type=bool, default=True, help='do anomaly detection')
-
-    # Forecasting Settings
-    parser.add_argument('--seq_len', type=int, default=113, help='input sequence length')
-    parser.add_argument('--pred_len', type=int, default=None, help='prediction sequence length')
-    parser.add_argument('--prompt_len', type=int, default=133, help='prompt sequence length')
-
-    # Model Settings
-    parser.add_argument('--lora', action="store_true", help='use lora')
-    parser.add_argument('--lora_dim', type=int, default=128, help='dimension of lora')
-    parser.add_argument('--gpt_layers', type=int, default=3, help='number of gpt layers')
-    parser.add_argument('--d_model', type=int, default=1024, help='dimension of model')
-    parser.add_argument('--n_heads', type=int, default=16, help='number of heads')
-    parser.add_argument('--d_ff', type=int, default=512, help='dimension of fcn')
-    parser.add_argument('--dropout', type=float, default=0.2, help='dropout')
-    parser.add_argument('--enc_in', type=int, default=1, help='encoder input size')
-    parser.add_argument('--c_out', type=int, default=862, help='output size')
-    parser.add_argument('--patch_size', type=int, default=16, help='patch size')
-    parser.add_argument('--pretrain', type=int, default=1, help='is pretrain')
-    parser.add_argument('--local_pretrain', type=str, default="None", help='local pretrain weight')
-    parser.add_argument('--freeze', type=int, default=1, help='is model weight frozen')
-    parser.add_argument('--model', type=str, default='model', help='model name, , options:[LTSM, LTSM_WordPrompt, LTSM_Tokenizer]')
-    parser.add_argument('--stride', type=int, default=8, help='stride')
-    parser.add_argument('--tmax', type=int, default=10, help='tmax')
-
-    # Training Settings 
-    parser.add_argument('--eval', type=int, default=0, help='evaluation')
-    parser.add_argument('--itr', type=int, default=1, help='experiments times')
-    parser.add_argument('--output_dir', type=str, default='output/ltsm_train_lr0005/', help='output directory')
-    parser.add_argument('--downsample_rate', type=int, default=100, help='downsample rate')
-    parser.add_argument('--llm_layers', type=int, default=32)
-    parser.add_argument('--decay_fac', type=float, default=0.75, help='decay factor')
-    parser.add_argument('--learning_rate', type=float, default=0.0001, help='learning rate')
-    parser.add_argument('--batch_size', type=int, default=512, help='batch size')
-    parser.add_argument('--num_workers', type=int, default=10, help='number of workers')
-    parser.add_argument('--train_epochs', type=int, default=1, help='number of epochs')
-    parser.add_argument('--lradj', type=str, default='type1', help='learning rate adjustment type')
-    parser.add_argument('--patience', type=int, default=3, help='early stopping patience')
-    parser.add_argument('--gradient_accumulation_steps', type=int, default=64, help='gradient accumulation steps')
+    parser.add_argument('--config_path', type=str, required=True, help='config path')
     args, unknown = parser.parse_known_args()
+    config_path = args.config_path
+
+    with open(config_path, 'r') as f:
+        config_dict = json.load(f)
+
+    args = argparse.Namespace(**config_dict)
 
     if args.pred_len is None:
         logging.info(f"Anomaly Mode, Set pred_len to seq_len")
         args.pred_len = args.seq_len
+
+    if 'output_dir_template' in config_dict:
+        args.output_dir = config_dict['output_dir_template'].format(
+            learning_rate=args.learning_rate,
+            downsample_rate=args.downsample_rate,
+            freeze=args.freeze,
+            train_epochs=args.train_epochs,
+            pred_len=args.pred_len
+        )
+    logging.info(f"Output Dir: {args.output_dir}")
 
     return args
 

tests/test_scripts/anomaly_config/config-1.json

@@ -0,0 +1,47 @@
+{
+    "model_id": "test_run",
+    "model_name_or_path": "gpt2-medium",
+    "seed": 2024,
+    "device": "cuda:0",
+    "checkpoints": "./checkpoints/",
+    "data_path": ["../../datasets/creditcard/creditcard.csv"],
+    "test_data_path_list": ["../../datasets/creditcard/creditcard.csv"],
+    "prompt_data_path": "../../prompt_bank/stat-prompt/prompt_data_normalize_split",
+    "data_processing": "standard_scaler",
+    "learning_rate": 1e-4,
+    "batch_size": 100,
+    "num_workers": 10,
+    "train_epochs": 4, 
+    "train_ratio": 0.7,
+    "val_ratio": 0.1,
+    "do_anomaly": true,
+    "seq_len": 133,
+    "pred_len": 133,
+    "prompt_len": 133,
+    "lora": false,
+    "lora_dim": 128,
+    "gpt_layers": 3,
+    "d_model": 1024,
+    "n_heads": 16,
+    "d_ff": 512,
+    "dropout": 0.2,
+    "enc_in": 1,
+    "c_out": 862,
+    "patch_size": 16,
+    "pretrain": 1,
+    "local_pretrain": "None",
+    "freeze": 0,
+    "model": "LTSM",
+    "stride": 8,
+    "tmax": 10,
+    "eval": 0,
+    "itr": 1,
+    "output_dir_template": "output/ltsm_lr{learning_rate}_loraFalse_down{downsample_rate}_freeze{freeze}_e{train_epochs}_pred{pred_len}_creditcard_113/",
+    "downsample_rate": 20,
+    "llm_layers": 32,
+    "decay_fac": 0.75,
+    "lradj": "type1",
+    "patience": 3,
+    "gradient_accumulation_steps": 64
+  }
+

tests/test_scripts/anomaly_config/config.json

@@ -0,0 +1,47 @@
+{
+    "model_id": "test_run",
+    "model_name_or_path": "gpt2-medium",
+    "seed": 2024,
+    "device": "cuda:0",
+    "checkpoints": "./checkpoints/",
+    "data_path": ["../../datasets/water_quality/water_quality.csv"],
+    "test_data_path_list": ["../../datasets/water_quality/water_quality.csv"],
+    "prompt_data_path": "../../prompt_bank/stat-prompt/prompt_data_normalize_split",
+    "data_processing": "standard_scaler",
+    "learning_rate": 2e-5,
+    "batch_size": 100,
+    "num_workers": 10,
+    "train_epochs": 4, 
+    "train_ratio": 0.7,
+    "val_ratio": 0.1,
+    "do_anomaly": true,
+    "seq_len": 133,
+    "pred_len": 133,
+    "prompt_len": 133,
+    "lora": false,
+    "lora_dim": 128,
+    "gpt_layers": 3,
+    "d_model": 1024,
+    "n_heads": 16,
+    "d_ff": 512,
+    "dropout": 0.2,
+    "enc_in": 1,
+    "c_out": 862,
+    "patch_size": 16,
+    "pretrain": 1,
+    "local_pretrain": "None",
+    "freeze": 0,
+    "model": "LTSM",
+    "stride": 8,
+    "tmax": 10,
+    "eval": 0,
+    "itr": 1,
+    "output_dir_template": "output/ltsm_lr{learning_rate}_loraFalse_down{downsample_rate}_freeze{freeze}_e{train_epochs}_pred{pred_len}_113",
+    "downsample_rate": 20,
+    "llm_layers": 32,
+    "decay_fac": 0.75,
+    "lradj": "type1",
+    "patience": 3,
+    "gradient_accumulation_steps": 64
+  }
+

tests/test_scripts/prompt_generation_norm.sh

@@ -1,5 +1,5 @@
 data_name="
-        multi-Synthetic"
+        web_attack"
 save_format="pth.tar"
 
 

tests/test_scripts/train_anomaly_main_ltsm.sh

@@ -1,33 +1,41 @@
-TRAIN="../../datasets/multi-Synthetic/0.csv"
+CONFIG_PATH="./anomaly_config/config-1.json"
 
+CUDA_VISIBLE_DEVICES=6,7 python3 anomaly_main_ltsm.py \
+    --config_path ${CONFIG_PATH}
+# #TRAIN="../../datasets/creditcard/creditcard.csv"
+# TRAIN="../../datasets/water_quality/water_quality.csv"
+# #TRAIN="../../datasets/multi-Synthetic/0.csv"
 
-TEST="../../datasets/multi-Synthetic/0.csv"
 
-PROMPT="../../prompt_bank/stat-prompt/prompt_data_normalize_split"
+# #TEST="../../datasets/creditcard/creditcard.csv"
+# TEST="../../datasets/water_quality/water_quality.csv"
+# #TEST="../../datasets/multi-Synthetic/0.csv"
 
-epoch=10
-downsample_rate=20
-freeze=0
-lr=1e-3
+# PROMPT="../../prompt_bank/stat-prompt/prompt_data_normalize_split"
 
+# epoch=4
+# downsample_rate=20
+# freeze=0
+# lr=1e-7
 
-for seq_len in 113
-do
-    OUTPUT_PATH="output/ltsm_lr${lr}_loraFalse_down${downsample_rate}_freeze${freeze}_e${epoch}_pred${pred_len}/"
-    echo "Current OUTPUT_PATH: ${OUTPUT_PATH}"
-    CUDA_VISIBLE_DEVICES=5,6,7 python3 anomaly_main_ltsm.py \
-    --model LTSM \
-    --model_name_or_path gpt2-medium \
-    --train_epochs ${epoch} \
-    --batch_size 100 \
-    --seq_len ${seq_len} \
-    --gradient_accumulation_steps 64 \
-    --data_path ${TRAIN} \
-    --test_data_path_list ${TEST} \
-    --prompt_data_path ${PROMPT} \
-    --freeze ${freeze} \
-    --learning_rate ${lr} \
-    --downsample_rate ${downsample_rate} \
-    --output_dir ${OUTPUT_PATH}\
-    --eval 0
-done
+
+# for seq_len in 133
+# do
+#     OUTPUT_PATH="output/ltsm_lr${lr}_loraFalse_down${downsample_rate}_freeze${freeze}_e${epoch}_pred${pred_len}_water_quality/"
+#     echo "Current OUTPUT_PATH: ${OUTPUT_PATH}"
+#     CUDA_VISIBLE_DEVICES=6,7 python3 anomaly_main_ltsm.py \
+#     --model LTSM \
+#     --model_name_or_path gpt2-medium \
+#     --train_epochs ${epoch} \
+#     --batch_size 100 \
+#     --seq_len ${seq_len} \
+#     --gradient_accumulation_steps 64 \
+#     --data_path ${TRAIN} \
+#     --test_data_path_list ${TEST} \
+#     --prompt_data_path ${PROMPT} \
+#     --freeze ${freeze} \
+#     --learning_rate ${lr} \
+#     --downsample_rate ${downsample_rate} \
+#     --output_dir ${OUTPUT_PATH}\
+#     --eval 0
+# done