experiment_temperature.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "5309a16a",
   "metadata": {},
   "source": [
    "# 导包"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "33a37f25",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Better speed can be achieved with apex installed from https://www.github.com/nvidia/apex.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-10-19 08:23:14 - INFO - <module> - 54 : Loading faiss with AVX2 support.\n",
      "2021-10-19 08:23:14 - INFO - <module> - 58 : Could not load library with AVX2 support due to:\n",
      "ModuleNotFoundError(\"No module named 'faiss.swigfaiss_avx2'\")\n",
      "2021-10-19 08:23:14 - INFO - <module> - 64 : Loading faiss.\n",
      "2021-10-19 08:23:14 - INFO - <module> - 66 : Successfully loaded faiss.\n",
      "2021-10-19 08:23:15 - INFO - from_pretrained - 125 : loading vocabulary file /data/nfs14/nfs/aisearch/asr/xhsun/CommonModel/chinese-roberta-wwm/vocab.txt\n"
     ]
    }
   ],
   "source": [
    "import nlp_basictasks\n",
    "import os,json\n",
    "import numpy as np\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import random\n",
    "from tqdm.autonotebook import tqdm, trange\n",
    "from torch.utils.data import DataLoader\n",
    "from nlp_basictasks.modules import SBERT\n",
    "from nlp_basictasks.modules.transformers import BertTokenizer,BertModel,BertConfig\n",
    "from nlp_basictasks.readers.sts import InputExample,convert_examples_to_features,getExamples,convert_sentences_to_features\n",
    "from nlp_basictasks.modules.utils import get_optimizer,get_scheduler\n",
    "from nlp_basictasks.Trainer import Trainer\n",
    "from nlp_basictasks.evaluation import stsEvaluator\n",
    "from sentence_transformers import SentenceTransformer,models\n",
    "model_path1='/data/nfs14/nfs/aisearch/asr/xhsun/bwbd_recall/distill-simcse/'\n",
    "# model_path2=\"/data/nfs14/nfs/aisearch/asr/xhsun/bwbd_recall/distiluse-base-multilingual-cased-v1/\"\n",
    "model_path3='/data/nfs14/nfs/aisearch/asr/xhsun/CommonModel/chinese-roberta-wwm/'\n",
    "# data_folder='/data/nfs14/nfs/aisearch/asr/xhsun/datasets/lcqmc/'\n",
    "# train_file=os.path.join(data_folder,'lcqmc_train.tsv')\n",
    "# dev_file=os.path.join(data_folder,'lcqmc_dev.tsv')\n",
    "#tokenizer=BertTokenizer.from_pretrained(os.path.join(model_path1,'0_Transformer'))\n",
    "tokenizer=BertTokenizer.from_pretrained(model_path3)\n",
    "max_seq_len=64\n",
    "batch_size=128"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1e4c4934",
   "metadata": {},
   "source": [
    "# 获取数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "28b82850",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_file='/data/nfs14/nfs/aisearch/asr/xhsun/bwbd_recall/STS-B/cnsd-sts-train.txt'\n",
    "dev_file='/data/nfs14/nfs/aisearch/asr/xhsun/bwbd_recall/STS-B/cnsd-sts-dev.txt'\n",
    "test_file='/data/nfs14/nfs/aisearch/asr/xhsun/bwbd_recall/STS-B/cnsd-sts-test.txt'\n",
    "def read_data(file_path):\n",
    "    sentences=[]\n",
    "    labels=[]\n",
    "    with open(file_path) as f:\n",
    "        lines=f.readlines()\n",
    "    for line in lines:\n",
    "        line_split=line.strip().split('||')\n",
    "        sentences.append([line_split[1],line_split[2]])\n",
    "        labels.append(line_split[3])\n",
    "    return sentences,labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "e668211b",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_sentences,train_labels=read_data(train_file)\n",
    "dev_sentences,dev_labels=read_data(dev_file)\n",
    "test_sentences,test_labels=read_data(test_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "559918ca",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[['一架飞机要起飞了。', '一架飞机正在起飞。'], ['一个男人在吹一支大笛子。', '一个人在吹长笛。']] ['5', '3']\n",
      "[['一个戴着安全帽的男人在跳舞。', '一个戴着安全帽的男人在跳舞。'], ['一个小孩在骑马。', '孩子在骑马。']] ['5', '4']\n",
      "[['一个女孩在给她的头发做发型。', '一个女孩在梳头。'], ['一群男人在海滩上踢足球。', '一群男孩在海滩上踢足球。']] ['2', '3']\n"
     ]
    }
   ],
   "source": [
    "print(train_sentences[:2],train_labels[:2])\n",
    "print(dev_sentences[:2],dev_labels[:2])\n",
    "print(test_sentences[:2],test_labels[:2])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e8624b54",
   "metadata": {},
   "source": [
    "## create unsupervised train_dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "20abe0c1",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5231\n",
      "['一架飞机要起飞了。', '一个男人在吹一支大笛子。', '一个人正把切碎的奶酪撒在比萨饼上。']\n",
      "<InputExample> label: 1, text pairs : 一架飞机要起飞了。; 一架飞机要起飞了。\n"
     ]
    }
   ],
   "source": [
    "train_sentences=[sentence[0] for sentence in train_sentences]#只取一般数据作为训练集\n",
    "print(len(train_sentences))\n",
    "print(train_sentences[:3])\n",
    "train_examples=[InputExample(text_list=[sentence,sentence],label=1) for sentence in train_sentences]\n",
    "train_dataloader=DataLoader(train_examples,shuffle=True,batch_size=batch_size)\n",
    "def smart_batching_collate(batch):\n",
    "    features_of_a,features_of_b,labels=convert_examples_to_features(examples=batch,tokenizer=tokenizer,max_seq_len=max_seq_len)\n",
    "    return features_of_a,features_of_b,labels\n",
    "train_dataloader.collate_fn=smart_batching_collate\n",
    "print(train_examples[0])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f36559fe",
   "metadata": {},
   "source": [
    "# SimCSE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "d71d4c9a",
   "metadata": {},
   "outputs": [],
   "source": [
    "class SimCSE(nn.Module):\n",
    "    def __init__(self,\n",
    "                 bert_model_path,\n",
    "                 is_sbert_model=True,\n",
    "                temperature=0.05,\n",
    "                is_distilbert=False,\n",
    "                device='cpu'):\n",
    "        super(SimCSE,self).__init__()\n",
    "        if is_sbert_model:\n",
    "            self.encoder=SentenceTransformer(model_name_or_path=bert_model_path,device=device)\n",
    "        else:\n",
    "            word_embedding_model = models.Transformer(bert_model_path, max_seq_length=max_seq_len)\n",
    "            pooling_model = models.Pooling(word_embedding_model.get_word_embedding_dimension())\n",
    "            self.encoder=SentenceTransformer(modules=[word_embedding_model, pooling_model],device=device)\n",
    "        self.temperature=temperature\n",
    "        self.is_distilbert=is_distilbert#蒸馏版本的BERT不支持token_type_ids\n",
    "    def cal_cos_sim(self,embeddings1,embeddings2):\n",
    "        embeddings1_norm=torch.nn.functional.normalize(embeddings1,p=2,dim=1)\n",
    "        embeddings2_norm=torch.nn.functional.normalize(embeddings2,p=2,dim=1)\n",
    "        return torch.mm(embeddings1_norm,embeddings2_norm.transpose(0,1))#(batch_size,batch_size)\n",
    "        \n",
    "    def forward(self,batch_inputs):\n",
    "        '''\n",
    "        为了实现兼容，所有model的batch_inputs最后一个位置必须是labels，即使为None\n",
    "        get token_embeddings,cls_token_embeddings,sentence_embeddings\n",
    "        sentence_embeddings是经过Pooling层后concat的embedding。维度=768*k，其中k取决于pooling的策略\n",
    "        一般来讲，只会取一种pooling策略，要么直接cls要么mean last or mean last2 or mean first and last layer，所以sentence_embeddings的维度也是768\n",
    "        '''\n",
    "        batch1_features,batch2_features,_=batch_inputs\n",
    "        if self.is_distilbert:\n",
    "            del batch1_features['token_type_ids']\n",
    "            del batch2_features['token_type_ids']\n",
    "        new_input_ids=[]\n",
    "        batch1_embeddings=self.encoder(batch1_features)['sentence_embedding']\n",
    "        batch2_embeddings=self.encoder(batch2_features)['sentence_embedding']\n",
    "        cos_sim=self.cal_cos_sim(batch1_embeddings,batch2_embeddings)/self.temperature#(batch_size,batch_size)\n",
    "        batch_size=cos_sim.size(0)\n",
    "        assert cos_sim.size()==(batch_size,batch_size)\n",
    "        labels=torch.arange(batch_size).to(cos_sim.device)\n",
    "        return nn.CrossEntropyLoss()(cos_sim,labels)\n",
    "    \n",
    "    def encode(self, sentences,\n",
    "               batch_size: int = 32,\n",
    "               show_progress_bar: bool = None,\n",
    "               output_value: str = 'sentence_embedding',\n",
    "               convert_to_numpy: bool = True,\n",
    "               convert_to_tensor: bool = False,\n",
    "               device: str = None,\n",
    "               normalize_embeddings: bool = False):\n",
    "        '''\n",
    "        传进来的sentences只能是single_batch\n",
    "        '''\n",
    "        return self.encoder.encode(sentences=sentences,\n",
    "                                         batch_size=batch_size,\n",
    "                                         show_progress_bar=show_progress_bar,\n",
    "                                         output_value=output_value,\n",
    "                                         convert_to_numpy=convert_to_numpy,\n",
    "                                         convert_to_tensor=convert_to_tensor,\n",
    "                                         device=device,\n",
    "                                         normalize_embeddings=normalize_embeddings)\n",
    "    \n",
    "    def save(self,output_path):\n",
    "        os.makedirs(output_path,exist_ok=True)\n",
    "        with open(os.path.join(output_path, 'model_param_config.json'), 'w') as fOut:\n",
    "            json.dump(self.get_config_dict(output_path), fOut)\n",
    "        self.encoder.save(output_path)\n",
    "        \n",
    "    def get_config_dict(self,output_path):\n",
    "        '''\n",
    "        一定要有dict，这样才能初始化Model\n",
    "        '''\n",
    "        return {'output_path':output_path,'temperature': self.temperature, 'is_distilbert': self.is_distilbert}\n",
    "    @staticmethod\n",
    "    def load(input_path):\n",
    "        with open(os.path.join(input_path, 'model_param_config.json')) as fIn:\n",
    "            config = json.load(fIn)\n",
    "        return SimCSE(**config)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "7d43f750",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-10-19 08:23:30 - INFO - __init__ - 41 : Load pretrained SentenceTransformer: /data/nfs14/nfs/aisearch/asr/xhsun/bwbd_recall/unsupervisedSTSModel/unSimCSE_STS-B/\n",
      "2021-10-19 08:23:30 - INFO - __init__ - 107 : Load SentenceTransformer from folder: /data/nfs14/nfs/aisearch/asr/xhsun/bwbd_recall/unsupervisedSTSModel/unSimCSE_STS-B/\n"
     ]
    }
   ],
   "source": [
    "device='cpu'\n",
    "#simcse=SimCSE(bert_model_path=model_path3,is_distilbert=False,device=device,is_sbert_model=False)\n",
    "simcse=SimCSE(bert_model_path=\"/data/nfs14/nfs/aisearch/asr/xhsun/bwbd_recall/unsupervisedSTSModel/unSimCSE_STS-B/\",is_distilbert=False,device=device,is_sbert_model=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "10c8b751",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-10-19 08:29:02 - INFO - __init__ - 41 : Load pretrained SentenceTransformer: /data/nfs14/nfs/aisearch/asr/xhsun/bwbd_recall/unsupervisedSTSModel/ESimCSE/\n",
      "2021-10-19 08:29:02 - INFO - __init__ - 107 : Load SentenceTransformer from folder: /data/nfs14/nfs/aisearch/asr/xhsun/bwbd_recall/unsupervisedSTSModel/ESimCSE/\n"
     ]
    }
   ],
   "source": [
    "e_simcse=SimCSE(bert_model_path=\"/data/nfs14/nfs/aisearch/asr/xhsun/bwbd_recall/unsupervisedSTSModel/ESimCSE/\",is_distilbert=False,is_sbert_model=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "461541e0",
   "metadata": {},
   "source": [
    "# 构造evaluator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "74c58c44",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['一个戴着安全帽的男人在跳舞。', '一个戴着安全帽的男人在跳舞。'] 5\n"
     ]
    }
   ],
   "source": [
    "#dev_sentences=[example.text_list for example in dev_examples]\n",
    "#dev_labels=[example.label for example in dev_examples]\n",
    "print(dev_sentences[0],dev_labels[0])\n",
    "sentences1_list=[sen[0] for sen in dev_sentences]\n",
    "sentences2_list=[sen[1] for sen in dev_sentences]\n",
    "dev_labels=[int(score) for score in dev_labels]\n",
    "evaluator=stsEvaluator(sentences1=sentences1_list,sentences2=sentences2_list,batch_size=64,write_csv=True,scores=dev_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "b885ae3f",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-10-19 08:24:26 - INFO - __call__ - 72 : EmbeddingSimilarityEvaluator: Evaluating the model on  dataset:\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f1870b0237834dbc8c298f693c01e3b5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Batches'), FloatProgress(value=0.0, max=23.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d065c0794ec54502a54ea17b9da1ec5b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Batches'), FloatProgress(value=0.0, max=23.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-10-19 08:24:39 - INFO - __call__ - 103 : Cosine-Similarity :\tPearson: 0.7585\tSpearman: 0.7632\n",
      "2021-10-19 08:24:39 - INFO - __call__ - 105 : Manhattan-Distance:\tPearson: 0.7228\tSpearman: 0.7396\n",
      "2021-10-19 08:24:39 - INFO - __call__ - 107 : Euclidean-Distance:\tPearson: 0.7195\tSpearman: 0.7360\n",
      "2021-10-19 08:24:39 - INFO - __call__ - 109 : Dot-Product-Similarity:\tPearson: 0.7377\tSpearman: 0.7477\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.7632353820991463"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "evaluator(simcse)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "087c044d",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-10-19 08:29:13 - INFO - __call__ - 72 : EmbeddingSimilarityEvaluator: Evaluating the model on  dataset:\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9941398149e34824997b48d2081adb1d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Batches'), FloatProgress(value=0.0, max=23.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a980920b883d4cbcab7b4c1d5d78c05b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Batches'), FloatProgress(value=0.0, max=23.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-10-19 08:29:26 - INFO - __call__ - 103 : Cosine-Similarity :\tPearson: 0.7900\tSpearman: 0.7929\n",
      "2021-10-19 08:29:26 - INFO - __call__ - 105 : Manhattan-Distance:\tPearson: 0.7555\tSpearman: 0.7709\n",
      "2021-10-19 08:29:26 - INFO - __call__ - 107 : Euclidean-Distance:\tPearson: 0.7549\tSpearman: 0.7702\n",
      "2021-10-19 08:29:26 - INFO - __call__ - 109 : Dot-Product-Similarity:\tPearson: 0.7507\tSpearman: 0.7531\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.7929133672005957"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "evaluator(e_simcse)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8940423c",
   "metadata": {},
   "source": [
    "# 训练模型温度的SimCSE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b4b9d0f4",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of the model checkpoint at /data/nfs14/nfs/aisearch/asr/xhsun/CommonModel/chinese-roberta-wwm/ were not used when initializing BertModel: ['cls.seq_relationship.weight', 'cls.predictions.transform.dense.bias', 'cls.predictions.transform.dense.weight', 'cls.predictions.bias', 'cls.seq_relationship.bias', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.transform.LayerNorm.bias', 'cls.predictions.decoder.weight']\n",
      "- This IS expected if you are initializing BertModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing BertModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "2021-10-17 20:00:33 - INFO - train - 56 : 一个epoch 下，每隔8个step会输出一次loss，每隔20个step会评估一次模型\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e0339f051bd7458a96892e2116ea1fbf",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value=''), FloatProgress(value=1.0, bar_style='info', layout=Layout(width='20px'), max=1.0…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-10-17 20:00:43 - INFO - __call__ - 72 : EmbeddingSimilarityEvaluator: Evaluating the model on  dataset in epoch 0 after 1 steps:\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b55ee14a7a144b338ae33946567f4b6a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Batches'), FloatProgress(value=0.0, max=23.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a75707e88aac4858b8435a0f0c8597c2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Batches'), FloatProgress(value=0.0, max=23.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-10-17 20:00:57 - INFO - __call__ - 103 : Cosine-Similarity :\tPearson: 0.6641\tSpearman: 0.6808\n",
      "2021-10-17 20:00:57 - INFO - __call__ - 105 : Manhattan-Distance:\tPearson: 0.6740\tSpearman: 0.6882\n",
      "2021-10-17 20:00:57 - INFO - __call__ - 107 : Euclidean-Distance:\tPearson: 0.6680\tSpearman: 0.6824\n",
      "2021-10-17 20:00:57 - INFO - __call__ - 109 : Dot-Product-Similarity:\tPearson: 0.4705\tSpearman: 0.4646\n",
      "2021-10-17 20:00:57 - INFO - save - 371 : Save model to /data/nfs14/nfs/aisearch/asr/xhsun/bwbd_recall/unsupervisedSTSModel/SimCSE_tauis1\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-10-17 20:00:58 - INFO - train - 98 : In epoch 0, training_step 0, the eval score is 0.6807509857943427, previous eval score is -9999999, model has been saved in /data/nfs14/nfs/aisearch/asr/xhsun/bwbd_recall/unsupervisedSTSModel/SimCSE_tauis1\n",
      "2021-10-17 20:02:00 - INFO - train - 75 : Epoch : 0, train_step : 8/205, loss_value : 5.105614900588989 \n",
      "2021-10-17 20:03:12 - INFO - train - 75 : Epoch : 0, train_step : 16/205, loss_value : 4.387591302394867 \n",
      "2021-10-17 20:03:53 - INFO - __call__ - 72 : EmbeddingSimilarityEvaluator: Evaluating the model on  dataset in epoch 0 after 21 steps:\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c6b15cd3dc744366bf0a645e1ad4ed5a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Batches'), FloatProgress(value=0.0, max=23.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1ffd2728fd7c4049a35193bfb667be02",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Batches'), FloatProgress(value=0.0, max=23.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-10-17 20:04:07 - INFO - __call__ - 103 : Cosine-Similarity :\tPearson: 0.6652\tSpearman: 0.6653\n",
      "2021-10-17 20:04:07 - INFO - __call__ - 105 : Manhattan-Distance:\tPearson: 0.6667\tSpearman: 0.6747\n",
      "2021-10-17 20:04:07 - INFO - __call__ - 107 : Euclidean-Distance:\tPearson: 0.6616\tSpearman: 0.6695\n",
      "2021-10-17 20:04:07 - INFO - __call__ - 109 : Dot-Product-Similarity:\tPearson: 0.6107\tSpearman: 0.6048\n",
      "2021-10-17 20:04:07 - INFO - train - 102 : No improvement over previous best eval score (0.665281 vs 0.680751), patience = 19\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-10-17 20:04:38 - INFO - train - 75 : Epoch : 0, train_step : 24/205, loss_value : 4.135252892971039 \n",
      "2021-10-17 20:05:53 - INFO - train - 75 : Epoch : 0, train_step : 32/205, loss_value : 4.0153725147247314 \n",
      "2021-10-17 20:07:04 - INFO - train - 75 : Epoch : 0, train_step : 40/205, loss_value : 3.9507781267166138 \n",
      "2021-10-17 20:07:09 - INFO - __call__ - 72 : EmbeddingSimilarityEvaluator: Evaluating the model on  dataset in epoch 0 after 41 steps:\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "873dce3b4e754449ba54f9419e4b26d0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Batches'), FloatProgress(value=0.0, max=23.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "65cf92ee856c47a986242c1cd8aca5eb",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Batches'), FloatProgress(value=0.0, max=23.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-10-17 20:07:23 - INFO - __call__ - 103 : Cosine-Similarity :\tPearson: 0.6622\tSpearman: 0.6624\n",
      "2021-10-17 20:07:23 - INFO - __call__ - 105 : Manhattan-Distance:\tPearson: 0.6658\tSpearman: 0.6662\n",
      "2021-10-17 20:07:23 - INFO - __call__ - 107 : Euclidean-Distance:\tPearson: 0.6652\tSpearman: 0.6654\n",
      "2021-10-17 20:07:23 - INFO - __call__ - 109 : Dot-Product-Similarity:\tPearson: 0.6401\tSpearman: 0.6303\n",
      "2021-10-17 20:07:23 - INFO - train - 102 : No improvement over previous best eval score (0.662450 vs 0.680751), patience = 18\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c0c09883d5a04bc7ae8076a4f54ee74a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value=''), FloatProgress(value=1.0, bar_style='info', layout=Layout(width='20px'), max=1.0…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-10-17 20:07:32 - INFO - __call__ - 72 : EmbeddingSimilarityEvaluator: Evaluating the model on  dataset in epoch 1 after 1 steps:\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e4e91986c03a41a5b3c80a1a6c733aeb",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Batches'), FloatProgress(value=0.0, max=23.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "68b474fe61314f2d8f3a920e18a4051f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Batches'), FloatProgress(value=0.0, max=23.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-10-17 20:07:46 - INFO - __call__ - 103 : Cosine-Similarity :\tPearson: 0.6643\tSpearman: 0.6647\n",
      "2021-10-17 20:07:46 - INFO - __call__ - 105 : Manhattan-Distance:\tPearson: 0.6678\tSpearman: 0.6678\n",
      "2021-10-17 20:07:46 - INFO - __call__ - 107 : Euclidean-Distance:\tPearson: 0.6673\tSpearman: 0.6670\n",
      "2021-10-17 20:07:46 - INFO - __call__ - 109 : Dot-Product-Similarity:\tPearson: 0.6439\tSpearman: 0.6343\n",
      "2021-10-17 20:07:46 - INFO - train - 102 : No improvement over previous best eval score (0.664672 vs 0.680751), patience = 17\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-10-17 20:08:57 - INFO - train - 75 : Epoch : 1, train_step : 16/205, loss_value : 4.440356075763702 \n",
      "2021-10-17 20:10:08 - INFO - train - 75 : Epoch : 1, train_step : 32/205, loss_value : 3.9337053894996643 \n",
      "2021-10-17 20:10:53 - INFO - __call__ - 72 : EmbeddingSimilarityEvaluator: Evaluating the model on  dataset in epoch 1 after 21 steps:\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "764cc7872f6c4c44805faf7da3743975",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Batches'), FloatProgress(value=0.0, max=23.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "117852c3d82e4fecbc311953b4a62464",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Batches'), FloatProgress(value=0.0, max=23.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-10-17 20:11:07 - INFO - __call__ - 103 : Cosine-Similarity :\tPearson: 0.6755\tSpearman: 0.6784\n",
      "2021-10-17 20:11:07 - INFO - __call__ - 105 : Manhattan-Distance:\tPearson: 0.6834\tSpearman: 0.6821\n",
      "2021-10-17 20:11:07 - INFO - __call__ - 107 : Euclidean-Distance:\tPearson: 0.6831\tSpearman: 0.6818\n",
      "2021-10-17 20:11:07 - INFO - __call__ - 109 : Dot-Product-Similarity:\tPearson: 0.6553\tSpearman: 0.6463\n",
      "2021-10-17 20:11:07 - INFO - train - 102 : No improvement over previous best eval score (0.678443 vs 0.680751), patience = 16\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-10-17 20:11:35 - INFO - train - 75 : Epoch : 1, train_step : 48/205, loss_value : 3.9258109629154205 \n",
      "2021-10-17 20:12:49 - INFO - train - 75 : Epoch : 1, train_step : 64/205, loss_value : 3.925343096256256 \n",
      "2021-10-17 20:13:59 - INFO - train - 75 : Epoch : 1, train_step : 80/205, loss_value : 3.906319409608841 \n",
      "2021-10-17 20:14:05 - INFO - __call__ - 72 : EmbeddingSimilarityEvaluator: Evaluating the model on  dataset in epoch 1 after 41 steps:\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "cd09e45b57564b7fa83b8fed9b608bd1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Batches'), FloatProgress(value=0.0, max=23.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e4467ccae6834defbda4374f5e87bc91",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Batches'), FloatProgress(value=0.0, max=23.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-10-17 20:14:19 - INFO - __call__ - 103 : Cosine-Similarity :\tPearson: 0.6762\tSpearman: 0.6784\n",
      "2021-10-17 20:14:19 - INFO - __call__ - 105 : Manhattan-Distance:\tPearson: 0.6839\tSpearman: 0.6822\n",
      "2021-10-17 20:14:19 - INFO - __call__ - 107 : Euclidean-Distance:\tPearson: 0.6836\tSpearman: 0.6819\n",
      "2021-10-17 20:14:19 - INFO - __call__ - 109 : Dot-Product-Similarity:\tPearson: 0.6568\tSpearman: 0.6464\n",
      "2021-10-17 20:14:19 - INFO - train - 102 : No improvement over previous best eval score (0.678400 vs 0.680751), patience = 15\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c38a437cf927484daf39183a2a46713a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value=''), FloatProgress(value=1.0, bar_style='info', layout=Layout(width='20px'), max=1.0…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-10-17 20:14:29 - INFO - __call__ - 72 : EmbeddingSimilarityEvaluator: Evaluating the model on  dataset in epoch 2 after 1 steps:\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "94126ffccba2455abbee27a2c94f6e99",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Batches'), FloatProgress(value=0.0, max=23.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c628abc4fff04f6387fdeaf9da6019fa",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Batches'), FloatProgress(value=0.0, max=23.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-10-17 20:14:43 - INFO - __call__ - 103 : Cosine-Similarity :\tPearson: 0.6762\tSpearman: 0.6784\n",
      "2021-10-17 20:14:43 - INFO - __call__ - 105 : Manhattan-Distance:\tPearson: 0.6841\tSpearman: 0.6822\n",
      "2021-10-17 20:14:43 - INFO - __call__ - 107 : Euclidean-Distance:\tPearson: 0.6838\tSpearman: 0.6820\n",
      "2021-10-17 20:14:43 - INFO - __call__ - 109 : Dot-Product-Similarity:\tPearson: 0.6569\tSpearman: 0.6466\n",
      "2021-10-17 20:14:43 - INFO - train - 102 : No improvement over previous best eval score (0.678441 vs 0.680751), patience = 14\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-10-17 20:15:50 - INFO - train - 75 : Epoch : 2, train_step : 24/205, loss_value : 4.410860866308212 \n",
      "2021-10-17 20:17:05 - INFO - train - 75 : Epoch : 2, train_step : 48/205, loss_value : 3.9177099466323853 \n",
      "2021-10-17 20:17:46 - INFO - __call__ - 72 : EmbeddingSimilarityEvaluator: Evaluating the model on  dataset in epoch 2 after 21 steps:\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3c24d3c713f543cb9a7a294d10167e31",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Batches'), FloatProgress(value=0.0, max=23.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b65f5df844344734be09c0c8c49d169f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Batches'), FloatProgress(value=0.0, max=23.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-10-17 20:18:00 - INFO - __call__ - 103 : Cosine-Similarity :\tPearson: 0.6762\tSpearman: 0.6775\n",
      "2021-10-17 20:18:00 - INFO - __call__ - 105 : Manhattan-Distance:\tPearson: 0.6837\tSpearman: 0.6818\n",
      "2021-10-17 20:18:00 - INFO - __call__ - 107 : Euclidean-Distance:\tPearson: 0.6834\tSpearman: 0.6816\n",
      "2021-10-17 20:18:00 - INFO - __call__ - 109 : Dot-Product-Similarity:\tPearson: 0.6560\tSpearman: 0.6456\n",
      "2021-10-17 20:18:00 - INFO - train - 102 : No improvement over previous best eval score (0.677485 vs 0.680751), patience = 13\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-10-17 20:18:32 - INFO - train - 75 : Epoch : 2, train_step : 72/205, loss_value : 3.916303902864456 \n",
      "2021-10-17 20:19:41 - INFO - train - 75 : Epoch : 2, train_step : 96/205, loss_value : 3.9153249859809875 \n",
      "2021-10-17 20:20:54 - INFO - train - 75 : Epoch : 2, train_step : 120/205, loss_value : 3.8989281952381134 \n",
      "2021-10-17 20:20:59 - INFO - __call__ - 72 : EmbeddingSimilarityEvaluator: Evaluating the model on  dataset in epoch 2 after 41 steps:\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d01a8e305c4c489781a2628665b42834",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Batches'), FloatProgress(value=0.0, max=23.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a4620e80033d4625add19961958fb185",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Batches'), FloatProgress(value=0.0, max=23.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-10-17 20:21:13 - INFO - __call__ - 103 : Cosine-Similarity :\tPearson: 0.6763\tSpearman: 0.6780\n",
      "2021-10-17 20:21:13 - INFO - __call__ - 105 : Manhattan-Distance:\tPearson: 0.6845\tSpearman: 0.6826\n",
      "2021-10-17 20:21:13 - INFO - __call__ - 107 : Euclidean-Distance:\tPearson: 0.6840\tSpearman: 0.6826\n",
      "2021-10-17 20:21:13 - INFO - __call__ - 109 : Dot-Product-Similarity:\tPearson: 0.6569\tSpearman: 0.6468\n",
      "2021-10-17 20:21:13 - INFO - train - 102 : No improvement over previous best eval score (0.678027 vs 0.680751), patience = 12\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4c68ce7f5d514e299530ec9431f30bf3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value=''), FloatProgress(value=1.0, bar_style='info', layout=Layout(width='20px'), max=1.0…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-10-17 20:21:23 - INFO - __call__ - 72 : EmbeddingSimilarityEvaluator: Evaluating the model on  dataset in epoch 3 after 1 steps:\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3cd1178baf474d2eb3cd2786276f53e9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Batches'), FloatProgress(value=0.0, max=23.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "eb8f4a0b72374e7f81fe1d1c06569060",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Batches'), FloatProgress(value=0.0, max=23.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-10-17 20:21:37 - INFO - __call__ - 103 : Cosine-Similarity :\tPearson: 0.6761\tSpearman: 0.6779\n",
      "2021-10-17 20:21:37 - INFO - __call__ - 105 : Manhattan-Distance:\tPearson: 0.6843\tSpearman: 0.6824\n",
      "2021-10-17 20:21:37 - INFO - __call__ - 107 : Euclidean-Distance:\tPearson: 0.6839\tSpearman: 0.6824\n",
      "2021-10-17 20:21:37 - INFO - __call__ - 109 : Dot-Product-Similarity:\tPearson: 0.6567\tSpearman: 0.6467\n",
      "2021-10-17 20:21:37 - INFO - train - 102 : No improvement over previous best eval score (0.677852 vs 0.680751), patience = 11\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-10-17 20:22:43 - INFO - train - 75 : Epoch : 3, train_step : 32/205, loss_value : 4.403318256139755 \n"
     ]
    }
   ],
   "source": [
    "model_temperature=SimCSE(bert_model_path=model_path3,is_distilbert=False,device=device,is_sbert_model=False,temperature=1)\n",
    "epochs=5\n",
    "output_path='/data/nfs14/nfs/aisearch/asr/xhsun/bwbd_recall/unsupervisedSTSModel/SimCSE_tauis1'\n",
    "tensorboard_logdir=os.path.join(output_path,'log')\n",
    "optimizer_type='AdamW'\n",
    "scheduler='WarmupLinear'\n",
    "warmup_proportion=0.1\n",
    "optimizer_params={'lr': 2e-5}\n",
    "weight_decay=0.01\n",
    "num_train_steps = int(len(train_dataloader) * epochs)\n",
    "warmup_steps = num_train_steps*warmup_proportion\n",
    "optimizer = get_optimizer(model=model_temperature,optimizer_type=optimizer_type,weight_decay=weight_decay,optimizer_params=optimizer_params)\n",
    "scheduler = get_scheduler(optimizer, scheduler=scheduler, warmup_steps=warmup_steps, t_total=num_train_steps)\n",
    "\n",
    "trainer=Trainer(epochs=epochs,output_path=output_path,tensorboard_logdir=tensorboard_logdir,early_stop_patience=20)\n",
    "trainer.train(train_dataloader=train_dataloader,\n",
    "             model=model_temperature,\n",
    "             optimizer=optimizer,\n",
    "             scheduler=scheduler,\n",
    "             evaluator=evaluator,\n",
    "             )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "ca55290c",
   "metadata": {},
   "outputs": [],
   "source": [
    "def uniform_loss(x, t=2):\n",
    "    return torch.pdist(x, p=2).pow(2).mul(-t).exp().mean().log()\n",
    "def align_loss(x, y, alpha=2):\n",
    "    return (x - y).norm(p=2, dim=1).pow(alpha).mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "c64fb93f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ebce5c9f7d4e494283034bce05003349",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Batches'), FloatProgress(value=0.0, max=46.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5261b0e37b60427ea09f1060987d05a8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Batches'), FloatProgress(value=0.0, max=46.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "sentences1_embeddings=simcse.encode(sentences1_list,convert_to_tensor=True)\n",
    "sentences2_embeddings=simcse.encode(sentences2_list,convert_to_tensor=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "9c226986",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(194.2286)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "align_loss(sentences1_embeddings,sentences2_embeddings)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "da0f7ba9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(-9.8505)"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "uniform_loss(sentences1_embeddings)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "c51bb17f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(-9.6273)"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "uniform_loss(sentences2_embeddings)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "037bedb1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "65f5e0f785e7466bb5b061536b391875",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Batches'), FloatProgress(value=0.0, max=46.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9528e9e68d2841d3889efe51b8c647e0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Batches'), FloatProgress(value=0.0, max=46.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "s1=e_simcse.encode(sentences1_list,convert_to_tensor=True)\n",
    "s2=e_simcse.encode(sentences2_list,convert_to_tensor=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "9dc279db",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(202.1972)"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "align_loss(s1,s2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "df3f37e4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(-9.8505)"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "uniform_loss(s1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "3ece7822",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(-9.6273)"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "uniform_loss(s2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "dd1c388e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([1458, 768])"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "s1.size()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "8a9ade00",
   "metadata": {},
   "outputs": [],
   "source": [
    "sentence=\"数据转换方式决定了最终学习的向量表示的不变性\"\n",
    "s1=simcse.encode(sentence,show_progress_bar=False,normalize_embeddings=True)\n",
    "s2=simcse.encode(sentence[::-1],show_progress_bar=False,normalize_embeddings=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "c09474f9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7955213659201235"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sum(s1*s2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "19bf529d",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d68ad179",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}