generate_incremental_bert_embeddings.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from IPython.core.debugger import set_trace\n",
    "from scipy.io import loadmat\n",
    "import pickle\n",
    "import os\n",
    "import sys\n",
    "import nltk\n",
    "from scipy import stats\n",
    "from sklearn.model_selection import KFold\n",
    "from scipy.stats import spearmanr, pearsonr\n",
    "import spacy\n",
    "import matplotlib.pyplot as plt\n",
    "from tqdm import tqdm\n",
    "from spacy import displacy\n",
    "from collections import defaultdict\n",
    "import torch\n",
    "from transformers import *\n",
    "from sklearn.decomposition import PCA\n",
    "\n",
    "np.random.seed(97)\n",
    "torch.manual_seed(97)\n",
    "\n",
    "save_dir = \"features\"\n",
    "if not os.path.exists(save_dir):\n",
    "    os.mkdir(save_dir)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "hp_text = open(\"chapter9.txt\",\"r\").readlines()\n",
    "hp_text = [i.strip() for i in hp_text]\n",
    "\n",
    "sent_text = open(\"sentences.txt\", \"r\").readlines()\n",
    "all_sentences = [\"\"]\n",
    "for i in range(len(sent_text)):\n",
    "    sent_text[i] = sent_text[i].strip()\n",
    "    if len(sent_text[i]) == 0 or sent_text[i] == \"+\":\n",
    "        all_sentences.append(\"\")\n",
    "    elif sent_text[i]!=\"+\":\n",
    "        all_sentences[-1] += sent_text[i] + \" \"\n",
    "\n",
    "all_sentences_clean = [i.strip() for i in all_sentences if len(i.strip()) > 0]\n",
    "all_sentences = all_sentences_clean"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_contexts = []\n",
    "max_len = 0\n",
    "\n",
    "for sent in all_sentences:\n",
    "    words = sent.strip().split(\" \")\n",
    "    max_len = max(max_len, len(words))\n",
    "    for i in range(len(words)):\n",
    "        if (len(all_contexts) != 0  and hp_text[len(all_contexts)] == \"+\"):\n",
    "            all_contexts.append(\"\")\n",
    "        if (len(all_contexts) != 0  and hp_text[len(all_contexts)] == \"+\"):\n",
    "            all_contexts.append(\"\")\n",
    "        all_contexts.append(\" \".join(words[:i+1]))\n",
    "            \n",
    "all_contexts.append([[\"\"]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(len(all_contexts))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer = BertTokenizer.from_pretrained('bert-large-cased')\n",
    "model = BertModel.from_pretrained('bert-large-cased', output_hidden_states=True)\n",
    "model.eval()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_embeddings = []\n",
    "selected_layer = 11\n",
    "\n",
    "for i in range(len(all_contexts)):\n",
    "    print(hp_text[i])\n",
    "    print(all_contexts[i])\n",
    "    if hp_text[i] == \"+\":\n",
    "        all_embeddings.append(np.zeros(1024))\n",
    "    else:\n",
    "        input_ids = torch.tensor([tokenizer.encode(all_contexts[i], add_special_tokens=True)])  # Add special tokens takes care of adding [CLS], [SEP], <s>... tokens in the right way for each model.\n",
    "        all_outputs = model(input_ids)\n",
    "        all_hidden_states = all_outputs[2]\n",
    "        selected_hidden_states = all_hidden_states[selected_layer]\n",
    "        selected_hidden_states = selected_hidden_states.reshape((selected_hidden_states.shape[1], selected_hidden_states.shape[2]))\n",
    "        mean_hidden_states = selected_hidden_states.mean(axis = 0).detach().numpy()\n",
    "        all_embeddings.append(mean_hidden_states)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.array(all_embeddings).shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_embeddings = np.load(os.path.join(save_dir,\"incremental_bert_embeddings_layer12.npy\"))\n",
    "pca = PCA(n_components=15)\n",
    "np.random.seed(97)\n",
    "\n",
    "reduced = pca.fit_transform(np.array(all_embeddings))\n",
    "np.save(os.path.join(save_dir,\"incremental_bert_embeddings_layer12_PCA_dims_15\"), reduced)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "reduced.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}