embedded_models.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "7747c1a1",
   "metadata": {},
   "source": [
    "# Summary source prediction: Embedded models\n",
    "\n",
    "Sébastien Meyer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "51387363",
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "import re\n",
    "import string\n",
    "\n",
    "from tqdm import tqdm\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "import nltk\n",
    "from nltk import word_tokenize\n",
    "from nltk.corpus import stopwords\n",
    "\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.model_selection import train_test_split, cross_val_score\n",
    "\n",
    "from xgboost import XGBClassifier\n",
    "\n",
    "import gensim\n",
    "from gensim.models import word2vec\n",
    "from gensim.models.callbacks import CallbackAny2Vec\n",
    "from gensim.utils import tokenize, effective_n_jobs\n",
    "from gensim.test.utils import common_texts, get_tmpfile\n",
    "\n",
    "from keras.preprocessing import sequence, text\n",
    "from keras.utils import np_utils\n",
    "from keras.models import Sequential\n",
    "from keras.layers import (\n",
    "    BatchNormalization, Bidirectional, Conv1D, Flatten, GlobalMaxPooling1D, MaxPooling1D, SpatialDropout1D\n",
    ")\n",
    "from keras.layers.core import Activation, Dense, Dropout\n",
    "from keras.layers.embeddings import Embedding\n",
    "from keras.layers.recurrent import GRU, LSTM\n",
    "from keras.callbacks import EarlyStopping\n",
    "\n",
    "from tensorflow.keras import preprocessing as kprocessing\n",
    "from tensorflow.keras import backend as K\n",
    "from tensorflow.keras import layers, models\n",
    "\n",
    "import transformers\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "\n",
    "\n",
    "from src.preprocessing.features.embeddings import text_to_tokens, text_to_vec"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "681762bd",
   "metadata": {},
   "outputs": [],
   "source": [
    "stopwords = nltk.corpus.stopwords.words(\"english\")\n",
    "punct = string.punctuation.replace(\"-\", \"\")\n",
    "\n",
    "tqdm.pandas()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2b81314c",
   "metadata": {},
   "source": [
    "## Dictionary and embeddings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5907360c",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_df = pd.read_json(\"data/train_set.json\")\n",
    "test_df = pd.read_json(\"data/test_set.json\")\n",
    "documents = pd.read_json(\"data/documents.json\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1e919935",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "print(\"Cleaning documents...\")\n",
    "\n",
    "train_df[\"document_token\"] = train_df[\"document\"].progress_apply(\n",
    "    lambda x: text_to_tokens(x, stopwords, punct, remove_stopwords=True)\n",
    ")\n",
    "\n",
    "documents[\"document_token\"] = documents[\"document\"].progress_apply(\n",
    "    lambda x: text_to_tokens(x, stopwords, punct, remove_stopwords=True)\n",
    ")\n",
    "\n",
    "print(\"All documents clean.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "473f90cf",
   "metadata": {},
   "outputs": [],
   "source": [
    "all_docs = train_df[\"document_token\"].to_list() + documents[\"document_token\"].to_list()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "032cb866",
   "metadata": {},
   "source": [
    "## Choose your embeddings..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "82c20c78",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Pretrained GloVe embeddings\n",
    "embeddings_index = {}\n",
    "\n",
    "f = open(\"data/embed/glove_300.txt\", encoding=\"utf8\")\n",
    "\n",
    "for line in tqdm(f):\n",
    "    \n",
    "    values = line.strip().split(\" \")\n",
    "    try:\n",
    "        word = values[0]\n",
    "        coefs = np.asarray(values[1:], dtype=\"float32\")\n",
    "    except ValueError:\n",
    "        print(values[0])\n",
    "    embeddings_index[word] = coefs\n",
    "    \n",
    "f.close()\n",
    "\n",
    "print(f\"Found {len(embeddings_index)} word vectors.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d64c9110",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Pretrained Google Word2Vec embeddings\n",
    "# embeddings_index = gensim.models.KeyedVectors.load_word2vec_format(\n",
    "#     \"data/embed/google_300.gz\", binary=True\n",
    "# )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7be833e6",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Train your own Word2Vec embeddings\n",
    "class callback(CallbackAny2Vec):\n",
    "    \"\"\"Callback to print loss after each epoch.\"\"\"\n",
    "    def __init__(self):\n",
    "        \n",
    "        self.epoch = 0\n",
    "        self.loss_to_be_subed = 0\n",
    "\n",
    "    def on_epoch_end(self, model):\n",
    "        \n",
    "        total_loss = model.get_latest_training_loss()\n",
    "        current_loss = total_loss - self.loss_to_be_subed\n",
    "        self.loss_to_be_subed = total_loss\n",
    "        \n",
    "        print(f\"Loss after epoch {self.epoch}: {current_loss}\")\n",
    "        \n",
    "        self.epoch += 1\n",
    "\n",
    "# w2v = word2vec.Word2Vec(\n",
    "#     all_docs, vector_size=300, window=20, min_count=5, workers=effective_n_jobs(-1), epochs=25,\n",
    "#     compute_loss=True, callbacks=[callback()]\n",
    "# )\n",
    "\n",
    "# embeddings_index = w2v.wv"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "218d8f77",
   "metadata": {},
   "source": [
    "## Train test split and transf."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "55ba58d3",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train, df_val = train_test_split(train_df, test_size=0.2, random_state=42)\n",
    "\n",
    "x_train = df_train[\"summary\"].to_numpy()\n",
    "y_train = df_train[\"label\"].to_numpy().flatten()\n",
    "x_val = df_val[\"summary\"].to_numpy()\n",
    "y_val = df_val[\"label\"].to_numpy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "35ee913c",
   "metadata": {},
   "outputs": [],
   "source": [
    "# create sentence vectors using the above function for training and validation set\n",
    "x_train_glove = [text_to_vec(embeddings_index, x, stopwords, punct, remove_stopwords=True) for x in tqdm(x_train)]\n",
    "x_val_glove = [text_to_vec(embeddings_index, x, stopwords, punct, remove_stopwords=True) for x in tqdm(x_val)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cd9252ef",
   "metadata": {},
   "outputs": [],
   "source": [
    "x_train_glove = np.array(x_train_glove)\n",
    "x_val_glove = np.array(x_val_glove)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5ee3e1ab",
   "metadata": {},
   "source": [
    "## Base models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7eda9923",
   "metadata": {},
   "outputs": [],
   "source": [
    "logreg = LogisticRegression()\n",
    "\n",
    "logreg.fit(x_train_glove, y_train)\n",
    "\n",
    "y_pred = logreg.predict(x_val_glove)\n",
    "print(\"Accuracy:\", accuracy_score(y_val, y_pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5a5e7b50",
   "metadata": {},
   "outputs": [],
   "source": [
    "xgb = XGBClassifier(\n",
    "    max_depth=7, n_estimators=200, colsample_bytree=0.8, subsample=0.8, nthread=10, \n",
    "    learning_rate=0.1, use_label_encoder=False, eval_metric=\"logloss\", random_state=42\n",
    ")\n",
    "\n",
    "xgb.fit(x_train_glove, y_train)\n",
    "\n",
    "y_pred = xgb.predict(x_val_glove)\n",
    "print(\"Accuracy:\", accuracy_score(y_val, y_pred))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9b654cb6",
   "metadata": {},
   "source": [
    "## MLP model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "60ca7362",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Scale the data before any neural net\n",
    "sc = StandardScaler()\n",
    "\n",
    "x_train_glove_scl = sc.fit_transform(x_train_glove)\n",
    "x_val_glove_scl = sc.transform(x_val_glove)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6c988e88",
   "metadata": {},
   "outputs": [],
   "source": [
    "# We need to binarize the labels for the neural net\n",
    "y_train_hot = np_utils.to_categorical(y_train)\n",
    "y_val_hot = np_utils.to_categorical(y_val)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ca43d458",
   "metadata": {},
   "outputs": [],
   "source": [
    "# create a simple 3 layer sequential neural net\n",
    "model = Sequential()\n",
    "\n",
    "model.add(Dense(300, input_dim=300, activation=\"relu\"))\n",
    "model.add(Dropout(0.2))\n",
    "model.add(BatchNormalization())\n",
    "\n",
    "model.add(Dense(300, activation=\"relu\"))\n",
    "model.add(Dropout(0.3))\n",
    "model.add(BatchNormalization())\n",
    "\n",
    "model.add(Dense(2))\n",
    "model.add(Activation(\"softmax\"))\n",
    "\n",
    "# compile the model\n",
    "model.compile(loss=\"categorical_crossentropy\", optimizer=\"adam\", metrics=[\"accuracy\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bd22caf8",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "model.fit(\n",
    "    x_train_glove_scl, y=y_train_hot, batch_size=64, \n",
    "    epochs=50, verbose=1, \n",
    "    validation_data=(x_val_glove_scl, y_val_hot)\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "469036fd",
   "metadata": {},
   "source": [
    "## LSTM model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "35e757c5",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Using keras tokenizer here\n",
    "token = text.Tokenizer(num_words=None)\n",
    "max_len = 70\n",
    "\n",
    "token.fit_on_texts(list(x_train) + list(x_val))\n",
    "x_train_seq = token.texts_to_sequences(x_train)\n",
    "x_val_seq = token.texts_to_sequences(x_val)\n",
    "\n",
    "# Zero pad the sequences\n",
    "x_train_pad = sequence.pad_sequences(x_train_seq, maxlen=max_len)\n",
    "x_val_pad = sequence.pad_sequences(x_val_seq, maxlen=max_len)\n",
    "\n",
    "word_index = token.word_index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4c261739",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create an embedding matrix for the words we have in the dataset\n",
    "embedding_matrix = np.zeros((len(word_index) + 1, 300))\n",
    "for word, i in tqdm(word_index.items()):\n",
    "\n",
    "    if word in embeddings_index:\n",
    "        embedding_matrix[i] = embeddings_index[word]\n",
    "    else:\n",
    "        np.random.normal(size=300)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f16448b1",
   "metadata": {},
   "outputs": [],
   "source": [
    "# A simple LSTM and two dense layers\n",
    "model = Sequential()\n",
    "model.add(Embedding(len(word_index) + 1,\n",
    "                     300,\n",
    "                     weights=[embedding_matrix],\n",
    "                     input_length=max_len,\n",
    "                     trainable=False))\n",
    "model.add(SpatialDropout1D(0.3))\n",
    "model.add(LSTM(100, dropout=0.3, recurrent_dropout=0.3))\n",
    "\n",
    "model.add(Dense(1024, activation=\"relu\"))\n",
    "model.add(Dropout(0.8))\n",
    "\n",
    "model.add(Dense(1024, activation=\"relu\"))\n",
    "model.add(Dropout(0.8))\n",
    "\n",
    "model.add(Dense(2))\n",
    "model.add(Activation(\"softmax\"))\n",
    "model.compile(loss=\"categorical_crossentropy\", optimizer=\"adam\", metrics=[\"accuracy\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "518981f5",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "model.fit(\n",
    "    x_train_pad, y=y_train_hot, batch_size=512,\n",
    "    epochs=30, verbose=1, \n",
    "    validation_data=(x_val_pad, y_val_hot)\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "687bc358",
   "metadata": {},
   "source": [
    "## Bidirectional LSTMs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4d22f2da",
   "metadata": {},
   "outputs": [],
   "source": [
    "# A simple bidirectional LSTM and two dense layers\n",
    "model = Sequential()\n",
    "model.add(Embedding(len(word_index) + 1,\n",
    "                     300,\n",
    "                     weights=[embedding_matrix],\n",
    "                     input_length=max_len,\n",
    "                     trainable=False))\n",
    "model.add(SpatialDropout1D(0.3))\n",
    "model.add(Bidirectional(LSTM(100, dropout=0.3, recurrent_dropout=0.3)))\n",
    "\n",
    "model.add(Dense(100, activation=\"relu\"))\n",
    "model.add(Dropout(0.8))\n",
    "\n",
    "model.add(Dense(100, activation=\"relu\"))\n",
    "model.add(Dropout(0.8))\n",
    "\n",
    "model.add(Dense(1))\n",
    "model.add(Activation(\"sigmoid\"))\n",
    "model.compile(loss=\"binary_crossentropy\", optimizer=\"adam\", metrics=[\"accuracy\"])\n",
    "\n",
    "# Fit the model with early stopping callback\n",
    "earlystop = EarlyStopping(monitor=\"val_loss\", min_delta=0, patience=3, verbose=0, mode=\"auto\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c4ab57d1",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "model.fit(\n",
    "    x_train_pad, y=y_train, batch_size=512, \n",
    "    epochs=100, verbose=1, \n",
    "    validation_data=(x_val_pad, y_val), callbacks=[earlystop]\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6097bfa1",
   "metadata": {},
   "outputs": [],
   "source": [
    "# GRU and two dense layers\n",
    "model = Sequential()\n",
    "model.add(Embedding(len(word_index) + 1,\n",
    "                     300,\n",
    "                     weights=[embedding_matrix],\n",
    "                     input_length=max_len,\n",
    "                     trainable=False))\n",
    "model.add(SpatialDropout1D(0.3))\n",
    "model.add(GRU(300, dropout=0.3, recurrent_dropout=0.3, return_sequences=True))\n",
    "model.add(GRU(300, dropout=0.3, recurrent_dropout=0.3))\n",
    "\n",
    "model.add(Dense(512, activation=\"relu\"))\n",
    "model.add(Dropout(0.8))\n",
    "\n",
    "model.add(Dense(512, activation=\"relu\"))\n",
    "model.add(Dropout(0.8))\n",
    "\n",
    "model.add(Dense(1))\n",
    "model.add(Activation(\"sigmoid\"))\n",
    "model.compile(loss=\"binary_crossentropy\", optimizer=\"adam\", metrics=[\"accuracy\"])\n",
    "\n",
    "# Fit the model with early stopping callback\n",
    "earlystop = EarlyStopping(monitor=\"val_loss\", min_delta=0, patience=3, verbose=0, mode=\"auto\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e2de99c4",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "model.fit(\n",
    "    x_train_pad, y=y_train, \n",
    "    batch_size=512, epochs=100, \n",
    "    verbose=1, validation_data=(x_val_pad, y_val), callbacks=[earlystop]\n",
    ")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "itmnlp",
   "language": "python",
   "name": "itmnlp"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}