sarcasm_word_embeddings.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "sarcasm_embeddings_2.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "_uo52C6OyqBC",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GPNwy_jdt8Wo",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/gdrive')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9CkNxOnxuRhW",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import pickle\n",
        "import os\n",
        "import nltk\n",
        "nltk.download('punkt')\n",
        "import numpy as _np\n",
        "import pandas as _pd\n",
        "from math import log as _log\n",
        "from gensim.test.utils import common_texts, get_tmpfile\n",
        "from gensim.models import Word2Vec, KeyedVectors\n",
        "from sklearn.model_selection import StratifiedKFold, train_test_split\n",
        "import matplotlib.pyplot as _plt\n",
        "from sklearn.naive_bayes import GaussianNB\n",
        "from sklearn.neural_network import MLPClassifier"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lM3-CM_sueu8",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def _get_training_data():\n",
        "  #pickle_in_x = open(\"gdrive/My Drive/Data/pickles/reddit_training_x.pickle\",\"rb\")\n",
        "  pickle_in_y = open(\"gdrive/My Drive/Data/pickles/reddit_training_y.pickle\",\"rb\")\n",
        "  #x_train = pickle.load(pickle_in_x)\n",
        "  y_train = pickle.load(pickle_in_y)\n",
        "  #return x_train, y_train\n",
        "  return y_train"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uB5iLw9jvO9E",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def _tokenize_sentence(samples):\n",
        "  _tokens_list = []\n",
        "  for sentence in samples:\n",
        "    _tokens_list.append(nltk.word_tokenize(sentence))\n",
        "  return _tokens_list  "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dtS9sM8IvlIC",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#takes list of tokenized sentences\n",
        "#returns a list of np.arrays. Each np.array is of size 1x300. The number of np.array equals the number of samples\n",
        "def _get_embeddings(_tokens):\n",
        "  print('loading model')\n",
        "  _filename = 'gdrive/Team Drives/NLP_Text_Summarizer/Data/word2vec/GoogleNews-vectors-negative300.bin'\n",
        "  _model = KeyedVectors.load_word2vec_format(_filename, binary=True)\n",
        "  print('model loaded')\n",
        "  \n",
        "  _embeddings = []\n",
        "  for _sentence in _tokens:\n",
        "    #for every sentence we will initialize a vector of zeros\n",
        "    _current_vector = _np.zeros(300)\n",
        "    #It's possible that some words will not have a vector representation. If so we cannot include them in the average\n",
        "    _excluded_words = 0\n",
        "    for _word in _sentence:\n",
        "      try:\n",
        "        #Add the current word to the vector representing the sentence\n",
        "        _current_vector = _np.add(_current_vector, _model[_word])\n",
        "      except KeyError:\n",
        "        #if the word is not in the model, skip to the next word\n",
        "        _excluded_words += 1\n",
        "    #After each word has been vectorized and added together -> take the average\n",
        "    #_current_vector = _np.divide(_current_vector, max(len(_sentence) - _excluded_words, 1))---->we removed this to test without average\n",
        "    #append the embedding for this sentence to the list\n",
        "    _embeddings.append(_current_vector)\n",
        "  #after all sentences have been vectorized, return the embeddings list\n",
        "  return _embeddings"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0MLq3QXpxTbu",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#x_train_whole, y_train_whole = _get_training_data()\n",
        "# x_train_whole = _tokenize_sentence(x_train_whole)\n",
        "# x_train_whole = _get_embeddings(x_train_whole)\n",
        "# _np.save('gdrive/My Drive/Data/sarcasm_embeddings/x_train_whole_embeddings_not_averaged', x_train_whole)\n",
        "y_train_whole = _get_training_data()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sm_Qxs094rf3",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def separate_fold(x_train_whole, y_train_whole, train_idx, test_idx):  \n",
        "#   x_train, x_test = x_train_whole.iloc[train_idx], x_train_whole.iloc[test_idx]\n",
        "  x_train, x_test = x_train_whole[train_idx], x_train_whole[test_idx]\n",
        "  y_train, y_test = y_train_whole.iloc[train_idx], y_train_whole.iloc[test_idx]\n",
        "  return x_train, y_train, _np.array(x_test.tolist()), _np.array(y_test.tolist())"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9efTKq4p7rEp",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def separate_classes(x_sentences, y_sentences):\n",
        "  sarc_list = []\n",
        "  not_sarc_list = []\n",
        "  for idx,label in enumerate(y_sentences):\n",
        "    if label == 1:\n",
        "      sarc_list.append(x_sentences[idx])\n",
        "    else:\n",
        "      not_sarc_list.append(x_sentences[idx])\n",
        "  return _np.array(sarc_list), _np.array(not_sarc_list)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NUeWamSY8FrU",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def _train_naive_bayes(_sarc, _not_sarc):\n",
        "  num_sarc = len(_sarc)\n",
        "  num_not_sarc = len(_not_sarc)\n",
        "  num_samples =  num_sarc + num_not_sarc\n",
        "  prior_sarc = _log(num_sarc/num_samples,10)\n",
        "  prior_not_sarc = _log(num_not_sarc/num_samples,10)\n",
        "  \n",
        "  num_features = _np.size(_sarc, 1)\n",
        "  \n",
        "  _average_sarc = _sarc.sum(axis=0)/num_sarc\n",
        "  _variance_sarc = ((_np.power(_sarc - _average_sarc,2)).sum(axis=0))/num_sarc\n",
        "  _sarc_tup = (_average_sarc, _variance_sarc, prior_sarc)\n",
        "  \n",
        "  _average_not_sarc = _not_sarc.sum(axis=0)/num_not_sarc\n",
        "  _variance_not_sarc = ((_np.power(_not_sarc - _average_not_sarc,2)).sum(axis=0))/num_not_sarc\n",
        "  _not_sarc_tup = (_average_not_sarc, _variance_not_sarc, prior_not_sarc)\n",
        "  \n",
        "  return _sarc_tup, _not_sarc_tup"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fkj7XK2l8Wlp",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def _test_naive_bayes(_sarc_tup, _not_sarc_tup, x_test, y_test):\n",
        "  y_pred_sarc = _np.log10(_np.divide(_np.exp(_np.divide(_np.power(_np.subtract(x_test,_sarc_tup[0]),2),-2*_sarc_tup[1])),_np.sqrt(2*_np.pi*_sarc_tup[1]))).sum(axis=1) + _sarc_tup[2]\n",
        "  print('y_pred_sarc - divide by zro')\n",
        "  y_pred_not_sarc = _np.log10(_np.divide(_np.exp(_np.divide(_np.power(_np.subtract(x_test,_not_sarc_tup[0]),2),-2*_not_sarc_tup[1])),_np.sqrt(2*_np.pi*_sarc_tup[1]))).sum(axis=1) + _not_sarc_tup[2]\n",
        "  \n",
        "  print(\" y pred sarc at loc 44\",y_pred_sarc[44])\n",
        "  print(\" y pred not sarc at loc 44\",y_pred_not_sarc[44])\n",
        "  \n",
        "  acc_list = []\n",
        "  #for alpha in range(-150,-75):\n",
        "  for alpha in range(0,1):\n",
        "  \n",
        "    how_many_sarc = 0\n",
        "    how_many_not_sarc = 0\n",
        "\n",
        "    y_predict = []\n",
        "    num_right = 0\n",
        "    for idx,actual_class in enumerate(y_test):\n",
        "        #if y_pred_sarc[idx] + alpha > y_pred_not_sarc[idx]:\n",
        "        #if y_pred_sarc[idx] < alpha:\n",
        "        if y_pred_sarc[idx] > y_pred_not_sarc[idx]:\n",
        "            how_many_sarc += 1\n",
        "            y_predict.append(1)\n",
        "            if actual_class == 1:\n",
        "                num_right+=1\n",
        "        else:\n",
        "            how_many_not_sarc += 1\n",
        "            y_predict.append(0)\n",
        "            if actual_class == 0:\n",
        "                num_right+=1\n",
        "    \n",
        "    print(\"if p(sarc) < \",alpha, end=\" acc = \")\n",
        "    #print(how_many_sarc,'sarcastic predictions','\\n',how_many_not_sarc,'not sarcastic predictions')\n",
        "    acc = num_right/len(y_test)\n",
        "    print(round(acc*100,3))\n",
        "    acc_list.append((alpha,acc))\n",
        "  return acc_list"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-0TrGS11CQRj",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#x_train_whole = _np.load('gdrive/My Drive/Data/sarcasm_embeddings/x_train_whole_embeddings_not_averaged.npy')\n",
        "x_train_whole = _np.load('gdrive/My Drive/Data/sarcasm_embeddings/x_train_whole_embeddings.npy')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xYFC9lchz9V4",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "fold_num=0\n",
        "skf = StratifiedKFold(n_splits=5, shuffle=True)\n",
        "\n",
        "acc_list = []\n",
        "\n",
        "for train_idx, test_idx in skf.split(x_train_whole, y_train_whole):\n",
        "  print('Fold', str(fold_num+1))\n",
        "  x_train, y_train, x_test, y_test = separate_fold(x_train_whole, y_train_whole, train_idx, test_idx)\n",
        "  \n",
        "  #split into sarcasm/not sarcasm\n",
        "  #sarc_mat, not_sarc_mat = separate_classes(x_train, y_train)\n",
        "  \n",
        "  #naive bayes that shit\n",
        "  #sarc_tup, not_sarc_tup = _train_naive_bayes(sarc_mat, not_sarc_mat)\n",
        "  \n",
        "  #test fold\n",
        "  #accs = _test_naive_bayes(sarc_tup, not_sarc_tup, x_test, y_test)\n",
        "  \n",
        "  \n",
        "  acc_list = []\n",
        "  ##IN TEST\n",
        "#   clf = GaussianNB()\n",
        "#   clf.fit(x_train, y_train)\n",
        "#   y_test_predict = clf.predict(x_test)\n",
        "  clf = MLPClassifier(solver='lbfgs', alpha=1e-5, hidden_layer_sizes=(5, 2), random_state=1)\n",
        "  clf.fit(x_train, y_train)\n",
        "  y_test_predict = clf.predict(x_test)\n",
        "  \n",
        "  \n",
        "  num_right = 0\n",
        "  for idx,prediction in enumerate(y_test_predict):\n",
        "    if prediction == y_test[idx]:\n",
        "      num_right+=1\n",
        "  acc = num_right/len(y_test)\n",
        "  acc_list.append(acc)\n",
        "  print(acc)\n",
        "  ##IN TEST\n",
        "  \n",
        "  #acc_list.append(accs)\n",
        "  \n",
        "  fold_num+=1\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2afLeIzJBJ28",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "len(acc_list)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uXRv_1W9Fdi_",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "_plt.scatter(*zip(*acc_list))\n",
        "_plt.show()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ay77BR4A1LiQ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}