Avg W2V with 300-D.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\cross_validation.py:41: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. Also note that the interface of the new CV iterators are different from that of this module. This module will be removed in 0.20.\n",
      "  \"This module will be removed in 0.20.\", DeprecationWarning)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\grid_search.py:42: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. This module will be removed in 0.20.\n",
      "  DeprecationWarning)\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.grid_search import GridSearchCV\n",
    "from sklearn.linear_model import LogisticRegression"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Reading Avg w2v vectors(300-D)l"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(122109, 301)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>...</th>\n",
       "      <th>291</th>\n",
       "      <th>292</th>\n",
       "      <th>293</th>\n",
       "      <th>294</th>\n",
       "      <th>295</th>\n",
       "      <th>296</th>\n",
       "      <th>297</th>\n",
       "      <th>298</th>\n",
       "      <th>299</th>\n",
       "      <th>300</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.000259</td>\n",
       "      <td>-0.000116</td>\n",
       "      <td>0.000290</td>\n",
       "      <td>-0.000109</td>\n",
       "      <td>0.000132</td>\n",
       "      <td>0.000120</td>\n",
       "      <td>0.000027</td>\n",
       "      <td>0.000366</td>\n",
       "      <td>0.000118</td>\n",
       "      <td>0.000003</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000197</td>\n",
       "      <td>-0.000364</td>\n",
       "      <td>0.000317</td>\n",
       "      <td>0.000149</td>\n",
       "      <td>0.000193</td>\n",
       "      <td>0.000358</td>\n",
       "      <td>0.000094</td>\n",
       "      <td>0.000235</td>\n",
       "      <td>-0.000202</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.000189</td>\n",
       "      <td>0.000059</td>\n",
       "      <td>0.000166</td>\n",
       "      <td>0.000056</td>\n",
       "      <td>-0.000087</td>\n",
       "      <td>0.000035</td>\n",
       "      <td>-0.000047</td>\n",
       "      <td>0.000082</td>\n",
       "      <td>-0.000092</td>\n",
       "      <td>-0.000212</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000141</td>\n",
       "      <td>-0.000036</td>\n",
       "      <td>0.000123</td>\n",
       "      <td>0.000098</td>\n",
       "      <td>0.000590</td>\n",
       "      <td>-0.000339</td>\n",
       "      <td>0.000291</td>\n",
       "      <td>0.000091</td>\n",
       "      <td>-0.000018</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>-0.000041</td>\n",
       "      <td>-0.000163</td>\n",
       "      <td>0.000031</td>\n",
       "      <td>-0.000424</td>\n",
       "      <td>-0.000413</td>\n",
       "      <td>0.000072</td>\n",
       "      <td>0.000474</td>\n",
       "      <td>-0.000154</td>\n",
       "      <td>-0.000171</td>\n",
       "      <td>0.000377</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.000183</td>\n",
       "      <td>0.000082</td>\n",
       "      <td>-0.000136</td>\n",
       "      <td>0.000131</td>\n",
       "      <td>-0.000009</td>\n",
       "      <td>-0.000278</td>\n",
       "      <td>0.000112</td>\n",
       "      <td>-0.000064</td>\n",
       "      <td>0.000302</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.000260</td>\n",
       "      <td>-0.000109</td>\n",
       "      <td>0.000150</td>\n",
       "      <td>-0.000249</td>\n",
       "      <td>-0.000043</td>\n",
       "      <td>-0.000044</td>\n",
       "      <td>0.000222</td>\n",
       "      <td>-0.000008</td>\n",
       "      <td>-0.000014</td>\n",
       "      <td>0.000081</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.000005</td>\n",
       "      <td>0.000003</td>\n",
       "      <td>0.000053</td>\n",
       "      <td>0.000074</td>\n",
       "      <td>-0.000224</td>\n",
       "      <td>0.000161</td>\n",
       "      <td>0.000051</td>\n",
       "      <td>-0.000008</td>\n",
       "      <td>0.000007</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.000038</td>\n",
       "      <td>0.000340</td>\n",
       "      <td>-0.000279</td>\n",
       "      <td>0.000020</td>\n",
       "      <td>-0.000101</td>\n",
       "      <td>-0.000203</td>\n",
       "      <td>-0.000285</td>\n",
       "      <td>0.000061</td>\n",
       "      <td>0.000015</td>\n",
       "      <td>0.000115</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.000407</td>\n",
       "      <td>-0.000198</td>\n",
       "      <td>-0.000115</td>\n",
       "      <td>-0.000093</td>\n",
       "      <td>-0.000195</td>\n",
       "      <td>-0.000170</td>\n",
       "      <td>-0.000173</td>\n",
       "      <td>-0.000333</td>\n",
       "      <td>-0.000010</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 301 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        0         1         2         3         4         5         6    \\\n",
       "0  0.000259 -0.000116  0.000290 -0.000109  0.000132  0.000120  0.000027   \n",
       "1  0.000189  0.000059  0.000166  0.000056 -0.000087  0.000035 -0.000047   \n",
       "2 -0.000041 -0.000163  0.000031 -0.000424 -0.000413  0.000072  0.000474   \n",
       "3  0.000260 -0.000109  0.000150 -0.000249 -0.000043 -0.000044  0.000222   \n",
       "4  0.000038  0.000340 -0.000279  0.000020 -0.000101 -0.000203 -0.000285   \n",
       "\n",
       "        7         8         9   ...        291       292       293       294  \\\n",
       "0  0.000366  0.000118  0.000003 ...   0.000197 -0.000364  0.000317  0.000149   \n",
       "1  0.000082 -0.000092 -0.000212 ...   0.000141 -0.000036  0.000123  0.000098   \n",
       "2 -0.000154 -0.000171  0.000377 ...  -0.000183  0.000082 -0.000136  0.000131   \n",
       "3 -0.000008 -0.000014  0.000081 ...  -0.000005  0.000003  0.000053  0.000074   \n",
       "4  0.000061  0.000015  0.000115 ...  -0.000407 -0.000198 -0.000115 -0.000093   \n",
       "\n",
       "        295       296       297       298       299  300  \n",
       "0  0.000193  0.000358  0.000094  0.000235 -0.000202    1  \n",
       "1  0.000590 -0.000339  0.000291  0.000091 -0.000018    1  \n",
       "2 -0.000009 -0.000278  0.000112 -0.000064  0.000302    0  \n",
       "3 -0.000224  0.000161  0.000051 -0.000008  0.000007    1  \n",
       "4 -0.000195 -0.000170 -0.000173 -0.000333 -0.000010    1  \n",
       "\n",
       "[5 rows x 301 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "avg_w2v =pd.read_pickle(\"avg_w2v_vec_300\")\n",
    "print(avg_w2v.shape)\n",
    "avg_w2v.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(122109, 300)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(122109,)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x =avg_w2v.iloc[:,:300]\n",
    "y =avg_w2v.iloc[:,300]\n",
    "print(x.shape)\n",
    "y.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Column Standardization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import StandardScaler"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "a =StandardScaler()\n",
    "x =a.fit_transform(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "x_train,x_test,y_train,y_test =train_test_split(x,y,test_size =0.3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(85476, 300)\n",
      "(36633, 300)\n",
      "(85476,)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(36633,)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(x_train.shape)\n",
    "print(x_test.shape)\n",
    "print(y_train.shape)\n",
    "y_test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "tuned_params = [{'C': [0.0001, 0.001, 0.01, 0.1, 1, 10, 100, 1000, 10000]}]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=None, error_score='raise',\n",
       "       estimator=LogisticRegression(C=1.0, class_weight=None, dual=False, fit_intercept=True,\n",
       "          intercept_scaling=1, max_iter=100, multi_class='ovr', n_jobs=1,\n",
       "          penalty='l2', random_state=None, solver='liblinear', tol=0.0001,\n",
       "          verbose=0, warm_start=False),\n",
       "       fit_params={}, iid=True, n_jobs=1,\n",
       "       param_grid=[{'C': [0.0001, 0.001, 0.01, 0.1, 1, 10, 100, 1000, 10000]}],\n",
       "       pre_dispatch='2*n_jobs', refit=True, scoring='accuracy', verbose=0)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model = GridSearchCV(LogisticRegression(), tuned_params, scoring = 'accuracy')\n",
    "model.fit(x_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import confusion_matrix\n",
    "from sklearn.metrics import accuracy_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LogisticRegression(C=0.01, class_weight=None, dual=False, fit_intercept=True,\n",
      "          intercept_scaling=1, max_iter=100, multi_class='ovr', n_jobs=1,\n",
      "          penalty='l2', random_state=None, solver='liblinear', tol=0.0001,\n",
      "          verbose=0, warm_start=False)\n",
      "0.777086233724\n"
     ]
    }
   ],
   "source": [
    "print(model.best_estimator_)\n",
    "print(model.score(x_test, y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "y_pred = model.best_estimator_.predict(x_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "77.708623372369175"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "acc = accuracy_score(y_test, y_pred, normalize=True) * float(100)\n",
    "acc"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We can clearly see increasing dimenions the model performs better than 100-D,200-D as the chances of data becomes linearly separable in higher dimensions increases.So we get the best hyperplane as dimensions increases due to which model behaves better."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Confusion Matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[12951,  3967],\n",
       "       [ 4199, 15516]], dtype=int64)"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "confusion_matrix(y_test,y_pred).T"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The confusion matrix of 300-D is better than other two"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## L1 regularization and sparsity"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "300\n"
     ]
    }
   ],
   "source": [
    "clf = LogisticRegression(C=1, penalty = 'l1')\n",
    "clf.fit(x_train, y_train)\n",
    "w = clf.coef_    \n",
    "print(np.count_nonzero(w))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This means 300 out of 300 are non zero.That means for regularization takes place."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "299\n"
     ]
    }
   ],
   "source": [
    "clf = LogisticRegression(C=0.1, penalty = 'l1')\n",
    "clf.fit(x_train, y_train)\n",
    "w = clf.coef_\n",
    "print(np.count_nonzero(w))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "As we can see clearly as c decreases sparsity increases.This time 299 features out of 300  are non zero,means 1 features is removed."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "125\n"
     ]
    }
   ],
   "source": [
    "clf = LogisticRegression(C=0.001, penalty = 'l1')\n",
    "clf.fit(x_train, y_train)\n",
    "w = clf.coef_\n",
    "print(np.count_nonzero(w))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now the 125 out of 300 are non zero.That means 175 are zero which are removed."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "51\n"
     ]
    }
   ],
   "source": [
    "clf = LogisticRegression(C=0.00045, penalty = 'l1')\n",
    "clf.fit(x_train, y_train)\n",
    "w = clf.coef_\n",
    "print(np.count_nonzero(w))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "At c=0.00045 we are getting top 51 features and rest becomes zero."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}