Sentence2Vector.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "from gensim import corpora\n",
    "import gensim.models as gsm\n",
    "import re\n",
    "from bs4 import BeautifulSoup\n",
    "import nltk\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.tokenize import word_tokenize \n",
    "from nltk.stem import SnowballStemmer\n",
    "from nltk.stem import PorterStemmer\n",
    "from nltk.tokenize import sent_tokenize, word_tokenize\n",
    "from gensim.models import Word2Vec"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 163,
   "metadata": {},
   "outputs": [],
   "source": [
    "t= pd.read_table('output_rj.txt', names=['docid', 'pageid','text'], header=None, delimiter=\"\\t\", quoting=3,encoding = \"utf-8\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 164,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>docid</th>\n",
       "      <th>pageid</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>12</td>\n",
       "      <td>0</td>\n",
       "      <td>Anarchism is often defined as a political phil...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>12</td>\n",
       "      <td>1</td>\n",
       "      <td>However, others argue that while anti-statism ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>12</td>\n",
       "      <td>2</td>\n",
       "      <td>Therefore, they argue instead that anarchism e...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>12</td>\n",
       "      <td>3</td>\n",
       "      <td>Proponents of anarchism, known as \"anarchists\"...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>12</td>\n",
       "      <td>4</td>\n",
       "      <td>As a subtle and anti-dogmatic philosophy, anar...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>12</td>\n",
       "      <td>5</td>\n",
       "      <td>Anarchism does not offer a fixed body of doctr...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>12</td>\n",
       "      <td>6</td>\n",
       "      <td>There are many types and traditions of anarchi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>12</td>\n",
       "      <td>7</td>\n",
       "      <td>Anarchist schools of thought can differ fundam...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>12</td>\n",
       "      <td>8</td>\n",
       "      <td>Strains of anarchism have often been divided i...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>12</td>\n",
       "      <td>9</td>\n",
       "      <td>Anarchism is often considered a radical left-w...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>12</td>\n",
       "      <td>10</td>\n",
       "      <td>Anarchism as a mass social movement has regula...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>12</td>\n",
       "      <td>11</td>\n",
       "      <td>The central tendency of anarchism as a social ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>12</td>\n",
       "      <td>12</td>\n",
       "      <td>Many anarchists oppose all forms of aggression...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>12</td>\n",
       "      <td>13</td>\n",
       "      <td>Etymology and terminology The term anarchism d...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>12</td>\n",
       "      <td>14</td>\n",
       "      <td>\"Anarchists\" was the term adopted by Maximilie...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>12</td>\n",
       "      <td>15</td>\n",
       "      <td>There would be many revolutionaries of the ear...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>12</td>\n",
       "      <td>16</td>\n",
       "      <td>Pierre-Joseph Proudhon was the first political...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>12</td>\n",
       "      <td>17</td>\n",
       "      <td>Since the 1890s from France, the term \"liberta...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>12</td>\n",
       "      <td>18</td>\n",
       "      <td>On the other hand, some use \"libertarianism\" t...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>12</td>\n",
       "      <td>19</td>\n",
       "      <td>History Origins The earliest anarchist themes ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>12</td>\n",
       "      <td>20</td>\n",
       "      <td>Zhuangzi's philosophy has been described by va...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>12</td>\n",
       "      <td>21</td>\n",
       "      <td>Zhuangzi wrote, \"A petty thief is put in jail.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>12</td>\n",
       "      <td>22</td>\n",
       "      <td>A great brigand becomes a ruler of a Nation.\"</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>12</td>\n",
       "      <td>23</td>\n",
       "      <td>Diogenes of Sinope and the Cynics, their conte...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>12</td>\n",
       "      <td>24</td>\n",
       "      <td>Jesus is sometimes considered the first anarch...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>12</td>\n",
       "      <td>25</td>\n",
       "      <td>Georges Lechartier wrote that \"The true founde...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>12</td>\n",
       "      <td>26</td>\n",
       "      <td>\" Such a distinction reverberates subversive r...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>12</td>\n",
       "      <td>27</td>\n",
       "      <td>This is exemplified when the glorification of ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>12</td>\n",
       "      <td>28</td>\n",
       "      <td>The French renaissance political philosopher É...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>12</td>\n",
       "      <td>29</td>\n",
       "      <td>The radical Protestant Christian Gerrard Winst...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14936</th>\n",
       "      <td>752</td>\n",
       "      <td>263</td>\n",
       "      <td>Turner, saw art's role as the communication by...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14937</th>\n",
       "      <td>752</td>\n",
       "      <td>264</td>\n",
       "      <td>The definition and evaluation of art has becom...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14938</th>\n",
       "      <td>752</td>\n",
       "      <td>265</td>\n",
       "      <td>Richard Wollheim distinguishes three approache...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14939</th>\n",
       "      <td>752</td>\n",
       "      <td>266</td>\n",
       "      <td>The arrival of Modernism in the late nineteent...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14940</th>\n",
       "      <td>752</td>\n",
       "      <td>267</td>\n",
       "      <td>Clement Greenberg's 1960 article \"Modernist Pa...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14941</th>\n",
       "      <td>752</td>\n",
       "      <td>268</td>\n",
       "      <td>Greenberg originally applied this idea to the ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14942</th>\n",
       "      <td>752</td>\n",
       "      <td>269</td>\n",
       "      <td>Though only originally intended as a way of un...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14943</th>\n",
       "      <td>752</td>\n",
       "      <td>270</td>\n",
       "      <td>Pop artists like Andy Warhol became both notew...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14944</th>\n",
       "      <td>752</td>\n",
       "      <td>271</td>\n",
       "      <td>Artists of the 1980s, 1990s, and 2000s expande...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14945</th>\n",
       "      <td>752</td>\n",
       "      <td>272</td>\n",
       "      <td>Duchamp once proposed that art is any activity...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14946</th>\n",
       "      <td>752</td>\n",
       "      <td>273</td>\n",
       "      <td>However the way that only certain activities a...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14947</th>\n",
       "      <td>752</td>\n",
       "      <td>274</td>\n",
       "      <td>There is evidence that there may be an element...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14948</th>\n",
       "      <td>752</td>\n",
       "      <td>275</td>\n",
       "      <td>The Invention of Art: A Cultural History is an...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14949</th>\n",
       "      <td>752</td>\n",
       "      <td>276</td>\n",
       "      <td>Fine Art.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14950</th>\n",
       "      <td>752</td>\n",
       "      <td>277</td>\n",
       "      <td>Shiner finds evidence that the older system of...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14951</th>\n",
       "      <td>752</td>\n",
       "      <td>278</td>\n",
       "      <td>Ancient Greek society did not possess the term...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14952</th>\n",
       "      <td>752</td>\n",
       "      <td>279</td>\n",
       "      <td>Techne can be understood neither as art or cra...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14953</th>\n",
       "      <td>752</td>\n",
       "      <td>280</td>\n",
       "      <td>Techne included painting, sculpting and music ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14954</th>\n",
       "      <td>752</td>\n",
       "      <td>281</td>\n",
       "      <td>Classification disputes Disputes as to whether...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14955</th>\n",
       "      <td>752</td>\n",
       "      <td>282</td>\n",
       "      <td>Classificatory disputes in the 20th century ha...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14956</th>\n",
       "      <td>752</td>\n",
       "      <td>283</td>\n",
       "      <td>Philosopher David Novitz has argued that disag...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14957</th>\n",
       "      <td>752</td>\n",
       "      <td>284</td>\n",
       "      <td>Rather, \"the passionate concerns and interests...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14958</th>\n",
       "      <td>752</td>\n",
       "      <td>285</td>\n",
       "      <td>According to Novitz, classificatory disputes a...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14959</th>\n",
       "      <td>752</td>\n",
       "      <td>286</td>\n",
       "      <td>For example, when the Daily Mail criticized Hi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14960</th>\n",
       "      <td>752</td>\n",
       "      <td>287</td>\n",
       "      <td>Today, pickled sheep and soiled beds threaten ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14961</th>\n",
       "      <td>752</td>\n",
       "      <td>288</td>\n",
       "      <td>In 1998, Arthur Danto, suggested a thought exp...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14962</th>\n",
       "      <td>752</td>\n",
       "      <td>289</td>\n",
       "      <td>Cultural interpretation (an art theory of some...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14963</th>\n",
       "      <td>752</td>\n",
       "      <td>290</td>\n",
       "      <td>Anti-art is a label for art that intentionally...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14964</th>\n",
       "      <td>752</td>\n",
       "      <td>291</td>\n",
       "      <td>One of these,  Fountain (1917), an ordinary ur...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14965</th>\n",
       "      <td>752</td>\n",
       "      <td>292</td>\n",
       "      <td>Anti-art is a feature of work by Situationist ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>14966 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       docid  pageid                                               text\n",
       "0         12       0  Anarchism is often defined as a political phil...\n",
       "1         12       1  However, others argue that while anti-statism ...\n",
       "2         12       2  Therefore, they argue instead that anarchism e...\n",
       "3         12       3  Proponents of anarchism, known as \"anarchists\"...\n",
       "4         12       4  As a subtle and anti-dogmatic philosophy, anar...\n",
       "5         12       5  Anarchism does not offer a fixed body of doctr...\n",
       "6         12       6  There are many types and traditions of anarchi...\n",
       "7         12       7  Anarchist schools of thought can differ fundam...\n",
       "8         12       8  Strains of anarchism have often been divided i...\n",
       "9         12       9  Anarchism is often considered a radical left-w...\n",
       "10        12      10  Anarchism as a mass social movement has regula...\n",
       "11        12      11  The central tendency of anarchism as a social ...\n",
       "12        12      12  Many anarchists oppose all forms of aggression...\n",
       "13        12      13  Etymology and terminology The term anarchism d...\n",
       "14        12      14  \"Anarchists\" was the term adopted by Maximilie...\n",
       "15        12      15  There would be many revolutionaries of the ear...\n",
       "16        12      16  Pierre-Joseph Proudhon was the first political...\n",
       "17        12      17  Since the 1890s from France, the term \"liberta...\n",
       "18        12      18  On the other hand, some use \"libertarianism\" t...\n",
       "19        12      19  History Origins The earliest anarchist themes ...\n",
       "20        12      20  Zhuangzi's philosophy has been described by va...\n",
       "21        12      21     Zhuangzi wrote, \"A petty thief is put in jail.\n",
       "22        12      22      A great brigand becomes a ruler of a Nation.\"\n",
       "23        12      23  Diogenes of Sinope and the Cynics, their conte...\n",
       "24        12      24  Jesus is sometimes considered the first anarch...\n",
       "25        12      25  Georges Lechartier wrote that \"The true founde...\n",
       "26        12      26  \" Such a distinction reverberates subversive r...\n",
       "27        12      27  This is exemplified when the glorification of ...\n",
       "28        12      28  The French renaissance political philosopher É...\n",
       "29        12      29  The radical Protestant Christian Gerrard Winst...\n",
       "...      ...     ...                                                ...\n",
       "14936    752     263  Turner, saw art's role as the communication by...\n",
       "14937    752     264  The definition and evaluation of art has becom...\n",
       "14938    752     265  Richard Wollheim distinguishes three approache...\n",
       "14939    752     266  The arrival of Modernism in the late nineteent...\n",
       "14940    752     267  Clement Greenberg's 1960 article \"Modernist Pa...\n",
       "14941    752     268  Greenberg originally applied this idea to the ...\n",
       "14942    752     269  Though only originally intended as a way of un...\n",
       "14943    752     270  Pop artists like Andy Warhol became both notew...\n",
       "14944    752     271  Artists of the 1980s, 1990s, and 2000s expande...\n",
       "14945    752     272  Duchamp once proposed that art is any activity...\n",
       "14946    752     273  However the way that only certain activities a...\n",
       "14947    752     274  There is evidence that there may be an element...\n",
       "14948    752     275  The Invention of Art: A Cultural History is an...\n",
       "14949    752     276                                          Fine Art.\n",
       "14950    752     277  Shiner finds evidence that the older system of...\n",
       "14951    752     278  Ancient Greek society did not possess the term...\n",
       "14952    752     279  Techne can be understood neither as art or cra...\n",
       "14953    752     280  Techne included painting, sculpting and music ...\n",
       "14954    752     281  Classification disputes Disputes as to whether...\n",
       "14955    752     282  Classificatory disputes in the 20th century ha...\n",
       "14956    752     283  Philosopher David Novitz has argued that disag...\n",
       "14957    752     284  Rather, \"the passionate concerns and interests...\n",
       "14958    752     285  According to Novitz, classificatory disputes a...\n",
       "14959    752     286  For example, when the Daily Mail criticized Hi...\n",
       "14960    752     287  Today, pickled sheep and soiled beds threaten ...\n",
       "14961    752     288  In 1998, Arthur Danto, suggested a thought exp...\n",
       "14962    752     289  Cultural interpretation (an art theory of some...\n",
       "14963    752     290  Anti-art is a label for art that intentionally...\n",
       "14964    752     291  One of these,  Fountain (1917), an ordinary ur...\n",
       "14965    752     292  Anti-art is a feature of work by Situationist ...\n",
       "\n",
       "[14966 rows x 3 columns]"
      ]
     },
     "execution_count": 164,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "t"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 165,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Anarchism is often defined as a political philosophy which holds the state to be undesirable, unnecessary, or harmful.'"
      ]
     },
     "execution_count": 165,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "t['text'][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def text_to_words( text ):\n",
    "    #convert emoji into name and append at the end of text\n",
    "    text = str(text)\n",
    "    for char in text:\n",
    "        if char in emoji.UNICODE_EMOJI:\n",
    "            text = text + unicodedata.name(char) + \" \"\n",
    "\n",
    "    #remove any url to URL\n",
    "    text = re.sub('((www\\.[^\\s]+)|(https?://[^\\s]+))','URL',text)\n",
    "    #Convert any @Username to \"AT_USER\"\n",
    "    text = re.sub('@[^\\s]+','AT_USER',text)\n",
    "    \n",
    "    #Remove additional white spaces\n",
    "    text = re.sub('[\\s]+', ' ', text)\n",
    "    text = re.sub('[\\n]+', ' ', text)\n",
    "    #Remove not alphanumeric symbols white spaces\n",
    "    text = re.sub(r'[^\\w]', ' ', text)\n",
    "    #Replace #word with word\n",
    "    text = re.sub(r'#([^\\s]+)', r'\\1', text)\n",
    "    #trim\n",
    "    text = text.strip('\\'\"')\n",
    "    \n",
    "    letters_only = re.sub(\"[^a-zA-Z]\", \" \", text)\n",
    "    \n",
    "    stop_words = set(stopwords.words('english'))\n",
    "    stemmer = SnowballStemmer('english')\n",
    "    tokens = word_tokenize(letters_only.lower())\n",
    "    filtered = [word for word in tokens if word not in stop_words]\n",
    "    stemmed = [stemmer.stem(word) for word in filtered] \n",
    "    \n",
    "    return  stemmed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 166,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Review 10000 of 14966\n",
      "\n"
     ]
    }
   ],
   "source": [
    "size = len(t['text'])\n",
    "text_clean = []\n",
    "for i in range(size):\n",
    "    #print states after 10000 reviews\n",
    "    if ((i+1)%10000 == 0):\n",
    "        print(\"Review %d of %d\\n\" % ( i+1, size ) )\n",
    "    text_clean.append( text_to_words( t['text'][i] ) )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = gsm.KeyedVectors.load_word2vec_format('word2vec_twitter_model.bin', binary=True,unicode_errors='ignore')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def makeFeatureVec_w2v(words):\n",
    "\n",
    "    featureVec = np.zeros((400)).astype(float)  \n",
    "    nwords = 0\n",
    "    \n",
    "    for word in words:\n",
    "        if word in model.vocab:\n",
    "            embedding_vector = model[word]\n",
    "            if embedding_vector is not None: \n",
    "                nwords = nwords + 1\n",
    "                featureVec = np.add(featureVec, embedding_vector)\n",
    "    # \n",
    "    # Divide the result by the number of words to get the average\n",
    "    if nwords != 0:\n",
    "        featureVec = np.divide(featureVec,nwords)\n",
    "    return featureVec"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 169,
   "metadata": {},
   "outputs": [],
   "source": [
    "text_vector = []\n",
    "size = len(t['text'])\n",
    "for i in range(size):\n",
    "    docid = np.asarray(t['docid'][i]).astype(float).reshape((1))\n",
    "    pageid = np.asarray(t['pageid'][i]).astype(float).reshape((1))\n",
    "    temp = np.concatenate((docid, pageid))\n",
    "    text_vector.append(np.concatenate((temp, makeFeatureVec_w2v(text_clean[i]))))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 173,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "text_vector = np.asarray(text_vector)\n",
    "np.savetxt('output_vector.txt', text_vector, delimiter=\",\", fmt=\"%.10f\") "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "py3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}