3.4 Practical.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 3.4 Practical task"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import nltk\n",
    "from nltk.tokenize import word_tokenize\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.stem import WordNetLemmatizer\n",
    "import spacy\n",
    "import re\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "bbc_data = pd.read_csv(\"bbc_news.csv\") # the data should be in the same folder as your notebook"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>index</th>\n",
       "      <th>title</th>\n",
       "      <th>pubDate</th>\n",
       "      <th>guid</th>\n",
       "      <th>link</th>\n",
       "      <th>description</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>6684</td>\n",
       "      <td>Can I refuse to work?</td>\n",
       "      <td>Wed, 10 Aug 2022 15:46:18 GMT</td>\n",
       "      <td>https://www.bbc.co.uk/news/business-62147992</td>\n",
       "      <td>https://www.bbc.co.uk/news/business-62147992?a...</td>\n",
       "      <td>With much of the UK enduring another period of...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>9267</td>\n",
       "      <td>'Liz Truss the Brief?' World reacts to UK poli...</td>\n",
       "      <td>Mon, 17 Oct 2022 11:35:12 GMT</td>\n",
       "      <td>https://www.bbc.co.uk/news/world-63285480</td>\n",
       "      <td>https://www.bbc.co.uk/news/world-63285480?at_m...</td>\n",
       "      <td>The UK's political chaos has been watched arou...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>7387</td>\n",
       "      <td>Rationing energy is nothing new for off-grid c...</td>\n",
       "      <td>Wed, 31 Aug 2022 05:20:18 GMT</td>\n",
       "      <td>https://www.bbc.co.uk/news/uk-scotland-highlan...</td>\n",
       "      <td>https://www.bbc.co.uk/news/uk-scotland-highlan...</td>\n",
       "      <td>Scoraig in the north west Highlands has long h...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>767</td>\n",
       "      <td>The hunt for superyachts of sanctioned Russian...</td>\n",
       "      <td>Tue, 22 Mar 2022 14:37:01 GMT</td>\n",
       "      <td>https://www.bbc.co.uk/news/60739336</td>\n",
       "      <td>https://www.bbc.co.uk/news/60739336?at_medium=...</td>\n",
       "      <td>Wealthy Russians sanctioned by the US, EU and ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>3712</td>\n",
       "      <td>Platinum Jubilee: 70 years of the Queen in 70 ...</td>\n",
       "      <td>Wed, 01 Jun 2022 23:17:33 GMT</td>\n",
       "      <td>https://www.bbc.co.uk/news/uk-61660128</td>\n",
       "      <td>https://www.bbc.co.uk/news/uk-61660128?at_medi...</td>\n",
       "      <td>A quick look back at the Queen's 70 years on t...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Unnamed: 0  index                                              title  \\\n",
       "0           0   6684                              Can I refuse to work?   \n",
       "1           1   9267  'Liz Truss the Brief?' World reacts to UK poli...   \n",
       "2           2   7387  Rationing energy is nothing new for off-grid c...   \n",
       "3           3    767  The hunt for superyachts of sanctioned Russian...   \n",
       "4           4   3712  Platinum Jubilee: 70 years of the Queen in 70 ...   \n",
       "\n",
       "                         pubDate  \\\n",
       "0  Wed, 10 Aug 2022 15:46:18 GMT   \n",
       "1  Mon, 17 Oct 2022 11:35:12 GMT   \n",
       "2  Wed, 31 Aug 2022 05:20:18 GMT   \n",
       "3  Tue, 22 Mar 2022 14:37:01 GMT   \n",
       "4  Wed, 01 Jun 2022 23:17:33 GMT   \n",
       "\n",
       "                                                guid  \\\n",
       "0       https://www.bbc.co.uk/news/business-62147992   \n",
       "1          https://www.bbc.co.uk/news/world-63285480   \n",
       "2  https://www.bbc.co.uk/news/uk-scotland-highlan...   \n",
       "3                https://www.bbc.co.uk/news/60739336   \n",
       "4             https://www.bbc.co.uk/news/uk-61660128   \n",
       "\n",
       "                                                link  \\\n",
       "0  https://www.bbc.co.uk/news/business-62147992?a...   \n",
       "1  https://www.bbc.co.uk/news/world-63285480?at_m...   \n",
       "2  https://www.bbc.co.uk/news/uk-scotland-highlan...   \n",
       "3  https://www.bbc.co.uk/news/60739336?at_medium=...   \n",
       "4  https://www.bbc.co.uk/news/uk-61660128?at_medi...   \n",
       "\n",
       "                                         description  \n",
       "0  With much of the UK enduring another period of...  \n",
       "1  The UK's political chaos has been watched arou...  \n",
       "2  Scoraig in the north west Highlands has long h...  \n",
       "3  Wealthy Russians sanctioned by the US, EU and ...  \n",
       "4  A quick look back at the Queen's 70 years on t...  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bbc_data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 1000 entries, 0 to 999\n",
      "Data columns (total 7 columns):\n",
      " #   Column       Non-Null Count  Dtype \n",
      "---  ------       --------------  ----- \n",
      " 0   Unnamed: 0   1000 non-null   int64 \n",
      " 1   index        1000 non-null   int64 \n",
      " 2   title        1000 non-null   object\n",
      " 3   pubDate      1000 non-null   object\n",
      " 4   guid         1000 non-null   object\n",
      " 5   link         1000 non-null   object\n",
      " 6   description  1000 non-null   object\n",
      "dtypes: int64(2), object(5)\n",
      "memory usage: 54.8+ KB\n"
     ]
    }
   ],
   "source": [
    "bbc_data.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>title</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Can I refuse to work?</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>'Liz Truss the Brief?' World reacts to UK poli...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Rationing energy is nothing new for off-grid c...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>The hunt for superyachts of sanctioned Russian...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Platinum Jubilee: 70 years of the Queen in 70 ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                               title\n",
       "0                              Can I refuse to work?\n",
       "1  'Liz Truss the Brief?' World reacts to UK poli...\n",
       "2  Rationing energy is nothing new for off-grid c...\n",
       "3  The hunt for superyachts of sanctioned Russian...\n",
       "4  Platinum Jubilee: 70 years of the Queen in 70 ..."
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "titles = pd.DataFrame(bbc_data['title'])\n",
    "titles.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Clean data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>title</th>\n",
       "      <th>lowercase</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Can I refuse to work?</td>\n",
       "      <td>can i refuse to work?</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>'Liz Truss the Brief?' World reacts to UK poli...</td>\n",
       "      <td>'liz truss the brief?' world reacts to uk poli...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Rationing energy is nothing new for off-grid c...</td>\n",
       "      <td>rationing energy is nothing new for off-grid c...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>The hunt for superyachts of sanctioned Russian...</td>\n",
       "      <td>the hunt for superyachts of sanctioned russian...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Platinum Jubilee: 70 years of the Queen in 70 ...</td>\n",
       "      <td>platinum jubilee: 70 years of the queen in 70 ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                               title  \\\n",
       "0                              Can I refuse to work?   \n",
       "1  'Liz Truss the Brief?' World reacts to UK poli...   \n",
       "2  Rationing energy is nothing new for off-grid c...   \n",
       "3  The hunt for superyachts of sanctioned Russian...   \n",
       "4  Platinum Jubilee: 70 years of the Queen in 70 ...   \n",
       "\n",
       "                                           lowercase  \n",
       "0                              can i refuse to work?  \n",
       "1  'liz truss the brief?' world reacts to uk poli...  \n",
       "2  rationing energy is nothing new for off-grid c...  \n",
       "3  the hunt for superyachts of sanctioned russian...  \n",
       "4  platinum jubilee: 70 years of the queen in 70 ...  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# lowercase\n",
    "titles['lowercase'] = titles['title'].str.lower()\n",
    "titles.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>title</th>\n",
       "      <th>lowercase</th>\n",
       "      <th>no_stopwords</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Can I refuse to work?</td>\n",
       "      <td>can i refuse to work?</td>\n",
       "      <td>refuse work?</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>'Liz Truss the Brief?' World reacts to UK poli...</td>\n",
       "      <td>'liz truss the brief?' world reacts to uk poli...</td>\n",
       "      <td>'liz truss brief?' world reacts uk political t...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Rationing energy is nothing new for off-grid c...</td>\n",
       "      <td>rationing energy is nothing new for off-grid c...</td>\n",
       "      <td>rationing energy nothing new off-grid community</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>The hunt for superyachts of sanctioned Russian...</td>\n",
       "      <td>the hunt for superyachts of sanctioned russian...</td>\n",
       "      <td>hunt superyachts sanctioned russian oligarchs</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Platinum Jubilee: 70 years of the Queen in 70 ...</td>\n",
       "      <td>platinum jubilee: 70 years of the queen in 70 ...</td>\n",
       "      <td>platinum jubilee: 70 years queen 70 seconds</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                               title  \\\n",
       "0                              Can I refuse to work?   \n",
       "1  'Liz Truss the Brief?' World reacts to UK poli...   \n",
       "2  Rationing energy is nothing new for off-grid c...   \n",
       "3  The hunt for superyachts of sanctioned Russian...   \n",
       "4  Platinum Jubilee: 70 years of the Queen in 70 ...   \n",
       "\n",
       "                                           lowercase  \\\n",
       "0                              can i refuse to work?   \n",
       "1  'liz truss the brief?' world reacts to uk poli...   \n",
       "2  rationing energy is nothing new for off-grid c...   \n",
       "3  the hunt for superyachts of sanctioned russian...   \n",
       "4  platinum jubilee: 70 years of the queen in 70 ...   \n",
       "\n",
       "                                        no_stopwords  \n",
       "0                                       refuse work?  \n",
       "1  'liz truss brief?' world reacts uk political t...  \n",
       "2    rationing energy nothing new off-grid community  \n",
       "3      hunt superyachts sanctioned russian oligarchs  \n",
       "4        platinum jubilee: 70 years queen 70 seconds  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# stop word removal\n",
    "en_stopwords = stopwords.words('english')\n",
    "titles['no_stopwords'] = titles['lowercase'].apply(lambda x: ' '.join([word for word in x.split() if word not in (en_stopwords)]))\n",
    "titles.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>title</th>\n",
       "      <th>lowercase</th>\n",
       "      <th>no_stopwords</th>\n",
       "      <th>no_stopwords_no_punct</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Can I refuse to work?</td>\n",
       "      <td>can i refuse to work?</td>\n",
       "      <td>refuse work?</td>\n",
       "      <td>refuse work</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>'Liz Truss the Brief?' World reacts to UK poli...</td>\n",
       "      <td>'liz truss the brief?' world reacts to uk poli...</td>\n",
       "      <td>'liz truss brief?' world reacts uk political t...</td>\n",
       "      <td>liz truss brief world reacts uk political turmoil</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Rationing energy is nothing new for off-grid c...</td>\n",
       "      <td>rationing energy is nothing new for off-grid c...</td>\n",
       "      <td>rationing energy nothing new off-grid community</td>\n",
       "      <td>rationing energy nothing new offgrid community</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>The hunt for superyachts of sanctioned Russian...</td>\n",
       "      <td>the hunt for superyachts of sanctioned russian...</td>\n",
       "      <td>hunt superyachts sanctioned russian oligarchs</td>\n",
       "      <td>hunt superyachts sanctioned russian oligarchs</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Platinum Jubilee: 70 years of the Queen in 70 ...</td>\n",
       "      <td>platinum jubilee: 70 years of the queen in 70 ...</td>\n",
       "      <td>platinum jubilee: 70 years queen 70 seconds</td>\n",
       "      <td>platinum jubilee 70 years queen 70 seconds</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                               title  \\\n",
       "0                              Can I refuse to work?   \n",
       "1  'Liz Truss the Brief?' World reacts to UK poli...   \n",
       "2  Rationing energy is nothing new for off-grid c...   \n",
       "3  The hunt for superyachts of sanctioned Russian...   \n",
       "4  Platinum Jubilee: 70 years of the Queen in 70 ...   \n",
       "\n",
       "                                           lowercase  \\\n",
       "0                              can i refuse to work?   \n",
       "1  'liz truss the brief?' world reacts to uk poli...   \n",
       "2  rationing energy is nothing new for off-grid c...   \n",
       "3  the hunt for superyachts of sanctioned russian...   \n",
       "4  platinum jubilee: 70 years of the queen in 70 ...   \n",
       "\n",
       "                                        no_stopwords  \\\n",
       "0                                       refuse work?   \n",
       "1  'liz truss brief?' world reacts uk political t...   \n",
       "2    rationing energy nothing new off-grid community   \n",
       "3      hunt superyachts sanctioned russian oligarchs   \n",
       "4        platinum jubilee: 70 years queen 70 seconds   \n",
       "\n",
       "                               no_stopwords_no_punct  \n",
       "0                                        refuse work  \n",
       "1  liz truss brief world reacts uk political turmoil  \n",
       "2     rationing energy nothing new offgrid community  \n",
       "3      hunt superyachts sanctioned russian oligarchs  \n",
       "4         platinum jubilee 70 years queen 70 seconds  "
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# punctation removal\n",
    "titles['no_stopwords_no_punct'] = titles.apply(lambda x: re.sub(r\"([^\\w\\s])\", \"\", x['no_stopwords']), axis=1)\n",
    "titles.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>title</th>\n",
       "      <th>lowercase</th>\n",
       "      <th>no_stopwords</th>\n",
       "      <th>no_stopwords_no_punct</th>\n",
       "      <th>tokens_raw</th>\n",
       "      <th>tokens_clean</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Can I refuse to work?</td>\n",
       "      <td>can i refuse to work?</td>\n",
       "      <td>refuse work?</td>\n",
       "      <td>refuse work</td>\n",
       "      <td>[Can, I, refuse, to, work, ?]</td>\n",
       "      <td>[refuse, work]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>'Liz Truss the Brief?' World reacts to UK poli...</td>\n",
       "      <td>'liz truss the brief?' world reacts to uk poli...</td>\n",
       "      <td>'liz truss brief?' world reacts uk political t...</td>\n",
       "      <td>liz truss brief world reacts uk political turmoil</td>\n",
       "      <td>['Liz, Truss, the, Brief, ?, ', World, reacts,...</td>\n",
       "      <td>[liz, truss, brief, world, reacts, uk, politic...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Rationing energy is nothing new for off-grid c...</td>\n",
       "      <td>rationing energy is nothing new for off-grid c...</td>\n",
       "      <td>rationing energy nothing new off-grid community</td>\n",
       "      <td>rationing energy nothing new offgrid community</td>\n",
       "      <td>[Rationing, energy, is, nothing, new, for, off...</td>\n",
       "      <td>[rationing, energy, nothing, new, offgrid, com...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>The hunt for superyachts of sanctioned Russian...</td>\n",
       "      <td>the hunt for superyachts of sanctioned russian...</td>\n",
       "      <td>hunt superyachts sanctioned russian oligarchs</td>\n",
       "      <td>hunt superyachts sanctioned russian oligarchs</td>\n",
       "      <td>[The, hunt, for, superyachts, of, sanctioned, ...</td>\n",
       "      <td>[hunt, superyachts, sanctioned, russian, oliga...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Platinum Jubilee: 70 years of the Queen in 70 ...</td>\n",
       "      <td>platinum jubilee: 70 years of the queen in 70 ...</td>\n",
       "      <td>platinum jubilee: 70 years queen 70 seconds</td>\n",
       "      <td>platinum jubilee 70 years queen 70 seconds</td>\n",
       "      <td>[Platinum, Jubilee, :, 70, years, of, the, Que...</td>\n",
       "      <td>[platinum, jubilee, 70, years, queen, 70, seco...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                               title  \\\n",
       "0                              Can I refuse to work?   \n",
       "1  'Liz Truss the Brief?' World reacts to UK poli...   \n",
       "2  Rationing energy is nothing new for off-grid c...   \n",
       "3  The hunt for superyachts of sanctioned Russian...   \n",
       "4  Platinum Jubilee: 70 years of the Queen in 70 ...   \n",
       "\n",
       "                                           lowercase  \\\n",
       "0                              can i refuse to work?   \n",
       "1  'liz truss the brief?' world reacts to uk poli...   \n",
       "2  rationing energy is nothing new for off-grid c...   \n",
       "3  the hunt for superyachts of sanctioned russian...   \n",
       "4  platinum jubilee: 70 years of the queen in 70 ...   \n",
       "\n",
       "                                        no_stopwords  \\\n",
       "0                                       refuse work?   \n",
       "1  'liz truss brief?' world reacts uk political t...   \n",
       "2    rationing energy nothing new off-grid community   \n",
       "3      hunt superyachts sanctioned russian oligarchs   \n",
       "4        platinum jubilee: 70 years queen 70 seconds   \n",
       "\n",
       "                               no_stopwords_no_punct  \\\n",
       "0                                        refuse work   \n",
       "1  liz truss brief world reacts uk political turmoil   \n",
       "2     rationing energy nothing new offgrid community   \n",
       "3      hunt superyachts sanctioned russian oligarchs   \n",
       "4         platinum jubilee 70 years queen 70 seconds   \n",
       "\n",
       "                                          tokens_raw  \\\n",
       "0                      [Can, I, refuse, to, work, ?]   \n",
       "1  ['Liz, Truss, the, Brief, ?, ', World, reacts,...   \n",
       "2  [Rationing, energy, is, nothing, new, for, off...   \n",
       "3  [The, hunt, for, superyachts, of, sanctioned, ...   \n",
       "4  [Platinum, Jubilee, :, 70, years, of, the, Que...   \n",
       "\n",
       "                                        tokens_clean  \n",
       "0                                     [refuse, work]  \n",
       "1  [liz, truss, brief, world, reacts, uk, politic...  \n",
       "2  [rationing, energy, nothing, new, offgrid, com...  \n",
       "3  [hunt, superyachts, sanctioned, russian, oliga...  \n",
       "4  [platinum, jubilee, 70, years, queen, 70, seco...  "
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# tokenize\n",
    "titles['tokens_raw'] = titles.apply(lambda x: word_tokenize(x['title']), axis=1)\n",
    "titles['tokens_clean'] = titles.apply(lambda x: word_tokenize(x['no_stopwords_no_punct']), axis=1)\n",
    "titles.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>title</th>\n",
       "      <th>lowercase</th>\n",
       "      <th>no_stopwords</th>\n",
       "      <th>no_stopwords_no_punct</th>\n",
       "      <th>tokens_raw</th>\n",
       "      <th>tokens_clean</th>\n",
       "      <th>tokens_clean_lemmatized</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Can I refuse to work?</td>\n",
       "      <td>can i refuse to work?</td>\n",
       "      <td>refuse work?</td>\n",
       "      <td>refuse work</td>\n",
       "      <td>[Can, I, refuse, to, work, ?]</td>\n",
       "      <td>[refuse, work]</td>\n",
       "      <td>[refuse, work]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>'Liz Truss the Brief?' World reacts to UK poli...</td>\n",
       "      <td>'liz truss the brief?' world reacts to uk poli...</td>\n",
       "      <td>'liz truss brief?' world reacts uk political t...</td>\n",
       "      <td>liz truss brief world reacts uk political turmoil</td>\n",
       "      <td>['Liz, Truss, the, Brief, ?, ', World, reacts,...</td>\n",
       "      <td>[liz, truss, brief, world, reacts, uk, politic...</td>\n",
       "      <td>[liz, truss, brief, world, reacts, uk, politic...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Rationing energy is nothing new for off-grid c...</td>\n",
       "      <td>rationing energy is nothing new for off-grid c...</td>\n",
       "      <td>rationing energy nothing new off-grid community</td>\n",
       "      <td>rationing energy nothing new offgrid community</td>\n",
       "      <td>[Rationing, energy, is, nothing, new, for, off...</td>\n",
       "      <td>[rationing, energy, nothing, new, offgrid, com...</td>\n",
       "      <td>[rationing, energy, nothing, new, offgrid, com...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>The hunt for superyachts of sanctioned Russian...</td>\n",
       "      <td>the hunt for superyachts of sanctioned russian...</td>\n",
       "      <td>hunt superyachts sanctioned russian oligarchs</td>\n",
       "      <td>hunt superyachts sanctioned russian oligarchs</td>\n",
       "      <td>[The, hunt, for, superyachts, of, sanctioned, ...</td>\n",
       "      <td>[hunt, superyachts, sanctioned, russian, oliga...</td>\n",
       "      <td>[hunt, superyachts, sanctioned, russian, oliga...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Platinum Jubilee: 70 years of the Queen in 70 ...</td>\n",
       "      <td>platinum jubilee: 70 years of the queen in 70 ...</td>\n",
       "      <td>platinum jubilee: 70 years queen 70 seconds</td>\n",
       "      <td>platinum jubilee 70 years queen 70 seconds</td>\n",
       "      <td>[Platinum, Jubilee, :, 70, years, of, the, Que...</td>\n",
       "      <td>[platinum, jubilee, 70, years, queen, 70, seco...</td>\n",
       "      <td>[platinum, jubilee, 70, year, queen, 70, second]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                               title  \\\n",
       "0                              Can I refuse to work?   \n",
       "1  'Liz Truss the Brief?' World reacts to UK poli...   \n",
       "2  Rationing energy is nothing new for off-grid c...   \n",
       "3  The hunt for superyachts of sanctioned Russian...   \n",
       "4  Platinum Jubilee: 70 years of the Queen in 70 ...   \n",
       "\n",
       "                                           lowercase  \\\n",
       "0                              can i refuse to work?   \n",
       "1  'liz truss the brief?' world reacts to uk poli...   \n",
       "2  rationing energy is nothing new for off-grid c...   \n",
       "3  the hunt for superyachts of sanctioned russian...   \n",
       "4  platinum jubilee: 70 years of the queen in 70 ...   \n",
       "\n",
       "                                        no_stopwords  \\\n",
       "0                                       refuse work?   \n",
       "1  'liz truss brief?' world reacts uk political t...   \n",
       "2    rationing energy nothing new off-grid community   \n",
       "3      hunt superyachts sanctioned russian oligarchs   \n",
       "4        platinum jubilee: 70 years queen 70 seconds   \n",
       "\n",
       "                               no_stopwords_no_punct  \\\n",
       "0                                        refuse work   \n",
       "1  liz truss brief world reacts uk political turmoil   \n",
       "2     rationing energy nothing new offgrid community   \n",
       "3      hunt superyachts sanctioned russian oligarchs   \n",
       "4         platinum jubilee 70 years queen 70 seconds   \n",
       "\n",
       "                                          tokens_raw  \\\n",
       "0                      [Can, I, refuse, to, work, ?]   \n",
       "1  ['Liz, Truss, the, Brief, ?, ', World, reacts,...   \n",
       "2  [Rationing, energy, is, nothing, new, for, off...   \n",
       "3  [The, hunt, for, superyachts, of, sanctioned, ...   \n",
       "4  [Platinum, Jubilee, :, 70, years, of, the, Que...   \n",
       "\n",
       "                                        tokens_clean  \\\n",
       "0                                     [refuse, work]   \n",
       "1  [liz, truss, brief, world, reacts, uk, politic...   \n",
       "2  [rationing, energy, nothing, new, offgrid, com...   \n",
       "3  [hunt, superyachts, sanctioned, russian, oliga...   \n",
       "4  [platinum, jubilee, 70, years, queen, 70, seco...   \n",
       "\n",
       "                             tokens_clean_lemmatized  \n",
       "0                                     [refuse, work]  \n",
       "1  [liz, truss, brief, world, reacts, uk, politic...  \n",
       "2  [rationing, energy, nothing, new, offgrid, com...  \n",
       "3  [hunt, superyachts, sanctioned, russian, oliga...  \n",
       "4   [platinum, jubilee, 70, year, queen, 70, second]  "
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# lemmatizing \n",
    "lemmatizer = WordNetLemmatizer()\n",
    "titles[\"tokens_clean_lemmatized\"] = titles[\"tokens_clean\"].apply(lambda tokens: [lemmatizer.lemmatize(token) for token in tokens])\n",
    "titles.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# create lists for just our tokens\n",
    "tokens_raw_list = sum(titles['tokens_raw'], []) #unpack our lists into a single list\n",
    "tokens_clean_list = sum(titles['tokens_clean_lemmatized'], [])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### POS Tagging"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "nlp = spacy.load('en_core_web_sm')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# create a spacy doc from our raw text - better for pos tagging\n",
    "spacy_doc = nlp(' '.join(tokens_raw_list))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# extract the tokens and pos tags into a dataframe\n",
    "pos_df = pd.DataFrame(columns=['token', 'pos_tag'])\n",
    "\n",
    "for token in spacy_doc:\n",
    "    pos_df = pd.concat([pos_df,\n",
    "                       pd.DataFrame.from_records([{'token': token.text,'pos_tag': token.pos_}])], ignore_index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>token</th>\n",
       "      <th>pos_tag</th>\n",
       "      <th>counts</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>95</th>\n",
       "      <td>:</td>\n",
       "      <td>PUNCT</td>\n",
       "      <td>543</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>'</td>\n",
       "      <td>PUNCT</td>\n",
       "      <td>300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2897</th>\n",
       "      <td>in</td>\n",
       "      <td>ADP</td>\n",
       "      <td>187</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4082</th>\n",
       "      <td>to</td>\n",
       "      <td>PART</td>\n",
       "      <td>175</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3268</th>\n",
       "      <td>of</td>\n",
       "      <td>ADP</td>\n",
       "      <td>172</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>-</td>\n",
       "      <td>PUNCT</td>\n",
       "      <td>166</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4043</th>\n",
       "      <td>the</td>\n",
       "      <td>DET</td>\n",
       "      <td>163</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1856</th>\n",
       "      <td>and</td>\n",
       "      <td>CCONJ</td>\n",
       "      <td>147</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>'s</td>\n",
       "      <td>PART</td>\n",
       "      <td>143</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>97</th>\n",
       "      <td>?</td>\n",
       "      <td>PUNCT</td>\n",
       "      <td>130</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     token pos_tag  counts\n",
       "95       :   PUNCT     543\n",
       "8        '   PUNCT     300\n",
       "2897    in     ADP     187\n",
       "4082    to    PART     175\n",
       "3268    of     ADP     172\n",
       "22       -   PUNCT     166\n",
       "4043   the     DET     163\n",
       "1856   and   CCONJ     147\n",
       "15      's    PART     143\n",
       "97       ?   PUNCT     130"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# token frequency count\n",
    "pos_df_counts = pos_df.groupby(['token','pos_tag']).size().reset_index(name='counts').sort_values(by='counts', ascending=False)\n",
    "pos_df_counts.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>token</th>\n",
       "      <th>pos_tag</th>\n",
       "      <th>counts</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>4267</th>\n",
       "      <td>war</td>\n",
       "      <td>NOUN</td>\n",
       "      <td>35</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3552</th>\n",
       "      <td>record</td>\n",
       "      <td>NOUN</td>\n",
       "      <td>15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3416</th>\n",
       "      <td>police</td>\n",
       "      <td>NOUN</td>\n",
       "      <td>14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4356</th>\n",
       "      <td>year</td>\n",
       "      <td>NOUN</td>\n",
       "      <td>14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4316</th>\n",
       "      <td>win</td>\n",
       "      <td>NOUN</td>\n",
       "      <td>14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3061</th>\n",
       "      <td>living</td>\n",
       "      <td>NOUN</td>\n",
       "      <td>13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4009</th>\n",
       "      <td>tax</td>\n",
       "      <td>NOUN</td>\n",
       "      <td>13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2326</th>\n",
       "      <td>day</td>\n",
       "      <td>NOUN</td>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3368</th>\n",
       "      <td>people</td>\n",
       "      <td>NOUN</td>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2031</th>\n",
       "      <td>boss</td>\n",
       "      <td>NOUN</td>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       token pos_tag  counts\n",
       "4267     war    NOUN      35\n",
       "3552  record    NOUN      15\n",
       "3416  police    NOUN      14\n",
       "4356    year    NOUN      14\n",
       "4316     win    NOUN      14\n",
       "3061  living    NOUN      13\n",
       "4009     tax    NOUN      13\n",
       "2326     day    NOUN      12\n",
       "3368  people    NOUN      12\n",
       "2031    boss    NOUN      11"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# most common nouns\n",
    "nouns = pos_df_counts[pos_df_counts.pos_tag == \"NOUN\"][0:10]\n",
    "nouns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>token</th>\n",
       "      <th>pos_tag</th>\n",
       "      <th>counts</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>3687</th>\n",
       "      <td>says</td>\n",
       "      <td>VERB</td>\n",
       "      <td>30</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>'</td>\n",
       "      <td>VERB</td>\n",
       "      <td>14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2670</th>\n",
       "      <td>found</td>\n",
       "      <td>VERB</td>\n",
       "      <td>13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4317</th>\n",
       "      <td>win</td>\n",
       "      <td>VERB</td>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4324</th>\n",
       "      <td>wins</td>\n",
       "      <td>VERB</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2713</th>\n",
       "      <td>get</td>\n",
       "      <td>VERB</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2388</th>\n",
       "      <td>dies</td>\n",
       "      <td>VERB</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3990</th>\n",
       "      <td>take</td>\n",
       "      <td>VERB</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2982</th>\n",
       "      <td>killed</td>\n",
       "      <td>VERB</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3686</th>\n",
       "      <td>say</td>\n",
       "      <td>VERB</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       token pos_tag  counts\n",
       "3687    says    VERB      30\n",
       "9          '    VERB      14\n",
       "2670   found    VERB      13\n",
       "4317     win    VERB      12\n",
       "4324    wins    VERB      10\n",
       "2713     get    VERB       9\n",
       "2388    dies    VERB       9\n",
       "3990    take    VERB       8\n",
       "2982  killed    VERB       8\n",
       "3686     say    VERB       8"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# most common verbs\n",
    "verbs = pos_df_counts[pos_df_counts.pos_tag == \"VERB\"][0:10]\n",
    "verbs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>token</th>\n",
       "      <th>pos_tag</th>\n",
       "      <th>counts</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>3244</th>\n",
       "      <td>new</td>\n",
       "      <td>ADJ</td>\n",
       "      <td>28</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1400</th>\n",
       "      <td>Russian</td>\n",
       "      <td>ADJ</td>\n",
       "      <td>21</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2606</th>\n",
       "      <td>final</td>\n",
       "      <td>ADJ</td>\n",
       "      <td>16</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>-</td>\n",
       "      <td>ADJ</td>\n",
       "      <td>14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2625</th>\n",
       "      <td>first</td>\n",
       "      <td>ADJ</td>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3199</th>\n",
       "      <td>more</td>\n",
       "      <td>ADJ</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1994</th>\n",
       "      <td>big</td>\n",
       "      <td>ADJ</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2835</th>\n",
       "      <td>high</td>\n",
       "      <td>ADJ</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3000</th>\n",
       "      <td>last</td>\n",
       "      <td>ADJ</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3304</th>\n",
       "      <td>other</td>\n",
       "      <td>ADJ</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        token pos_tag  counts\n",
       "3244      new     ADJ      28\n",
       "1400  Russian     ADJ      21\n",
       "2606    final     ADJ      16\n",
       "19          -     ADJ      14\n",
       "2625    first     ADJ      12\n",
       "3199     more     ADJ      10\n",
       "1994      big     ADJ       9\n",
       "2835     high     ADJ       9\n",
       "3000     last     ADJ       8\n",
       "3304    other     ADJ       8"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# most common adjectives\n",
    "adj = pos_df_counts[pos_df_counts.pos_tag == \"ADJ\"][0:10]\n",
    "adj"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### NER"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "# extract the tokens and entity tags into a dataframe\n",
    "ner_df = pd.DataFrame(columns=['token', 'ner_tag'])\n",
    "\n",
    "for token in spacy_doc.ents:\n",
    "    if pd.isna(token.label_) is False:\n",
    "        ner_df = pd.concat([ner_df, pd.DataFrame.from_records(\n",
    "            [{'token': token.text, 'ner_tag': token.label_}])], ignore_index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>token</th>\n",
       "      <th>ner_tag</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Liz Truss</td>\n",
       "      <td>PERSON</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>UK</td>\n",
       "      <td>GPE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Rationing</td>\n",
       "      <td>PRODUCT</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>superyachts</td>\n",
       "      <td>CARDINAL</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Russian</td>\n",
       "      <td>NORP</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         token   ner_tag\n",
       "0    Liz Truss    PERSON\n",
       "1           UK       GPE\n",
       "2    Rationing   PRODUCT\n",
       "3  superyachts  CARDINAL\n",
       "4      Russian      NORP"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ner_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>token</th>\n",
       "      <th>ner_tag</th>\n",
       "      <th>counts</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>965</th>\n",
       "      <td>Ukraine</td>\n",
       "      <td>GPE</td>\n",
       "      <td>47</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>955</th>\n",
       "      <td>UK</td>\n",
       "      <td>GPE</td>\n",
       "      <td>36</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>329</th>\n",
       "      <td>England</td>\n",
       "      <td>GPE</td>\n",
       "      <td>32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>819</th>\n",
       "      <td>Russian</td>\n",
       "      <td>NORP</td>\n",
       "      <td>20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>957</th>\n",
       "      <td>US</td>\n",
       "      <td>GPE</td>\n",
       "      <td>19</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1031</th>\n",
       "      <td>World Cup 2022</td>\n",
       "      <td>EVENT</td>\n",
       "      <td>18</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1058</th>\n",
       "      <td>first</td>\n",
       "      <td>ORDINAL</td>\n",
       "      <td>13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>918</th>\n",
       "      <td>The Papers</td>\n",
       "      <td>WORK_OF_ART</td>\n",
       "      <td>13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>378</th>\n",
       "      <td>France</td>\n",
       "      <td>GPE</td>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>226</th>\n",
       "      <td>China</td>\n",
       "      <td>GPE</td>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               token      ner_tag  counts\n",
       "965          Ukraine          GPE      47\n",
       "955               UK          GPE      36\n",
       "329          England          GPE      32\n",
       "819          Russian         NORP      20\n",
       "957               US          GPE      19\n",
       "1031  World Cup 2022        EVENT      18\n",
       "1058           first      ORDINAL      13\n",
       "918       The Papers  WORK_OF_ART      13\n",
       "378           France          GPE      12\n",
       "226            China          GPE      11"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# token frequency count\n",
    "ner_df_counts = ner_df.groupby(['token','ner_tag']).size().reset_index(name='counts').sort_values(by='counts', ascending=False)\n",
    "ner_df_counts.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>token</th>\n",
       "      <th>ner_tag</th>\n",
       "      <th>counts</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>257</th>\n",
       "      <td>Covid</td>\n",
       "      <td>PERSON</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>760</th>\n",
       "      <td>Queen</td>\n",
       "      <td>PERSON</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>757</th>\n",
       "      <td>Putin</td>\n",
       "      <td>PERSON</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>169</th>\n",
       "      <td>Boris Johnson</td>\n",
       "      <td>PERSON</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>563</th>\n",
       "      <td>Liz Truss</td>\n",
       "      <td>PERSON</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>788</th>\n",
       "      <td>Rishi Sunak</td>\n",
       "      <td>PERSON</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>581</th>\n",
       "      <td>Macron</td>\n",
       "      <td>PERSON</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>762</th>\n",
       "      <td>Quiz</td>\n",
       "      <td>PERSON</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>515</th>\n",
       "      <td>Jurgen Klopp</td>\n",
       "      <td>PERSON</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>325</th>\n",
       "      <td>Emma Raducanu</td>\n",
       "      <td>PERSON</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             token ner_tag  counts\n",
       "257          Covid  PERSON       9\n",
       "760          Queen  PERSON       8\n",
       "757          Putin  PERSON       8\n",
       "169  Boris Johnson  PERSON       6\n",
       "563      Liz Truss  PERSON       6\n",
       "788    Rishi Sunak  PERSON       5\n",
       "581         Macron  PERSON       4\n",
       "762           Quiz  PERSON       4\n",
       "515   Jurgen Klopp  PERSON       4\n",
       "325  Emma Raducanu  PERSON       4"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# most common people\n",
    "people = ner_df_counts[ner_df_counts.ner_tag == \"PERSON\"][0:10]\n",
    "people"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>token</th>\n",
       "      <th>ner_tag</th>\n",
       "      <th>counts</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>965</th>\n",
       "      <td>Ukraine</td>\n",
       "      <td>GPE</td>\n",
       "      <td>47</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>955</th>\n",
       "      <td>UK</td>\n",
       "      <td>GPE</td>\n",
       "      <td>36</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>329</th>\n",
       "      <td>England</td>\n",
       "      <td>GPE</td>\n",
       "      <td>32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>957</th>\n",
       "      <td>US</td>\n",
       "      <td>GPE</td>\n",
       "      <td>19</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>378</th>\n",
       "      <td>France</td>\n",
       "      <td>GPE</td>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>226</th>\n",
       "      <td>China</td>\n",
       "      <td>GPE</td>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>817</th>\n",
       "      <td>Russia</td>\n",
       "      <td>GPE</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>454</th>\n",
       "      <td>India</td>\n",
       "      <td>GPE</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>132</th>\n",
       "      <td>Australia</td>\n",
       "      <td>GPE</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>566</th>\n",
       "      <td>London</td>\n",
       "      <td>GPE</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         token ner_tag  counts\n",
       "965    Ukraine     GPE      47\n",
       "955         UK     GPE      36\n",
       "329    England     GPE      32\n",
       "957         US     GPE      19\n",
       "378     France     GPE      12\n",
       "226      China     GPE      11\n",
       "817     Russia     GPE       9\n",
       "454      India     GPE       8\n",
       "132  Australia     GPE       7\n",
       "566     London     GPE       7"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# most common places\n",
    "places = ner_df_counts[ner_df_counts.ner_tag == \"GPE\"][0:10]\n",
    "places"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "nlp_course_env",
   "language": "python",
   "name": "nlp_course_env"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}