2.9 Practical.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 2.9 Practical"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import nltk\n",
    "from nltk.tokenize import word_tokenize\n",
    "from nltk.stem import PorterStemmer, WordNetLemmatizer\n",
    "from nltk.corpus import stopwords\n",
    "import re\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = pd.read_csv(\"tripadvisor_hotel_reviews.csv\") # the data should be in the same folder as your notebook"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 109 entries, 0 to 108\n",
      "Data columns (total 2 columns):\n",
      " #   Column  Non-Null Count  Dtype \n",
      "---  ------  --------------  ----- \n",
      " 0   Review  109 non-null    object\n",
      " 1   Rating  109 non-null    int64 \n",
      "dtypes: int64(1), object(1)\n",
      "memory usage: 1.8+ KB\n"
     ]
    }
   ],
   "source": [
    "data.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Review</th>\n",
       "      <th>Rating</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>nice hotel expensive parking got good deal sta...</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ok nothing special charge diamond member hilto...</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>nice rooms not 4* experience hotel monaco seat...</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>unique, great stay, wonderful time hotel monac...</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>great stay great stay, went seahawk game aweso...</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                              Review  Rating\n",
       "0  nice hotel expensive parking got good deal sta...       4\n",
       "1  ok nothing special charge diamond member hilto...       2\n",
       "2  nice rooms not 4* experience hotel monaco seat...       3\n",
       "3  unique, great stay, wonderful time hotel monac...       5\n",
       "4  great stay great stay, went seahawk game aweso...       5"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'nice hotel expensive parking got good deal stay hotel anniversary, arrived late evening took advice previous reviews did valet parking, check quick easy, little disappointed non-existent view room room clean nice size, bed comfortable woke stiff neck high pillows, not soundproof like heard music room night morning loud bangs doors opening closing hear people talking hallway, maybe just noisy neighbors, aveda bath products nice, did not goldfish stay nice touch taken advantage staying longer, location great walking distance shopping, overall nice experience having pay 40 parking night,  '"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data['Review'][0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Lowercase"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "data['review_lowercase'] = data['Review'].str.lower()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Review</th>\n",
       "      <th>Rating</th>\n",
       "      <th>review_lowercase</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>nice hotel expensive parking got good deal sta...</td>\n",
       "      <td>4</td>\n",
       "      <td>nice hotel expensive parking got good deal sta...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ok nothing special charge diamond member hilto...</td>\n",
       "      <td>2</td>\n",
       "      <td>ok nothing special charge diamond member hilto...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>nice rooms not 4* experience hotel monaco seat...</td>\n",
       "      <td>3</td>\n",
       "      <td>nice rooms not 4* experience hotel monaco seat...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>unique, great stay, wonderful time hotel monac...</td>\n",
       "      <td>5</td>\n",
       "      <td>unique, great stay, wonderful time hotel monac...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>great stay great stay, went seahawk game aweso...</td>\n",
       "      <td>5</td>\n",
       "      <td>great stay great stay, went seahawk game aweso...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                              Review  Rating  \\\n",
       "0  nice hotel expensive parking got good deal sta...       4   \n",
       "1  ok nothing special charge diamond member hilto...       2   \n",
       "2  nice rooms not 4* experience hotel monaco seat...       3   \n",
       "3  unique, great stay, wonderful time hotel monac...       5   \n",
       "4  great stay great stay, went seahawk game aweso...       5   \n",
       "\n",
       "                                    review_lowercase  \n",
       "0  nice hotel expensive parking got good deal sta...  \n",
       "1  ok nothing special charge diamond member hilto...  \n",
       "2  nice rooms not 4* experience hotel monaco seat...  \n",
       "3  unique, great stay, wonderful time hotel monac...  \n",
       "4  great stay great stay, went seahawk game aweso...  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Stop word removal"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "en_stopwords = stopwords.words('english')\n",
    "en_stopwords.remove(\"not\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "data['review_no_stopwords'] = data['review_lowercase'] \\\n",
    "                                .apply(lambda x: ' ' \\\n",
    "                                .join([word for word \\\n",
    "                                in x.split() if word \\\n",
    "                                not in (en_stopwords)]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Review</th>\n",
       "      <th>Rating</th>\n",
       "      <th>review_lowercase</th>\n",
       "      <th>review_no_stopwords</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>nice hotel expensive parking got good deal sta...</td>\n",
       "      <td>4</td>\n",
       "      <td>nice hotel expensive parking got good deal sta...</td>\n",
       "      <td>nice hotel expensive parking got good deal sta...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ok nothing special charge diamond member hilto...</td>\n",
       "      <td>2</td>\n",
       "      <td>ok nothing special charge diamond member hilto...</td>\n",
       "      <td>ok nothing special charge diamond member hilto...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>nice rooms not 4* experience hotel monaco seat...</td>\n",
       "      <td>3</td>\n",
       "      <td>nice rooms not 4* experience hotel monaco seat...</td>\n",
       "      <td>nice rooms not 4* experience hotel monaco seat...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>unique, great stay, wonderful time hotel monac...</td>\n",
       "      <td>5</td>\n",
       "      <td>unique, great stay, wonderful time hotel monac...</td>\n",
       "      <td>unique, great stay, wonderful time hotel monac...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>great stay great stay, went seahawk game aweso...</td>\n",
       "      <td>5</td>\n",
       "      <td>great stay great stay, went seahawk game aweso...</td>\n",
       "      <td>great stay great stay, went seahawk game aweso...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                              Review  Rating  \\\n",
       "0  nice hotel expensive parking got good deal sta...       4   \n",
       "1  ok nothing special charge diamond member hilto...       2   \n",
       "2  nice rooms not 4* experience hotel monaco seat...       3   \n",
       "3  unique, great stay, wonderful time hotel monac...       5   \n",
       "4  great stay great stay, went seahawk game aweso...       5   \n",
       "\n",
       "                                    review_lowercase  \\\n",
       "0  nice hotel expensive parking got good deal sta...   \n",
       "1  ok nothing special charge diamond member hilto...   \n",
       "2  nice rooms not 4* experience hotel monaco seat...   \n",
       "3  unique, great stay, wonderful time hotel monac...   \n",
       "4  great stay great stay, went seahawk game aweso...   \n",
       "\n",
       "                                 review_no_stopwords  \n",
       "0  nice hotel expensive parking got good deal sta...  \n",
       "1  ok nothing special charge diamond member hilto...  \n",
       "2  nice rooms not 4* experience hotel monaco seat...  \n",
       "3  unique, great stay, wonderful time hotel monac...  \n",
       "4  great stay great stay, went seahawk game aweso...  "
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'nice hotel expensive parking got good deal stay hotel anniversary, arrived late evening took advice previous reviews valet parking, check quick easy, little disappointed non-existent view room room clean nice size, bed comfortable woke stiff neck high pillows, not soundproof like heard music room night morning loud bangs doors opening closing hear people talking hallway, maybe noisy neighbors, aveda bath products nice, not goldfish stay nice touch taken advantage staying longer, location great walking distance shopping, overall nice experience pay 40 parking night,'"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data['review_no_stopwords'][0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Punctuation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We want to remove punctation but we notice we have a few reviews with the symbol * instead of the word star. We want to keep this as it might add meaning to the review. We can do this using what we learned about regular expressions."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "data['review_no_stopwords_no_punct'] = data \\\n",
    "                                    .apply(lambda x: \\\n",
    "                                    re.sub(r\"[*]\", \\\n",
    "                                    \"star\", \\\n",
    "                                    x['review_no_stopwords'] \\\n",
    "                                    ), axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Review</th>\n",
       "      <th>Rating</th>\n",
       "      <th>review_lowercase</th>\n",
       "      <th>review_no_stopwords</th>\n",
       "      <th>review_no_stopwords_no_punct</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>nice hotel expensive parking got good deal sta...</td>\n",
       "      <td>4</td>\n",
       "      <td>nice hotel expensive parking got good deal sta...</td>\n",
       "      <td>nice hotel expensive parking got good deal sta...</td>\n",
       "      <td>nice hotel expensive parking got good deal sta...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ok nothing special charge diamond member hilto...</td>\n",
       "      <td>2</td>\n",
       "      <td>ok nothing special charge diamond member hilto...</td>\n",
       "      <td>ok nothing special charge diamond member hilto...</td>\n",
       "      <td>ok nothing special charge diamond member hilto...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>nice rooms not 4* experience hotel monaco seat...</td>\n",
       "      <td>3</td>\n",
       "      <td>nice rooms not 4* experience hotel monaco seat...</td>\n",
       "      <td>nice rooms not 4* experience hotel monaco seat...</td>\n",
       "      <td>nice rooms not 4star experience hotel monaco s...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>unique, great stay, wonderful time hotel monac...</td>\n",
       "      <td>5</td>\n",
       "      <td>unique, great stay, wonderful time hotel monac...</td>\n",
       "      <td>unique, great stay, wonderful time hotel monac...</td>\n",
       "      <td>unique, great stay, wonderful time hotel monac...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>great stay great stay, went seahawk game aweso...</td>\n",
       "      <td>5</td>\n",
       "      <td>great stay great stay, went seahawk game aweso...</td>\n",
       "      <td>great stay great stay, went seahawk game aweso...</td>\n",
       "      <td>great stay great stay, went seahawk game aweso...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                              Review  Rating  \\\n",
       "0  nice hotel expensive parking got good deal sta...       4   \n",
       "1  ok nothing special charge diamond member hilto...       2   \n",
       "2  nice rooms not 4* experience hotel monaco seat...       3   \n",
       "3  unique, great stay, wonderful time hotel monac...       5   \n",
       "4  great stay great stay, went seahawk game aweso...       5   \n",
       "\n",
       "                                    review_lowercase  \\\n",
       "0  nice hotel expensive parking got good deal sta...   \n",
       "1  ok nothing special charge diamond member hilto...   \n",
       "2  nice rooms not 4* experience hotel monaco seat...   \n",
       "3  unique, great stay, wonderful time hotel monac...   \n",
       "4  great stay great stay, went seahawk game aweso...   \n",
       "\n",
       "                                 review_no_stopwords  \\\n",
       "0  nice hotel expensive parking got good deal sta...   \n",
       "1  ok nothing special charge diamond member hilto...   \n",
       "2  nice rooms not 4* experience hotel monaco seat...   \n",
       "3  unique, great stay, wonderful time hotel monac...   \n",
       "4  great stay great stay, went seahawk game aweso...   \n",
       "\n",
       "                        review_no_stopwords_no_punct  \n",
       "0  nice hotel expensive parking got good deal sta...  \n",
       "1  ok nothing special charge diamond member hilto...  \n",
       "2  nice rooms not 4star experience hotel monaco s...  \n",
       "3  unique, great stay, wonderful time hotel monac...  \n",
       "4  great stay great stay, went seahawk game aweso...  "
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "data['review_no_stopwords_no_punct'] = data. \\\n",
    "                                    apply(lambda x: \\\n",
    "                                    re.sub(r\"([^\\w\\s])\", \\\n",
    "                                    \"\", \\\n",
    "                                    x['review_no_stopwords_no_punct'] \\\n",
    "                                    ), axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Review</th>\n",
       "      <th>Rating</th>\n",
       "      <th>review_lowercase</th>\n",
       "      <th>review_no_stopwords</th>\n",
       "      <th>review_no_stopwords_no_punct</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>nice hotel expensive parking got good deal sta...</td>\n",
       "      <td>4</td>\n",
       "      <td>nice hotel expensive parking got good deal sta...</td>\n",
       "      <td>nice hotel expensive parking got good deal sta...</td>\n",
       "      <td>nice hotel expensive parking got good deal sta...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ok nothing special charge diamond member hilto...</td>\n",
       "      <td>2</td>\n",
       "      <td>ok nothing special charge diamond member hilto...</td>\n",
       "      <td>ok nothing special charge diamond member hilto...</td>\n",
       "      <td>ok nothing special charge diamond member hilto...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>nice rooms not 4* experience hotel monaco seat...</td>\n",
       "      <td>3</td>\n",
       "      <td>nice rooms not 4* experience hotel monaco seat...</td>\n",
       "      <td>nice rooms not 4* experience hotel monaco seat...</td>\n",
       "      <td>nice rooms not 4star experience hotel monaco s...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>unique, great stay, wonderful time hotel monac...</td>\n",
       "      <td>5</td>\n",
       "      <td>unique, great stay, wonderful time hotel monac...</td>\n",
       "      <td>unique, great stay, wonderful time hotel monac...</td>\n",
       "      <td>unique great stay wonderful time hotel monaco ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>great stay great stay, went seahawk game aweso...</td>\n",
       "      <td>5</td>\n",
       "      <td>great stay great stay, went seahawk game aweso...</td>\n",
       "      <td>great stay great stay, went seahawk game aweso...</td>\n",
       "      <td>great stay great stay went seahawk game awesom...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                              Review  Rating  \\\n",
       "0  nice hotel expensive parking got good deal sta...       4   \n",
       "1  ok nothing special charge diamond member hilto...       2   \n",
       "2  nice rooms not 4* experience hotel monaco seat...       3   \n",
       "3  unique, great stay, wonderful time hotel monac...       5   \n",
       "4  great stay great stay, went seahawk game aweso...       5   \n",
       "\n",
       "                                    review_lowercase  \\\n",
       "0  nice hotel expensive parking got good deal sta...   \n",
       "1  ok nothing special charge diamond member hilto...   \n",
       "2  nice rooms not 4* experience hotel monaco seat...   \n",
       "3  unique, great stay, wonderful time hotel monac...   \n",
       "4  great stay great stay, went seahawk game aweso...   \n",
       "\n",
       "                                 review_no_stopwords  \\\n",
       "0  nice hotel expensive parking got good deal sta...   \n",
       "1  ok nothing special charge diamond member hilto...   \n",
       "2  nice rooms not 4* experience hotel monaco seat...   \n",
       "3  unique, great stay, wonderful time hotel monac...   \n",
       "4  great stay great stay, went seahawk game aweso...   \n",
       "\n",
       "                        review_no_stopwords_no_punct  \n",
       "0  nice hotel expensive parking got good deal sta...  \n",
       "1  ok nothing special charge diamond member hilto...  \n",
       "2  nice rooms not 4star experience hotel monaco s...  \n",
       "3  unique great stay wonderful time hotel monaco ...  \n",
       "4  great stay great stay went seahawk game awesom...  "
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Tokenizing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "data['tokenized'] = data.apply(lambda x: \\\n",
    "                               word_tokenize( \\\n",
    "                               x['review_no_stopwords_no_punct'] \\\n",
    "                               ), axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Review</th>\n",
       "      <th>Rating</th>\n",
       "      <th>review_lowercase</th>\n",
       "      <th>review_no_stopwords</th>\n",
       "      <th>review_no_stopwords_no_punct</th>\n",
       "      <th>tokenized</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>nice hotel expensive parking got good deal sta...</td>\n",
       "      <td>4</td>\n",
       "      <td>nice hotel expensive parking got good deal sta...</td>\n",
       "      <td>nice hotel expensive parking got good deal sta...</td>\n",
       "      <td>nice hotel expensive parking got good deal sta...</td>\n",
       "      <td>[nice, hotel, expensive, parking, got, good, d...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ok nothing special charge diamond member hilto...</td>\n",
       "      <td>2</td>\n",
       "      <td>ok nothing special charge diamond member hilto...</td>\n",
       "      <td>ok nothing special charge diamond member hilto...</td>\n",
       "      <td>ok nothing special charge diamond member hilto...</td>\n",
       "      <td>[ok, nothing, special, charge, diamond, member...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>nice rooms not 4* experience hotel monaco seat...</td>\n",
       "      <td>3</td>\n",
       "      <td>nice rooms not 4* experience hotel monaco seat...</td>\n",
       "      <td>nice rooms not 4* experience hotel monaco seat...</td>\n",
       "      <td>nice rooms not 4star experience hotel monaco s...</td>\n",
       "      <td>[nice, rooms, not, 4star, experience, hotel, m...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>unique, great stay, wonderful time hotel monac...</td>\n",
       "      <td>5</td>\n",
       "      <td>unique, great stay, wonderful time hotel monac...</td>\n",
       "      <td>unique, great stay, wonderful time hotel monac...</td>\n",
       "      <td>unique great stay wonderful time hotel monaco ...</td>\n",
       "      <td>[unique, great, stay, wonderful, time, hotel, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>great stay great stay, went seahawk game aweso...</td>\n",
       "      <td>5</td>\n",
       "      <td>great stay great stay, went seahawk game aweso...</td>\n",
       "      <td>great stay great stay, went seahawk game aweso...</td>\n",
       "      <td>great stay great stay went seahawk game awesom...</td>\n",
       "      <td>[great, stay, great, stay, went, seahawk, game...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                              Review  Rating  \\\n",
       "0  nice hotel expensive parking got good deal sta...       4   \n",
       "1  ok nothing special charge diamond member hilto...       2   \n",
       "2  nice rooms not 4* experience hotel monaco seat...       3   \n",
       "3  unique, great stay, wonderful time hotel monac...       5   \n",
       "4  great stay great stay, went seahawk game aweso...       5   \n",
       "\n",
       "                                    review_lowercase  \\\n",
       "0  nice hotel expensive parking got good deal sta...   \n",
       "1  ok nothing special charge diamond member hilto...   \n",
       "2  nice rooms not 4* experience hotel monaco seat...   \n",
       "3  unique, great stay, wonderful time hotel monac...   \n",
       "4  great stay great stay, went seahawk game aweso...   \n",
       "\n",
       "                                 review_no_stopwords  \\\n",
       "0  nice hotel expensive parking got good deal sta...   \n",
       "1  ok nothing special charge diamond member hilto...   \n",
       "2  nice rooms not 4* experience hotel monaco seat...   \n",
       "3  unique, great stay, wonderful time hotel monac...   \n",
       "4  great stay great stay, went seahawk game aweso...   \n",
       "\n",
       "                        review_no_stopwords_no_punct  \\\n",
       "0  nice hotel expensive parking got good deal sta...   \n",
       "1  ok nothing special charge diamond member hilto...   \n",
       "2  nice rooms not 4star experience hotel monaco s...   \n",
       "3  unique great stay wonderful time hotel monaco ...   \n",
       "4  great stay great stay went seahawk game awesom...   \n",
       "\n",
       "                                           tokenized  \n",
       "0  [nice, hotel, expensive, parking, got, good, d...  \n",
       "1  [ok, nothing, special, charge, diamond, member...  \n",
       "2  [nice, rooms, not, 4star, experience, hotel, m...  \n",
       "3  [unique, great, stay, wonderful, time, hotel, ...  \n",
       "4  [great, stay, great, stay, went, seahawk, game...  "
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['nice',\n",
       " 'hotel',\n",
       " 'expensive',\n",
       " 'parking',\n",
       " 'got',\n",
       " 'good',\n",
       " 'deal',\n",
       " 'stay',\n",
       " 'hotel',\n",
       " 'anniversary',\n",
       " 'arrived',\n",
       " 'late',\n",
       " 'evening',\n",
       " 'took',\n",
       " 'advice',\n",
       " 'previous',\n",
       " 'reviews',\n",
       " 'valet',\n",
       " 'parking',\n",
       " 'check',\n",
       " 'quick',\n",
       " 'easy',\n",
       " 'little',\n",
       " 'disappointed',\n",
       " 'nonexistent',\n",
       " 'view',\n",
       " 'room',\n",
       " 'room',\n",
       " 'clean',\n",
       " 'nice',\n",
       " 'size',\n",
       " 'bed',\n",
       " 'comfortable',\n",
       " 'woke',\n",
       " 'stiff',\n",
       " 'neck',\n",
       " 'high',\n",
       " 'pillows',\n",
       " 'not',\n",
       " 'soundproof',\n",
       " 'like',\n",
       " 'heard',\n",
       " 'music',\n",
       " 'room',\n",
       " 'night',\n",
       " 'morning',\n",
       " 'loud',\n",
       " 'bangs',\n",
       " 'doors',\n",
       " 'opening',\n",
       " 'closing',\n",
       " 'hear',\n",
       " 'people',\n",
       " 'talking',\n",
       " 'hallway',\n",
       " 'maybe',\n",
       " 'noisy',\n",
       " 'neighbors',\n",
       " 'aveda',\n",
       " 'bath',\n",
       " 'products',\n",
       " 'nice',\n",
       " 'not',\n",
       " 'goldfish',\n",
       " 'stay',\n",
       " 'nice',\n",
       " 'touch',\n",
       " 'taken',\n",
       " 'advantage',\n",
       " 'staying',\n",
       " 'longer',\n",
       " 'location',\n",
       " 'great',\n",
       " 'walking',\n",
       " 'distance',\n",
       " 'shopping',\n",
       " 'overall',\n",
       " 'nice',\n",
       " 'experience',\n",
       " 'pay',\n",
       " '40',\n",
       " 'parking',\n",
       " 'night']"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data['tokenized'][0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Stemming"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "ps = PorterStemmer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "data[\"stemmed\"] = data[\"tokenized\"] \\\n",
    "                  .apply(lambda tokens: \\\n",
    "                  [ps.stem(token) \\\n",
    "                   for token in tokens])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Review</th>\n",
       "      <th>Rating</th>\n",
       "      <th>review_lowercase</th>\n",
       "      <th>review_no_stopwords</th>\n",
       "      <th>review_no_stopwords_no_punct</th>\n",
       "      <th>tokenized</th>\n",
       "      <th>stemmed</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>nice hotel expensive parking got good deal sta...</td>\n",
       "      <td>4</td>\n",
       "      <td>nice hotel expensive parking got good deal sta...</td>\n",
       "      <td>nice hotel expensive parking got good deal sta...</td>\n",
       "      <td>nice hotel expensive parking got good deal sta...</td>\n",
       "      <td>[nice, hotel, expensive, parking, got, good, d...</td>\n",
       "      <td>[nice, hotel, expens, park, got, good, deal, s...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ok nothing special charge diamond member hilto...</td>\n",
       "      <td>2</td>\n",
       "      <td>ok nothing special charge diamond member hilto...</td>\n",
       "      <td>ok nothing special charge diamond member hilto...</td>\n",
       "      <td>ok nothing special charge diamond member hilto...</td>\n",
       "      <td>[ok, nothing, special, charge, diamond, member...</td>\n",
       "      <td>[ok, noth, special, charg, diamond, member, hi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>nice rooms not 4* experience hotel monaco seat...</td>\n",
       "      <td>3</td>\n",
       "      <td>nice rooms not 4* experience hotel monaco seat...</td>\n",
       "      <td>nice rooms not 4* experience hotel monaco seat...</td>\n",
       "      <td>nice rooms not 4star experience hotel monaco s...</td>\n",
       "      <td>[nice, rooms, not, 4star, experience, hotel, m...</td>\n",
       "      <td>[nice, room, not, 4star, experi, hotel, monaco...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>unique, great stay, wonderful time hotel monac...</td>\n",
       "      <td>5</td>\n",
       "      <td>unique, great stay, wonderful time hotel monac...</td>\n",
       "      <td>unique, great stay, wonderful time hotel monac...</td>\n",
       "      <td>unique great stay wonderful time hotel monaco ...</td>\n",
       "      <td>[unique, great, stay, wonderful, time, hotel, ...</td>\n",
       "      <td>[uniqu, great, stay, wonder, time, hotel, mona...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>great stay great stay, went seahawk game aweso...</td>\n",
       "      <td>5</td>\n",
       "      <td>great stay great stay, went seahawk game aweso...</td>\n",
       "      <td>great stay great stay, went seahawk game aweso...</td>\n",
       "      <td>great stay great stay went seahawk game awesom...</td>\n",
       "      <td>[great, stay, great, stay, went, seahawk, game...</td>\n",
       "      <td>[great, stay, great, stay, went, seahawk, game...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                              Review  Rating  \\\n",
       "0  nice hotel expensive parking got good deal sta...       4   \n",
       "1  ok nothing special charge diamond member hilto...       2   \n",
       "2  nice rooms not 4* experience hotel monaco seat...       3   \n",
       "3  unique, great stay, wonderful time hotel monac...       5   \n",
       "4  great stay great stay, went seahawk game aweso...       5   \n",
       "\n",
       "                                    review_lowercase  \\\n",
       "0  nice hotel expensive parking got good deal sta...   \n",
       "1  ok nothing special charge diamond member hilto...   \n",
       "2  nice rooms not 4* experience hotel monaco seat...   \n",
       "3  unique, great stay, wonderful time hotel monac...   \n",
       "4  great stay great stay, went seahawk game aweso...   \n",
       "\n",
       "                                 review_no_stopwords  \\\n",
       "0  nice hotel expensive parking got good deal sta...   \n",
       "1  ok nothing special charge diamond member hilto...   \n",
       "2  nice rooms not 4* experience hotel monaco seat...   \n",
       "3  unique, great stay, wonderful time hotel monac...   \n",
       "4  great stay great stay, went seahawk game aweso...   \n",
       "\n",
       "                        review_no_stopwords_no_punct  \\\n",
       "0  nice hotel expensive parking got good deal sta...   \n",
       "1  ok nothing special charge diamond member hilto...   \n",
       "2  nice rooms not 4star experience hotel monaco s...   \n",
       "3  unique great stay wonderful time hotel monaco ...   \n",
       "4  great stay great stay went seahawk game awesom...   \n",
       "\n",
       "                                           tokenized  \\\n",
       "0  [nice, hotel, expensive, parking, got, good, d...   \n",
       "1  [ok, nothing, special, charge, diamond, member...   \n",
       "2  [nice, rooms, not, 4star, experience, hotel, m...   \n",
       "3  [unique, great, stay, wonderful, time, hotel, ...   \n",
       "4  [great, stay, great, stay, went, seahawk, game...   \n",
       "\n",
       "                                             stemmed  \n",
       "0  [nice, hotel, expens, park, got, good, deal, s...  \n",
       "1  [ok, noth, special, charg, diamond, member, hi...  \n",
       "2  [nice, room, not, 4star, experi, hotel, monaco...  \n",
       "3  [uniqu, great, stay, wonder, time, hotel, mona...  \n",
       "4  [great, stay, great, stay, went, seahawk, game...  "
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['nice',\n",
       " 'hotel',\n",
       " 'expens',\n",
       " 'park',\n",
       " 'got',\n",
       " 'good',\n",
       " 'deal',\n",
       " 'stay',\n",
       " 'hotel',\n",
       " 'anniversari',\n",
       " 'arriv',\n",
       " 'late',\n",
       " 'even',\n",
       " 'took',\n",
       " 'advic',\n",
       " 'previou',\n",
       " 'review',\n",
       " 'valet',\n",
       " 'park',\n",
       " 'check',\n",
       " 'quick',\n",
       " 'easi',\n",
       " 'littl',\n",
       " 'disappoint',\n",
       " 'nonexist',\n",
       " 'view',\n",
       " 'room',\n",
       " 'room',\n",
       " 'clean',\n",
       " 'nice',\n",
       " 'size',\n",
       " 'bed',\n",
       " 'comfort',\n",
       " 'woke',\n",
       " 'stiff',\n",
       " 'neck',\n",
       " 'high',\n",
       " 'pillow',\n",
       " 'not',\n",
       " 'soundproof',\n",
       " 'like',\n",
       " 'heard',\n",
       " 'music',\n",
       " 'room',\n",
       " 'night',\n",
       " 'morn',\n",
       " 'loud',\n",
       " 'bang',\n",
       " 'door',\n",
       " 'open',\n",
       " 'close',\n",
       " 'hear',\n",
       " 'peopl',\n",
       " 'talk',\n",
       " 'hallway',\n",
       " 'mayb',\n",
       " 'noisi',\n",
       " 'neighbor',\n",
       " 'aveda',\n",
       " 'bath',\n",
       " 'product',\n",
       " 'nice',\n",
       " 'not',\n",
       " 'goldfish',\n",
       " 'stay',\n",
       " 'nice',\n",
       " 'touch',\n",
       " 'taken',\n",
       " 'advantag',\n",
       " 'stay',\n",
       " 'longer',\n",
       " 'locat',\n",
       " 'great',\n",
       " 'walk',\n",
       " 'distanc',\n",
       " 'shop',\n",
       " 'overal',\n",
       " 'nice',\n",
       " 'experi',\n",
       " 'pay',\n",
       " '40',\n",
       " 'park',\n",
       " 'night']"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data['stemmed'][0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Lemmatization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "lemmatizer = WordNetLemmatizer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "data[\"lemmatized\"] = data[\"tokenized\"] \\\n",
    "                    .apply(lambda tokens: \\\n",
    "                    [lemmatizer.lemmatize(token) \\\n",
    "                     for token in tokens])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['nice',\n",
       " 'hotel',\n",
       " 'expensive',\n",
       " 'parking',\n",
       " 'got',\n",
       " 'good',\n",
       " 'deal',\n",
       " 'stay',\n",
       " 'hotel',\n",
       " 'anniversary',\n",
       " 'arrived',\n",
       " 'late',\n",
       " 'evening',\n",
       " 'took',\n",
       " 'advice',\n",
       " 'previous',\n",
       " 'review',\n",
       " 'valet',\n",
       " 'parking',\n",
       " 'check',\n",
       " 'quick',\n",
       " 'easy',\n",
       " 'little',\n",
       " 'disappointed',\n",
       " 'nonexistent',\n",
       " 'view',\n",
       " 'room',\n",
       " 'room',\n",
       " 'clean',\n",
       " 'nice',\n",
       " 'size',\n",
       " 'bed',\n",
       " 'comfortable',\n",
       " 'woke',\n",
       " 'stiff',\n",
       " 'neck',\n",
       " 'high',\n",
       " 'pillow',\n",
       " 'not',\n",
       " 'soundproof',\n",
       " 'like',\n",
       " 'heard',\n",
       " 'music',\n",
       " 'room',\n",
       " 'night',\n",
       " 'morning',\n",
       " 'loud',\n",
       " 'bang',\n",
       " 'door',\n",
       " 'opening',\n",
       " 'closing',\n",
       " 'hear',\n",
       " 'people',\n",
       " 'talking',\n",
       " 'hallway',\n",
       " 'maybe',\n",
       " 'noisy',\n",
       " 'neighbor',\n",
       " 'aveda',\n",
       " 'bath',\n",
       " 'product',\n",
       " 'nice',\n",
       " 'not',\n",
       " 'goldfish',\n",
       " 'stay',\n",
       " 'nice',\n",
       " 'touch',\n",
       " 'taken',\n",
       " 'advantage',\n",
       " 'staying',\n",
       " 'longer',\n",
       " 'location',\n",
       " 'great',\n",
       " 'walking',\n",
       " 'distance',\n",
       " 'shopping',\n",
       " 'overall',\n",
       " 'nice',\n",
       " 'experience',\n",
       " 'pay',\n",
       " '40',\n",
       " 'parking',\n",
       " 'night']"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data['lemmatized'][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Review</th>\n",
       "      <th>Rating</th>\n",
       "      <th>review_lowercase</th>\n",
       "      <th>review_no_stopwords</th>\n",
       "      <th>review_no_stopwords_no_punct</th>\n",
       "      <th>tokenized</th>\n",
       "      <th>stemmed</th>\n",
       "      <th>lemmatized</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>nice hotel expensive parking got good deal sta...</td>\n",
       "      <td>4</td>\n",
       "      <td>nice hotel expensive parking got good deal sta...</td>\n",
       "      <td>nice hotel expensive parking got good deal sta...</td>\n",
       "      <td>nice hotel expensive parking got good deal sta...</td>\n",
       "      <td>[nice, hotel, expensive, parking, got, good, d...</td>\n",
       "      <td>[nice, hotel, expens, park, got, good, deal, s...</td>\n",
       "      <td>[nice, hotel, expensive, parking, got, good, d...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ok nothing special charge diamond member hilto...</td>\n",
       "      <td>2</td>\n",
       "      <td>ok nothing special charge diamond member hilto...</td>\n",
       "      <td>ok nothing special charge diamond member hilto...</td>\n",
       "      <td>ok nothing special charge diamond member hilto...</td>\n",
       "      <td>[ok, nothing, special, charge, diamond, member...</td>\n",
       "      <td>[ok, noth, special, charg, diamond, member, hi...</td>\n",
       "      <td>[ok, nothing, special, charge, diamond, member...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>nice rooms not 4* experience hotel monaco seat...</td>\n",
       "      <td>3</td>\n",
       "      <td>nice rooms not 4* experience hotel monaco seat...</td>\n",
       "      <td>nice rooms not 4* experience hotel monaco seat...</td>\n",
       "      <td>nice rooms not 4star experience hotel monaco s...</td>\n",
       "      <td>[nice, rooms, not, 4star, experience, hotel, m...</td>\n",
       "      <td>[nice, room, not, 4star, experi, hotel, monaco...</td>\n",
       "      <td>[nice, room, not, 4star, experience, hotel, mo...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>unique, great stay, wonderful time hotel monac...</td>\n",
       "      <td>5</td>\n",
       "      <td>unique, great stay, wonderful time hotel monac...</td>\n",
       "      <td>unique, great stay, wonderful time hotel monac...</td>\n",
       "      <td>unique great stay wonderful time hotel monaco ...</td>\n",
       "      <td>[unique, great, stay, wonderful, time, hotel, ...</td>\n",
       "      <td>[uniqu, great, stay, wonder, time, hotel, mona...</td>\n",
       "      <td>[unique, great, stay, wonderful, time, hotel, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>great stay great stay, went seahawk game aweso...</td>\n",
       "      <td>5</td>\n",
       "      <td>great stay great stay, went seahawk game aweso...</td>\n",
       "      <td>great stay great stay, went seahawk game aweso...</td>\n",
       "      <td>great stay great stay went seahawk game awesom...</td>\n",
       "      <td>[great, stay, great, stay, went, seahawk, game...</td>\n",
       "      <td>[great, stay, great, stay, went, seahawk, game...</td>\n",
       "      <td>[great, stay, great, stay, went, seahawk, game...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                              Review  Rating  \\\n",
       "0  nice hotel expensive parking got good deal sta...       4   \n",
       "1  ok nothing special charge diamond member hilto...       2   \n",
       "2  nice rooms not 4* experience hotel monaco seat...       3   \n",
       "3  unique, great stay, wonderful time hotel monac...       5   \n",
       "4  great stay great stay, went seahawk game aweso...       5   \n",
       "\n",
       "                                    review_lowercase  \\\n",
       "0  nice hotel expensive parking got good deal sta...   \n",
       "1  ok nothing special charge diamond member hilto...   \n",
       "2  nice rooms not 4* experience hotel monaco seat...   \n",
       "3  unique, great stay, wonderful time hotel monac...   \n",
       "4  great stay great stay, went seahawk game aweso...   \n",
       "\n",
       "                                 review_no_stopwords  \\\n",
       "0  nice hotel expensive parking got good deal sta...   \n",
       "1  ok nothing special charge diamond member hilto...   \n",
       "2  nice rooms not 4* experience hotel monaco seat...   \n",
       "3  unique, great stay, wonderful time hotel monac...   \n",
       "4  great stay great stay, went seahawk game aweso...   \n",
       "\n",
       "                        review_no_stopwords_no_punct  \\\n",
       "0  nice hotel expensive parking got good deal sta...   \n",
       "1  ok nothing special charge diamond member hilto...   \n",
       "2  nice rooms not 4star experience hotel monaco s...   \n",
       "3  unique great stay wonderful time hotel monaco ...   \n",
       "4  great stay great stay went seahawk game awesom...   \n",
       "\n",
       "                                           tokenized  \\\n",
       "0  [nice, hotel, expensive, parking, got, good, d...   \n",
       "1  [ok, nothing, special, charge, diamond, member...   \n",
       "2  [nice, rooms, not, 4star, experience, hotel, m...   \n",
       "3  [unique, great, stay, wonderful, time, hotel, ...   \n",
       "4  [great, stay, great, stay, went, seahawk, game...   \n",
       "\n",
       "                                             stemmed  \\\n",
       "0  [nice, hotel, expens, park, got, good, deal, s...   \n",
       "1  [ok, noth, special, charg, diamond, member, hi...   \n",
       "2  [nice, room, not, 4star, experi, hotel, monaco...   \n",
       "3  [uniqu, great, stay, wonder, time, hotel, mona...   \n",
       "4  [great, stay, great, stay, went, seahawk, game...   \n",
       "\n",
       "                                          lemmatized  \n",
       "0  [nice, hotel, expensive, parking, got, good, d...  \n",
       "1  [ok, nothing, special, charge, diamond, member...  \n",
       "2  [nice, room, not, 4star, experience, hotel, mo...  \n",
       "3  [unique, great, stay, wonderful, time, hotel, ...  \n",
       "4  [great, stay, great, stay, went, seahawk, game...  "
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## N-grams"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokens_clean = sum(data['lemmatized'], [])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(hotel,)           292\n",
      "(room,)            275\n",
      "(great,)           126\n",
      "(not,)             122\n",
      "(stay,)             95\n",
      "                  ... \n",
      "(175,)               1\n",
      "(smackagainst,)      1\n",
      "(2x,)                1\n",
      "(80,)                1\n",
      "(connected,)         1\n",
      "Name: count, Length: 2589, dtype: int64\n"
     ]
    }
   ],
   "source": [
    "# unigrams: n=1\n",
    "unigrams = (pd.Series \\\n",
    "            (nltk.ngrams(tokens_clean, 1)) \\\n",
    "            .value_counts()) \n",
    "print(unigrams)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(great, location)     24\n",
      "(space, needle)       21\n",
      "(hotel, monaco)       16\n",
      "(great, hotel)        12\n",
      "(staff, friendly)     12\n",
      "                      ..\n",
      "(didnt, make)          1\n",
      "(personnel, didnt)     1\n",
      "(minute, stay)         1\n",
      "(starting, minute)     1\n",
      "(food, raffle)         1\n",
      "Name: count, Length: 8263, dtype: int64\n"
     ]
    }
   ],
   "source": [
    "# bigrams: n=2\n",
    "bigrams = (pd.Series \\\n",
    "           (nltk.ngrams(tokens_clean, 2)) \\\n",
    "           .value_counts()) \n",
    "print(bigrams)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(high, floor, great, view)                   2\n",
      "(definitely, stay, crowne, plaza)            2\n",
      "(needle, experience, music, project)         2\n",
      "(nice, hotel, husband, stayed)               2\n",
      "(really, comfortable, clean, location)       2\n",
      "                                            ..\n",
      "(unfortunately, warwick, seattle, hotel)     1\n",
      "(warwick, seattle, hotel, dissapointment)    1\n",
      "(seattle, hotel, dissapointment, trip)       1\n",
      "(hotel, dissapointment, trip, 3)             1\n",
      "(raffle, hotel, right, street)               1\n",
      "Name: count, Length: 9387, dtype: int64\n"
     ]
    }
   ],
   "source": [
    "ngrams_4 = (pd.Series \\\n",
    "            (nltk.ngrams(tokens_clean, 4)) \\\n",
    "            .value_counts()) \n",
    "print(ngrams_4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "nlp_course_env",
   "language": "python",
   "name": "nlp_course_env"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}