latent-manifold-regression_2.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Dataset\n",
    "# u.data - 100000 ratings by 943 users on 1682 items - user id | item id | rating | timestamp \n",
    "# u.item - Information about the items (movies) - movie id | movie title | release date | video release date |\n",
    "        # IMDb URL | unknown | Action | Adventure | Animation |Children's | Comedy | Crime | Documentary | Drama | Fantasy |\n",
    "        # Film-Noir | Horror | Musical | Mystery | Romance | Sci-Fi |Thriller | War | Western |\n",
    "# u.genre - A list of the genres.\n",
    "# u.user  - Demographic information about the users - user id | age | gender | occupation | zip code\n",
    "# u.occupation - A list of the occupations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd;\n",
    "import numpy as np;\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "#ignore python warning\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "#*****************Create panda data frame for u.data*************#"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(100000, 4)\n"
     ]
    }
   ],
   "source": [
    "rating_data = pd.read_csv('u.data', delim_whitespace=True, header=None)\n",
    "rating_data.columns = [\"user_id\", \"movie_id\", \"ratings\", \"timestamp\"]\n",
    "print(rating_data.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([1, 2, 3, 4, 5])"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.unique(rating_data['ratings'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "#*****************Create panda data frame for u.user*************#"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(943, 5)\n"
     ]
    }
   ],
   "source": [
    "user_data=pd.read_csv('u.user',sep='|',header=None)\n",
    "user_data.columns=['user_id','age','gender','occupation','zip_code']\n",
    "print(user_data.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "#*****************Merge u.data and u.user and create a new data frame*************#"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(100000, 8)"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "user_ratings_data=pd.merge(user_data,rating_data,on='user_id')\n",
    "user_ratings_data.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>age</th>\n",
       "      <th>gender</th>\n",
       "      <th>occupation</th>\n",
       "      <th>zip_code</th>\n",
       "      <th>movie_id</th>\n",
       "      <th>ratings</th>\n",
       "      <th>timestamp</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>24</td>\n",
       "      <td>M</td>\n",
       "      <td>technician</td>\n",
       "      <td>85711</td>\n",
       "      <td>61</td>\n",
       "      <td>4</td>\n",
       "      <td>878542420</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>24</td>\n",
       "      <td>M</td>\n",
       "      <td>technician</td>\n",
       "      <td>85711</td>\n",
       "      <td>189</td>\n",
       "      <td>3</td>\n",
       "      <td>888732928</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>24</td>\n",
       "      <td>M</td>\n",
       "      <td>technician</td>\n",
       "      <td>85711</td>\n",
       "      <td>33</td>\n",
       "      <td>4</td>\n",
       "      <td>878542699</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>24</td>\n",
       "      <td>M</td>\n",
       "      <td>technician</td>\n",
       "      <td>85711</td>\n",
       "      <td>160</td>\n",
       "      <td>4</td>\n",
       "      <td>875072547</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>24</td>\n",
       "      <td>M</td>\n",
       "      <td>technician</td>\n",
       "      <td>85711</td>\n",
       "      <td>20</td>\n",
       "      <td>4</td>\n",
       "      <td>887431883</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   user_id  age gender  occupation zip_code  movie_id  ratings  timestamp\n",
       "0        1   24      M  technician    85711        61        4  878542420\n",
       "1        1   24      M  technician    85711       189        3  888732928\n",
       "2        1   24      M  technician    85711        33        4  878542699\n",
       "3        1   24      M  technician    85711       160        4  875072547\n",
       "4        1   24      M  technician    85711        20        4  887431883"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "user_ratings_data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "#*****************Converting categorical values to numerical for user_ratings_data*************#"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>age</th>\n",
       "      <th>gender</th>\n",
       "      <th>occupation</th>\n",
       "      <th>zip_code</th>\n",
       "      <th>movie_id</th>\n",
       "      <th>ratings</th>\n",
       "      <th>timestamp</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>24</td>\n",
       "      <td>1</td>\n",
       "      <td>19</td>\n",
       "      <td>85711</td>\n",
       "      <td>61</td>\n",
       "      <td>4</td>\n",
       "      <td>878542420</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>24</td>\n",
       "      <td>1</td>\n",
       "      <td>19</td>\n",
       "      <td>85711</td>\n",
       "      <td>189</td>\n",
       "      <td>3</td>\n",
       "      <td>888732928</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>24</td>\n",
       "      <td>1</td>\n",
       "      <td>19</td>\n",
       "      <td>85711</td>\n",
       "      <td>33</td>\n",
       "      <td>4</td>\n",
       "      <td>878542699</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>24</td>\n",
       "      <td>1</td>\n",
       "      <td>19</td>\n",
       "      <td>85711</td>\n",
       "      <td>160</td>\n",
       "      <td>4</td>\n",
       "      <td>875072547</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>24</td>\n",
       "      <td>1</td>\n",
       "      <td>19</td>\n",
       "      <td>85711</td>\n",
       "      <td>20</td>\n",
       "      <td>4</td>\n",
       "      <td>887431883</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   user_id  age  gender  occupation zip_code  movie_id  ratings  timestamp\n",
       "0        1   24       1          19    85711        61        4  878542420\n",
       "1        1   24       1          19    85711       189        3  888732928\n",
       "2        1   24       1          19    85711        33        4  878542699\n",
       "3        1   24       1          19    85711       160        4  875072547\n",
       "4        1   24       1          19    85711        20        4  887431883"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Converting categorical values to numerical\n",
    "#user_ratings_date['ratings']=user_ratings_date['ratings'].map(lambda x: 1 if x>2 else 0)\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "le = LabelEncoder()\n",
    "user_ratings_data['gender'] = le.fit_transform(user_ratings_data['gender'])\n",
    "user_ratings_data['occupation'] = le.fit_transform(user_ratings_data['occupation'])\n",
    "user_ratings_data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 7, 10, 11, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26,\n",
       "       27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43,\n",
       "       44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60,\n",
       "       61, 62, 63, 64, 65, 66, 68, 69, 70, 73])"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.unique(user_ratings_data['age'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "#*****************Checking if there is any Linear Relationship between age and ratings*************#"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD8CAYAAACMwORRAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4wLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvqOYd8AAAGIdJREFUeJzt3X9sXeWd5/H3J45JkzjYDPGE2I6TjopmC+0MtFfQilXFkC5qKQ3SDNJQ0WlZdWUBnR26g1S1m1GjoI201S7ddkRLlZbugIgoW/pjKW1nh6aMOl0Junb4TdoddoDYcSCmSWyM05A43/3Dx6l9fX+ca1/7Xj/5vKSre885zznna/uej4+f81wfRQRmZpaWFY0uwMzM6s/hbmaWIIe7mVmCHO5mZglyuJuZJcjhbmaWIIe7mVmCHO5mZglyuJuZJWhlo3a8fv362LJlS6N2b2a2LA0MDLweEZ3V2jUs3Lds2UJ/f3+jdm9mtixJeiVPO3fLmJklyOFuZpYgh7uZWYIc7mZmCXK4m5klKFe4S3pZ0rOSnpI0Z4iLpvytpBclPSPpPfUv1czM8qplKOSfRMTrZZZ9GLgwe1wO3J0915V2as682FH9TlKl1rv/T+9n+97tHBg9QG97L7u27uLGd99YdT0zW1qrW1ZzfPJ42en56mrr4uDtB+fMX/Of1sxr+/PNojzrzUe9umWuA+6LKY8DHZI21mnbQPmgrRbA5ZZ//Hsf55XRVwiCV0Zfoe+Hfex5dk/u7ZrZ0igO2noEO8Dw+DDdd3bPmjffYIf5Z9FiZU3ecA/gHyQNSOorsbwbGJwxPZTNWzYmTk6wfe/2RpdhZktoeHx41nS9fnE0g7zdMldExLCk3wcelfSriPj5jOWlfvXM+Vsj+8XQB9Db21tzsYvtwOiBRpdgZlYXuc7cI2I4ez4MfB+4rKjJELBpxnQPMFzUhojYHRGFiCh0dlb91whLrre9+X7hmJnNR9Vwl7RW0rrp18DVwHNFzR4GPpGNmnkfMBoRh+pe7SJa07qGXVt3NboMM1tCXW1ds6ZXt6xuUCX1l+fMfQPwC0lPA78EfhQRfy/pZkk3Z21+DPwL8CLwDeDWehda7opytSvN5Zbf/6f3s7l9M0Jsbt/M7o/unjVaZrGuYJtZbYoDt14BXGq0zMTfTMx7+/PNosXKGkU0JsQKhUL4v0KamdVG0kBEFKq18ydUzcwS5HA3M0uQw93MLEEOdzOzBDnczcwS5HA3M0uQw93MLEEOdzOzBDnczcwS5HA3M0uQw93MLEEOdzOzBDnczcwS5HA3M0uQw93MLEEOdzOzBOUOd0ktkp6U9EiJZTdJGpH0VPb4d/Ut08zMarGyhra3AfuBc8ssfzAi/nLhJZmZ2ULlOnOX1AN8BPjm4pZjZmb1kLdb5svAZ4HTFdr8maRnJD0kaVOpBpL6JPVL6h8ZGam1VjMzy6lquEu6FjgcEQMVmv0Q2BIRfwT8FLi3VKOI2B0RhYgodHZ2zqtgMzOrLs+Z+xXANkkvA98GrpJ0/8wGEfGbiDiRTX4DeG9dqzQzs5pUDfeI+HxE9ETEFuAG4GcR8fGZbSRtnDG5jakLr2Zm1iC1jJaZRdIdQH9EPAz8laRtwCngCHBTfcozM7P5UEQ0ZMeFQiH6+/sbsm8zs+VK0kBEFKq18ydUzcwS5HA3M0uQw93MLEEOdzOzBDnczcwS5HA3M0uQw93MLEEOdzOzBDnczcwS5HA3M0uQw93MLEEOdzOzBDnczcwS5HA3M0uQw93MLEG5w11Si6QnJT1SYtkqSQ9KelHSE5K21LNIMzOrTS13YrqNqdvnnVti2aeAoxHxDkk3AF8E/rwO9c2inZozL3ZUv9lI953dDI8Pn5nuauti5M0RTsbJM/Na1cpbX3ir6v7MLA0rWMHkjsk58+d73OfJovlm2HzkOnOX1AN8BPhmmSbXAfdmrx8CtkqqazKW+4ZX+0EUBzvA8PjwrGAHOBknOeeOc3Jv18yWt9OcpmVny6x5Cznuq6073wybr7zdMl8GPgucLrO8GxgEiIhTwChw/oKrq4PiYK+kOPDNLG2ny0ba8lc13CVdCxyOiIFKzUrMm/O3hqQ+Sf2S+kdGRmoo08zMapHnzP0KYJukl4FvA1dJur+ozRCwCUDSSqAdOFK8oYjYHRGFiCh0dnYuqHAzMyuvarhHxOcjoicitgA3AD+LiI8XNXsY+GT2+vqszeJcJahRV1tX7ratal3ESsys2axIeDT4vL8ySXdI2pZN3gOcL+lF4K+Bz9WjuJnKXVGudqX54O0H5wR8V1vXnCAvHi2zWFewzaw5lBots5Djvtq6882w+VKjTrALhUL09/c3ZN9mZsuVpIGIKFRrl+7fJGZmZzGHu5lZghzuZmYJcribmSXI4W5mliCHu5lZghzuZmZLZHx8nKGhoSXZVy3/8tfM7Kz01ltvcfDgQQYHB+c8hoaGGBwc5PXXX8+9ve985ztcf/31i1ixw93MEnb69Glee+21iqG8VGfSAGvXruXyyy/n6quvXvR9OdzNrOlEBMeOHasYyoODg5w4cWJJ6mlpaWHTpk1s2rSJnp6eM69nPtavX0+db2OxIA53M6uriYmJWQE88/X0Y3R0dMnq2bBhQ8kwnn5s3LiRlSvTi8L0viIzm5dTp04xPDxcMZRfe+21Jaunvb29Yih3d3ezZs2aJatnuXG4my1zEcHIyEjJMJ75OH16ae46tGrVqrKBPN2l0dHR0VRdGClyuJs10BtvvFExkAcHB5mYmFiyeqbDt1y/8oYNG1ixwiOolwOHu9k8nDhxouzQuOnHkSNzbka2aM4///yKXRhdXV2sWrVqyeqxxnO421llcnKy7NC46X7mgwcPLlk9a9eurRjKPT09rFu3bsnqsXRUDXdJbwN+DqzK2j8UETuK2twE/Bdg+qi4KyK+Wd9S7WwWERw9erRiKA8ODvLWW29V31gdzBwaV65fudmGxtnZJc+Z+wngqogYl9QK/ELSTyLi8aJ2D0bEX9a/RFvuJiYmKo5VHhwcZGxsbMnqueCCC0qGcepD4+zsUvUdnN3oejybbM0evsHoWeDkyZOzhsaVCuXDhw8vWT0dHR0VL/b19PSwevXqJavHrJnlOj2R1AIMAO8AvhoRT5Ro9meSPgD8X+A/RMRg/cq0WkQEb775JmNjY4yNjTE6OsrY2Bj33XcfL7300plgXqr751YaGjf9aG9vdxeGWR3lCveImAQukdQBfF/SuyLiuRlNfgg8EBEnJN0M3AtcVbwdSX1AH0Bvb++Ci0/R6OhoxTPlwcFBjh8/Xnb9jo4OxsbG6jamudJZsofGmTUv1Xr2JmkH8GZE/Ncyy1uAIxHRXmk7hUIh+vv7a9p3M/vtb3/L0NBQxQ+SHD16dFFr2LRpE9dddx3t7e2ce+65c54PHTp05qPY3d3dnHPOOYtaj5nVn6SBiChUa5dntEwncDIijklaDXwQ+GJRm40RcSib3Absn0fNDTE5Ocmrr75acbzyoUOHqm+oTjw0zszqIU+3zEbg3uyMfAXwPyLiEUl3AP0R8TDwV5K2AaeAI8BNi1XwtIjgyJEjFUN5aGiIkydPLnYpgIfGmVlzqblbpl7m2y1z4MABNm/eXNdaPDTOzJaLunXLNJvzzjuPSy+9lCeffPLM0LhyoeyhcWZ2tlp24b5u3Tr27dvX6DLMzJqax7CZmSXI4W5mliCHu5lZghzuZmYJcribmSXI4W5mliCHu5lZghzuZmYJcribmSXI4W5mliCHu5lZghzuZmYJcribmSWoarhLepukX0p6WtLzknaWaLNK0oOSXpT0hKQti1GsmZnlk+df/p4AroqIcUmtwC8k/SQiHp/R5lPA0Yh4h6QbmLoN35/Xu9iWnS2c5nc3fl7BCiZ3TFZdr/vObobHh89Md7V18er4q1W3pZ2+a5JZymLH3JsVzfe4L7WtPNvOs958VD1zjynj2WRr9iiu5jrg3uz1Q8BW1fl+csXBDnCa07TsbKm4XnGwAwyPD1fdloPdLH3Fx/lCjvtq65ZbvlhZk6vPXVKLpKeAw8CjEfFEUZNuYBAgIk4Bo8D59Sy0OIyrzZ9WHOzz2YeZ2XKTK9wjYjIiLgF6gMskvauoSalfPXP+1pDUJ6lfUv/IyEjt1ZqZWS41jZaJiGPAPwIfKlo0BGwCkLQSaAeOlFh/d0QUIqLQ2dk5r4LNzKy6PKNlOiV1ZK9XAx8EflXU7GHgk9nr64GfRURdrxKsKFNqufnTutq6FrwPM7PlJk+abQQek/QM8H+Y6nN/RNIdkrZlbe4Bzpf0IvDXwOfqXejkjsk54ZtntMzB2w/OCfiutq6q21qsK9hm1jyKj/OFHPfV1i23fLGyRnU+wc6tUChEf39/Q/ZtZrZcSRqIiEK1du6HMDNLkMPdzCxBDnczswQ53M3MEuRwNzNLkMPdzCxBDnczswQ53M3MEuRwNzNLkMPdzCxBDnczswQ53M3MEuRwNzNLkMPdzCxBDnczswQ53M3MEpTnNnubJD0mab+k5yXdVqLNlZJGJT2VPb6wOOWamVkeK3O0OQXcHhH7JK0DBiQ9GhEvFLX7p4i4tv4lmplZraqeuUfEoYjYl71+A9gPdC92YWZmNn819blL2gJcCjxRYvH7JT0t6SeSLq5DbWZmNk95umUAkNQGfBf4TESMFS3eB2yOiHFJ1wA/AC4ssY0+oA+gt7d33kWbmVlluc7cJbUyFex7IuJ7xcsjYiwixrPXPwZaJa0v0W53RBQiotDZ2bnA0s3MrJw8o2UE3APsj4gvlWlzQdYOSZdl2/1NPQs1M7P88nTLXAH8BfCspKeyef8R6AWIiK8D1wO3SDoFHAduiIhYhHrNzCyHquEeEb8AVKXNXcBd9SrKzMwWxp9QNTNLkMPdzCxBDnczswQ53M3MEuRwNzNLkMPdzCxBDnczswQ53M3MEuRwNzNLkMPdzCxBDnczswQ53M3MEuRwNzNLkMPdzCxBDnczswTluRPTJkmPSdov6XlJt5VoI0l/K+lFSc9Ies/ilGtmZnnkuRPTKeD2iNgnaR0wIOnRiHhhRpsPM3VD7AuBy4G7s+e6uvirF/PC67/b7UXrL+L5Tz9fdb3uO7sZHh8+M93V1sXEyQmOnTh2Zl7Hqg6Ofu7orPW0s+I9SsxsATpWdcw6Bsvpauuac/wePX6U45PHy66zghWc5nTV/Rcf8zD/4z52NNfN56qeuUfEoYjYl71+A9gPdBc1uw64L6Y8DnRI2ljPQouDHeCF11/g4q9eXHG94mAHGB4fnvOmOnbiGOf95/POTDvYzRZXnmAHSh6/lYIdqBrs0/ufeczDwo77ZsuMmvrcJW0BLgWeKFrUDQzOmB5i7i+ABSkO9mrzpxW/MSrJ+2YzszSkfMznDndJbcB3gc9ExFjx4hKrzPkbRVKfpH5J/SMjI7VVamZmueUKd0mtTAX7noj4XokmQ8CmGdM9wJxT5ojYHRGFiCh0dnbOp14zM8shz2gZAfcA+yPiS2WaPQx8Ihs18z5gNCIO1bFOLlp/UU3zp3W1deXeR8eqjppqMrPlLeVjPs+Z+xXAXwBXSXoqe1wj6WZJN2dtfgz8C/Ai8A3g1noX+vynn58T5HlGyxy8/eCcgO9q65rzQy2+ct5sV77NUpM3WEsdv6tbVldcZ0WOaCs1WmYhx32zZYYiGlNQoVCI/v7+huzbzGy5kjQQEYVq7fwJVTOzBDnczcwS5HA3M0uQw93MLEEOdzOzBDnczcwS5HA3M0uQw93MLEEOdzOzBDnczcwS5HA3M0uQw93MLEEOdzOzBDnczcwS5HA3M0uQw93MLEF5brP3LUmHJT1XZvmVkkZn3KXpC/Uv08zMarEyR5u/A+4C7qvQ5p8i4tq6VGRmZgtW9cw9In4OHFmCWszMrE7q1ef+fklPS/qJpIvrtE0zM5unPN0y1ewDNkfEuKRrgB8AF5ZqKKkP6APo7e2tw67NzKyUBZ+5R8RYRIxnr38MtEpaX6bt7ogoREShs7Nzobs2M7MyFhzuki6QpOz1Zdk2f7PQ7ZqZ2fxV7ZaR9ABwJbBe0hCwA2gFiIivA9cDt0g6BRwHboiIWLSKzcysqqrhHhEfq7L8LqaGSpqZWZPwJ1TNzBLkcDczS5DD3cwsQQ53M7MEOdzNzBLkcDczS5DD3cwsQQ53M7MEOdzNzBLkcDczS5DD3cwsQQ53M7MEOdzNzBLkcDczS5DD3cwsQQ53M7ME5bkT07eAa4HDEfGuEssFfAW4BpgAboqIffUuFGDPs3vYvnc7B0YP0Nvey66tu7jx3TdWXa/7zm6Gx4fPTHe1dfHOzney96W9Z+ZtfftWfvqJn85aTztVv+LNzjK3FG5h98BuJmOSFrWwYe2GWcfh1rdvnXUMltKqVjrXds45foFZ84p1tXUx8uYIJ+Nk2TaljnmAW39066y6z1lxDscnj8/a9tHjR2fNW92ymom/maj4tZTadt97+/jaR75Wdb35ULU74kn6ADAO3Fcm3K8B/j1T4X458JWIuLzajguFQvT39+cudM+ze+j7YR8TJ3/3DVzTuobdH91dMeCLg72SmT9sB7tZ+ooD/tYf3crd/XfPa1vVAr7ctm8p3FJTwEsaiIhCtXZVu2Ui4ufAkQpNrmMq+CMiHgc6JG3MXWlO2/dunxXsABMnJ9i+d3vF9fIGO1D1LMLM0lJ8zO8e2D3vbc08ky+l3LYXss9K6tHn3g0MzpgeyubNIalPUr+k/pGRkZp2cmD0QE3zzcxqNRmTS77txdpnPcK9VP9Fyb6eiNgdEYWIKHR2dta0k9723prmm5nVqkUtS77txdpnPcJ9CNg0Y7oHyN8XktOurbtY07pm1rw1rWvYtXVXxfWmL77ksfXtW+dVm5ktT8XHfN97++a9rdUtqysuL7ftheyzknqE+8PAJzTlfcBoRByqw3ZnufHdN7L7o7vZ3L4ZITa3b656MRXg4O0H5wR8V1vXnB9q8YWV2FH5QrOZVXZL4ZYzZ6UtaplzHOY5mWpVa8njt9pJW1dbF61qrdim1GiZr33ka3PqLg7trrauOfPyjJYpte1aL6bWIs9omQeAK4H1wGvADqAVICK+ng2FvAv4EFNDIf9tRFQdBlPraBkzM8s/WqbqOPeI+FiV5QF8uobazMxskfkTqmZmCXK4m5klyOFuZpYgh7uZWYIc7mZmCao6FHLRdiyNAK8s8W7XA68v8T7rwXUvveVau+teWo2oe3NEVP2If8PCvREk9ecZH9psXPfSW661u+6l1cx1u1vGzCxBDnczswSdbeG+OP84efG57qW3XGt33Uuraes+q/rczczOFmfbmbuZ2Vkh2XCX9C1JhyU9N2Pe70l6VNI/Z8/nNbLGUiRtkvSYpP2Snpd0Wza/qWuX9DZJv5T0dFb3zmz+2yU9kdX9oKRzGl1rKZJaJD0p6ZFsuunrlvSypGclPSWpP5vX1O8TAEkdkh6S9Kvsff7+Zq9b0h9m3+fpx5ikzzRz3cmGO/B3TP0b4pk+B+yNiAuBvdl0szkF3B4R7wTeB3xa0kU0f+0ngKsi4o+BS4APZf/f/4vAf8vqPgp8qoE1VnIbsH/G9HKp+08i4pIZw/Ga/X0C8BXg7yPiXwF/zNT3vanrjohfZ9/nS4D3MvXvzb9PM9cdEck+gC3AczOmfw1szF5vBH7d6BpzfA3/E/g3y6l2YA2wD7icqQ94rMzmvx/4X42ur0S9PUwdmFcBjzB168jlUPfLwPqieU39PgHOBV4iu963XOouqvVq4H83e90pn7mXsiGyu0Rlz7/f4HoqkrQFuBR4gmVQe9a18RRwGHgU+H/AsYg4lTUpe/P0Bvsy8FngdDZ9Psuj7gD+QdKApOl7tTX7++QPgBHgv2fdYN+UtJbmr3umG4AHstdNW/fZFu7LhqQ24LvAZyJirNH15BERkzH1Z2sPcBnwzlLNlraqyiRdCxyOiIGZs0s0baq6M1dExHuADzPVffeBRheUw0rgPcDdEXEp8CbN1JVRRXbtZRvwnUbXUs3ZFu6vSdoIkD0fbnA9JUlqZSrY90TE97LZy6J2gIg4BvwjU9cMOiRN3/FrUW6evkBXANskvQx8m6mumS/T/HUTEcPZ82Gm+n8vo/nfJ0PAUEQ8kU0/xFTYN3vd0z4M7IuI17Lppq37bAv3h4FPZq8/yVR/dlPJ7kl7D7A/Ir40Y1FT1y6pU1JH9no18EGmLpQ9BlyfNWu6uiPi8xHRExFbmPpz+2cRcSNNXrektZLWTb9mqh/4OZr8fRIRrwKDkv4wm7UVeIEmr3uGj/G7Lhlo5rob3em/iBc9HgAOASeZOlv4FFN9qXuBf86ef6/RdZao+18z1QXwDPBU9rim2WsH/gh4Mqv7OeAL2fw/AH4JvMjUn7KrGl1rha/hSuCR5VB3Vt/T2eN5YHs2v6nfJ1mNlwD92XvlB8B5y6TuNcBvgPYZ85q2bn9C1cwsQWdbt4yZ2VnB4W5mliCHu5lZghzuZmYJcribmSXI4W5mliCHu5lZghzuZmYJ+v8pNLoeYyCLHQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from sklearn import linear_model\n",
    "from matplotlib import pyplot as plt\n",
    "linear_reg = linear_model.LinearRegression() \n",
    "age_data=pd.DataFrame(user_ratings_data['age'])\n",
    "ratings_data=user_ratings_data['ratings']\n",
    "ratings_data.head()\n",
    "linear_reg.fit(age_data, ratings_data)\n",
    "plt.scatter(age_data, ratings_data,color='g')\n",
    "plt.plot(age_data, linear_reg.predict(age_data),color='k')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "#*******************MODELS FOR PREDICTING USER RATING using user_ratings_data**********************#"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "#*****************DecisionTree Classification to predict rating*************#"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.tree import DecisionTreeClassifier\n",
    "import numpy as np\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "X=user_ratings_data[['age','gender','occupation']].values;\n",
    "y=user_ratings_data[['ratings']].values;\n",
    "dtree_model = DecisionTreeClassifier(max_depth = 5).fit(X, y)\n",
    "dtree_predictions = dtree_model.predict(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([1, 3, 4, 5])"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.unique(dtree_predictions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.35203\n"
     ]
    }
   ],
   "source": [
    "accuracy = dtree_model.score(X, y)\n",
    "print(accuracy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Naive bayes Classification to predict rating\n",
    "from sklearn.naive_bayes import GaussianNB\n",
    "gnb = GaussianNB().fit(X, y.ravel())\n",
    "gnb_predictions = gnb.predict(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([4])"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.unique(gnb_predictions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.34174\n"
     ]
    }
   ],
   "source": [
    "# accuracy on X_test\n",
    "accuracy = gnb.score(X, y)\n",
    "print(accuracy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "#*******************ADDING MOVIE DATA**********************#"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "#*************************CURATING u.item data************************#"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "#************ Creating panda dataframe for u.item *********************#"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>movie_id</th>\n",
       "      <th>movie_title</th>\n",
       "      <th>release_date</th>\n",
       "      <th>video_release_date</th>\n",
       "      <th>imdb_url</th>\n",
       "      <th>unknown</th>\n",
       "      <th>Action</th>\n",
       "      <th>Adventure</th>\n",
       "      <th>Animation</th>\n",
       "      <th>Children</th>\n",
       "      <th>...</th>\n",
       "      <th>Fantasy</th>\n",
       "      <th>Film-Noir</th>\n",
       "      <th>Horror</th>\n",
       "      <th>Musical</th>\n",
       "      <th>Mystery</th>\n",
       "      <th>Romance</th>\n",
       "      <th>Sci-Fi</th>\n",
       "      <th>Thriller</th>\n",
       "      <th>War</th>\n",
       "      <th>Western</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>Toy Story (1995)</td>\n",
       "      <td>01-Jan-1995</td>\n",
       "      <td>NaN</td>\n",
       "      <td>http://us.imdb.com/M/title-exact?Toy%20Story%2...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>GoldenEye (1995)</td>\n",
       "      <td>01-Jan-1995</td>\n",
       "      <td>NaN</td>\n",
       "      <td>http://us.imdb.com/M/title-exact?GoldenEye%20(...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>Four Rooms (1995)</td>\n",
       "      <td>01-Jan-1995</td>\n",
       "      <td>NaN</td>\n",
       "      <td>http://us.imdb.com/M/title-exact?Four%20Rooms%...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>Get Shorty (1995)</td>\n",
       "      <td>01-Jan-1995</td>\n",
       "      <td>NaN</td>\n",
       "      <td>http://us.imdb.com/M/title-exact?Get%20Shorty%...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>Copycat (1995)</td>\n",
       "      <td>01-Jan-1995</td>\n",
       "      <td>NaN</td>\n",
       "      <td>http://us.imdb.com/M/title-exact?Copycat%20(1995)</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 24 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   movie_id        movie_title release_date  video_release_date  \\\n",
       "0         1   Toy Story (1995)  01-Jan-1995                 NaN   \n",
       "1         2   GoldenEye (1995)  01-Jan-1995                 NaN   \n",
       "2         3  Four Rooms (1995)  01-Jan-1995                 NaN   \n",
       "3         4  Get Shorty (1995)  01-Jan-1995                 NaN   \n",
       "4         5     Copycat (1995)  01-Jan-1995                 NaN   \n",
       "\n",
       "                                            imdb_url  unknown  Action  \\\n",
       "0  http://us.imdb.com/M/title-exact?Toy%20Story%2...        0       0   \n",
       "1  http://us.imdb.com/M/title-exact?GoldenEye%20(...        0       1   \n",
       "2  http://us.imdb.com/M/title-exact?Four%20Rooms%...        0       0   \n",
       "3  http://us.imdb.com/M/title-exact?Get%20Shorty%...        0       1   \n",
       "4  http://us.imdb.com/M/title-exact?Copycat%20(1995)        0       0   \n",
       "\n",
       "   Adventure  Animation  Children   ...     Fantasy  Film-Noir  Horror  \\\n",
       "0          0          1         1   ...           0          0       0   \n",
       "1          1          0         0   ...           0          0       0   \n",
       "2          0          0         0   ...           0          0       0   \n",
       "3          0          0         0   ...           0          0       0   \n",
       "4          0          0         0   ...           0          0       0   \n",
       "\n",
       "   Musical  Mystery  Romance  Sci-Fi  Thriller  War  Western  \n",
       "0        0        0        0       0         0    0        0  \n",
       "1        0        0        0       0         1    0        0  \n",
       "2        0        0        0       0         1    0        0  \n",
       "3        0        0        0       0         0    0        0  \n",
       "4        0        0        0       0         1    0        0  \n",
       "\n",
       "[5 rows x 24 columns]"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "movie_data=pd.read_csv('u.item',sep='|',header=None,encoding='latin-1')\n",
    "#movie_data=movie_data.iloc[:,0:6]\n",
    "movie_data.columns=['movie_id','movie_title','release_date','video_release_date','imdb_url','unknown','Action','Adventure','Animation','Children','Comedy','Crime','Documentary','Drama','Fantasy','Film-Noir','Horror','Musical','Mystery','Romance','Sci-Fi','Thriller','War','Western']\n",
    "movie_data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "#**********function to bucket movie years, create new feature movie decade*********************#"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2020\n"
     ]
    }
   ],
   "source": [
    "def movie_bucket(year):\n",
    "    year=int(year);\n",
    "    if (year<1970):\n",
    "      return 1960;\n",
    "    elif (year>=1970 and year<=1980):\n",
    "      return 1980;\n",
    "    elif (year>=1980 and year<=1990):\n",
    "      return 1990;\n",
    "    elif (year>=1990 and year<=2000):\n",
    "      return 2000;\n",
    "    elif (year>=2000 and year<=2010):\n",
    "      return 2010;\n",
    "    elif (year>2010):\n",
    "      return 2020\n",
    "\n",
    "# validating the function\n",
    "str1=\"01-Jan-2011\"\n",
    "print(movie_bucket(int(str1.split('-')[2])));"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "#**********Adding two new features movie_year and movie_decade to movie_data*********************#"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "movie_data.dropna(subset=[\"release_date\"],inplace= True)\n",
    "movie_data['movie_year']=movie_data['release_date'].str.split('-').str.get(2);\n",
    "movie_data['movie_decade']=movie_data['movie_year'].apply(movie_bucket);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1681, 26)"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "movie_data.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "#**********DATA ENRICHMENT USING IMDB DATA*********************#"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Content Rating is not part of the data set , we need this in order to make sure we are recommending appropriate\n",
    "# rated movies to the appropriate age group of users"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "#**********Getting content rating from IMDB*********************#"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>movie_title</th>\n",
       "      <th>content_rating</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [movie_title, content_rating]\n",
       "Index: []"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "imdb_data=pd.read_csv('imdb_content_rating.csv')\n",
    "# If content_rating is missing , adding default values as R (MOST RESTRICTED)\n",
    "imdb_data['content_rating'].fillna('R',inplace=True)\n",
    "imdb_data[imdb_data['content_rating'].isnull()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(5043, 2)"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "imdb_data.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "#**********Merging movie data set and IMDB data set*********************#"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1657"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "movie_data['movie_title']=movie_data['movie_title'].map(lambda x:x.split(\"(\")[0].rstrip())\n",
    "np.size(np.unique(movie_data['movie_title']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "# merge u.item and imdb_cr to create a new data frame\n",
    "movies_imdb_data=pd.merge(movie_data,imdb_data,on='movie_title',how='left')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1657"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.size(np.unique(movies_imdb_data['movie_title']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "#**********Merging user_ratings_data and movies_imdb_data********************#"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "# merge u.user, u.data, u.item and create a new data frame\n",
    "user_movies_data=pd.merge(user_ratings_data,movies_imdb_data,on='movie_id',how='left')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "user_movies_data['content_rating'].fillna(\"R\",inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['Approved', 'G', 'GP', 'M', 'NC-17', 'Not Rated', 'PG', 'PG-13',\n",
       "       'R', 'Unrated', 'X'], dtype=object)"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.unique(user_movies_data['content_rating'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([1, 2, 3, 4, 5])"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.unique(user_movies_data['ratings'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Label encoding content_rating\n",
    "user_movies_data['content_rating']=le.fit_transform(user_movies_data['content_rating'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 0,  1,  2,  3,  4,  5,  6,  7,  8,  9, 10])"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.unique(user_movies_data['content_rating'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([1, 2, 3, 4, 5])"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.unique(user_movies_data['ratings'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Series([], Name: movie_year, dtype: object)\n"
     ]
    }
   ],
   "source": [
    "user_movies_data.dropna(subset=['movie_year'],inplace=True)\n",
    "print(user_movies_data[user_movies_data['movie_year'].isna()]['movie_year'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "#*******************MODELS FOR PREDICTING USER RATING using user_movies_data**********************#"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "#**************Creating Training and Testing Set********************************#"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "X=user_movies_data[['age','gender','occupation','movie_year','content_rating','unknown','Action','Adventure','Animation','Children','Comedy','Crime','Documentary','Drama','Fantasy','Film-Noir','Horror','Musical','Mystery','Romance','Sci-Fi','Thriller','War','Western']].values;\n",
    "y=user_movies_data[['ratings']].values;"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Lets do an split for training and testing\n",
    "from sklearn.model_selection import train_test_split\n",
    "X_train,X_test,y_train,y_test=train_test_split(X, y,test_size = .3, random_state = 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "#**************Naive Bayes********************************#"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "gnb = GaussianNB().fit(X_train, y_train)\n",
    "gnb_predictions = gnb.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([1, 2, 3, 4, 5])"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.unique(gnb_predictions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.2954694897422409\n"
     ]
    }
   ],
   "source": [
    "# accuracy on X_test\n",
    "accuracy = gnb.score(X_test, y_test)\n",
    "print(accuracy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "#**************Logistics Regression********************************#"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Running LogisticRegression on new data\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "logisticRegr = LogisticRegression()\n",
    "logisticRegr.fit(X_train, y_train)\n",
    "lr_predictions = logisticRegr.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.34274723829563386\n"
     ]
    }
   ],
   "source": [
    "# accuracy\n",
    "accuracy = logisticRegr.score(X_test, y_test)\n",
    "print(accuracy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "#**********************Binary Classification to Predict liked***************************#"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1\n"
     ]
    }
   ],
   "source": [
    "# function to create a new feature liked based on user rating\n",
    "# user rating > 3, user liked the movie\n",
    "# user rating <= 3, user did not like the movie\n",
    "def like_identification(rating):\n",
    "        if(rating>3):\n",
    "            liked=1\n",
    "        else:\n",
    "            liked=0  \n",
    "        return liked;    \n",
    "print(like_identification(4))  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0, 1])"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Apply like_identification to new data set\n",
    "user_movies_data['liked']=user_movies_data['ratings'].map(lambda x: like_identification(x))\n",
    "np.unique(user_movies_data['liked'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 239,
   "metadata": {},
   "outputs": [],
   "source": [
    "y=user_movies_data[['liked']].values;\n",
    "X_train,X_test,y_train,y_test=train_test_split(X, y,test_size = .7, random_state = 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 240,
   "metadata": {},
   "outputs": [],
   "source": [
    "#**************Logistics Regression********************************#"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 241,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Running LogisticRegression to predict binary target\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "logisticRegr = LogisticRegression()\n",
    "y=user_movies_data[['liked']];\n",
    "logisticRegr.fit(X_train, y_train)\n",
    "lr_predictions = logisticRegr.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 242,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.5791097772329543\n"
     ]
    }
   ],
   "source": [
    "# accuracy\n",
    "accuracy = logisticRegr.score(X_test, y_test)\n",
    "print(accuracy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 243,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1 1 0 1 1 1 0 0 1 0 1 1 1 1 1 0 1 1 1 1 0 1 1 1 1 1 1 0 1 0 1 1 1 0 0 1 0\n",
      " 1 1 1 1 1 1 1 0 0 1 0 1 1 1 0 1 1 1 0 0 0 1 0 1 1 1 1 0 0 1 0 1 0 1 0 0 1\n",
      " 1 1 1 1 0 0 0 0 1 0 1 0 0 0 0 0 0 0 1 1 0 0 1 1 0 1]\n"
     ]
    }
   ],
   "source": [
    "print(logisticRegr.predict(user_movies_data[['age','gender','occupation','movie_year','content_rating','unknown','Action','Adventure','Animation','Children','Comedy','Crime','Documentary','Drama','Fantasy','Film-Noir','Horror','Musical','Mystery','Romance','Sci-Fi','Thriller','War','Western']].iloc[0:100]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 244,
   "metadata": {},
   "outputs": [],
   "source": [
    "#**************Naive Bayes********************************#"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 245,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Running Naive beyes to predict binary target\n",
    "gnb = GaussianNB().fit(X_train, y_train)\n",
    "gnb_predictions = gnb.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 246,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0, 1])"
      ]
     },
     "execution_count": 246,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.unique(gnb_predictions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 247,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.5675698524749545\n"
     ]
    }
   ],
   "source": [
    "# accuracy on X_test\n",
    "accuracy = gnb.score(X_test, y_test)\n",
    "print(accuracy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 248,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.5645898405392076\n"
     ]
    }
   ],
   "source": [
    "# accuracy on X_train\n",
    "accuracy = gnb.score(X_train, y_train)\n",
    "print(accuracy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 249,
   "metadata": {},
   "outputs": [],
   "source": [
    "#**************Random Forrest********************************#"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 250,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier\n",
    "clf=RandomForestClassifier(n_estimators=80)\n",
    "clf.fit(X_train,y_train)\n",
    "y_pred=clf.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 251,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.6138563638669315\n"
     ]
    }
   ],
   "source": [
    "from sklearn import metrics\n",
    "print(\"Accuracy:\",metrics.accuracy_score(y_test, y_pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 252,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.57      0.55      0.56     31504\n",
      "          1       0.65      0.66      0.66     39467\n",
      "\n",
      "avg / total       0.61      0.61      0.61     70971\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(metrics.classification_report(y_test,y_pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 198,
   "metadata": {},
   "outputs": [],
   "source": [
    "# NOT COMPLETING , waited for more than 15 mins\n",
    "#from sklearn.svm import SVC\n",
    "#svc = SVC(kernel='linear') \n",
    "#svc.fit(X_train, y_train) \n",
    "#svc_pred=svc.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 199,
   "metadata": {},
   "outputs": [],
   "source": [
    "actual_values=np.reshape(user_movies_data[['liked']].iloc[0:100].values,100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 200,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1 1 1 1 1 1 1 0 0 1 0 0 1 1 1 1 1 1 1 1 0 1 1 1 0 1 0 1 1 1 0 1 1 1 0 0 0\n",
      " 1 0 1 1 1 0 0 0 0 1 1 1 1 0 0 1 1 1 0 1 1 0 0 1 1 0 0 0 1 0 1 1 1 1 0 0 1\n",
      " 1 1 0 1 1 1 0 1 1 0 1 0 1 1 1 1 0 1 0 0 0 0 1 0 1 1]\n"
     ]
    }
   ],
   "source": [
    "predicted_values=clf.predict(user_movies_data[['age','gender','occupation','movie_year','content_rating','unknown','Action','Adventure','Animation','Children','Comedy','Crime','Documentary','Drama','Fantasy','Film-Noir','Horror','Musical','Mystery','Romance','Sci-Fi','Thriller','War','Western']].iloc[0:100])\n",
    "print(predicted_values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 201,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[23 16]\n",
      " [14 47]]\n"
     ]
    }
   ],
   "source": [
    "print(metrics.confusion_matrix(actual_values,predicted_values))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 202,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.7\n"
     ]
    }
   ],
   "source": [
    "print(\"Accuracy:\",metrics.accuracy_score(actual_values, predicted_values))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 989,
   "metadata": {},
   "outputs": [],
   "source": [
    "def is_user_eligble(movie_rating,user_age):\n",
    "    if user_age <= 17 and movie_rating in ['R','X','NC-17', 'Not Rated','unRated']:\n",
    "        return False;\n",
    "    return True;        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 990,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "False"
      ]
     },
     "execution_count": 990,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "is_user_eligble('R',16)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#**************Prediction*************************#"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "user=[\"9\",\"29\",\"M\",\"student\",\"01002\"]\n",
    "movie=[]"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}