Modeling missing data.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "\n",
    "df = pd.DataFrame(list(Rating.objects.values_list('user_id', 'work_id', 'choice')), columns=('user_id', 'work_id', 'choice'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "from collections import defaultdict\n",
    "rating_values = ['dislike', 'wontsee', 'neutral', 'willsee', 'like', 'favorite']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['rating'] = df['choice'].map(lambda choice: rating_values.index(choice))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "titles = dict(Work.objects.values_list('id', 'title'))\n",
    "df['title'] = df['work_id'].map(titles)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>work_id</th>\n",
       "      <th>choice</th>\n",
       "      <th>rating</th>\n",
       "      <th>title</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>169</td>\n",
       "      <td>1</td>\n",
       "      <td>like</td>\n",
       "      <td>4</td>\n",
       "      <td>Death Note</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>1146</td>\n",
       "      <td>like</td>\n",
       "      <td>4</td>\n",
       "      <td>Final Fantasy: The Spirits Within</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>177</td>\n",
       "      <td>8083</td>\n",
       "      <td>like</td>\n",
       "      <td>4</td>\n",
       "      <td>Kimagure Orange☆Road</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>181</td>\n",
       "      <td>1864</td>\n",
       "      <td>willsee</td>\n",
       "      <td>3</td>\n",
       "      <td>Haibane Renmei</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>188</td>\n",
       "      <td>9</td>\n",
       "      <td>like</td>\n",
       "      <td>4</td>\n",
       "      <td>Elfen Lied</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   user_id  work_id   choice  rating                              title\n",
       "0      169        1     like       4                         Death Note\n",
       "1        1     1146     like       4  Final Fantasy: The Spirits Within\n",
       "2      177     8083     like       4               Kimagure Orange☆Road\n",
       "3      181     1864  willsee       3                     Haibane Renmei\n",
       "4      188        9     like       4                         Elfen Lied"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "count_user = df.groupby('user_id').size().to_frame('count_user')\n",
    "df = df.join(count_user, on='user_id')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "count_work = df.groupby('work_id').size().to_frame('count_work')\n",
    "df = df.join(count_work, on='work_id')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "345491"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df[(df.count_work >= 10) & (df.count_user >= 10)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "encode_user = dict(zip(df['user_id'].unique(), range(3000)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "encode_work = dict(zip(df['work_id'].unique(), range(20000)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['encoded_user_id'] = df['user_id'].map(encode_user)\n",
    "df['encoded_work_id'] = df['work_id'].map(encode_work)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1869, 3409)"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['encoded_user_id'].max() + 1, df['encoded_work_id'].max() + 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Train with every rating\n",
    "#df[['encoded_user_id', 'encoded_work_id', 'rating']].to_csv('/tmp/train.csv', header=False, index=False)\n",
    "#df[['encoded_user_id', 'encoded_work_id', 'rating']].to_csv('/tmp/val.csv', header=False, index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "import random\n",
    "\n",
    "#test_users = random.sample(list(df[df.count_user <= 100]['user_id'].unique()), 187)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "#trainval_users, test_users = train_test_split(df['user_id'].unique(), test_size=0.2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "#trainval_users, test_users = train_test_split(df['user_id'].unique(), test_size=0.2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "trainval, test = train_test_split(df, test_size=0.2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "64287"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#len(test_users)\n",
    "len(test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "#test_users = [1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "#test = df.query('user_id in @test_users')[['encoded_user_id', 'encoded_work_id', 'rating']]\n",
    "test[['encoded_user_id', 'encoded_work_id', 'rating']].to_csv('/tmp/test.csv', header=False, index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Train et Val"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "#train, val = train_test_split(df.query('user_id not in @test_users'), test_size=0.2)\n",
    "train, val = train_test_split(trainval, test_size=0.1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "train[['encoded_user_id', 'encoded_work_id', 'rating']].to_csv('/tmp/train.csv', header=False, index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "val[['encoded_user_id', 'encoded_work_id', 'rating']].to_csv('/tmp/val.csv', header=False, index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "import yaml\n",
    "\n",
    "with open('/tmp/config.yml', 'w') as f:\n",
    "    config = {\n",
    "        'USER_NUM': len(df['user_id'].unique()),\n",
    "        'ITEM_NUM': len(df['work_id'].unique()),\n",
    "        'NB_CLASSES': len(df['rating'].unique()),\n",
    "        'BATCH_SIZE': len(train)\n",
    "    }\n",
    "    f.write(yaml.dump(config, default_flow_style=False))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [],
   "source": [
    "df[['encoded_work_id', 'title']].groupby('encoded_work_id').first().reset_index().to_csv('/tmp/works.csv', index=False, header=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(231433, 25715, 64287)"
      ]
     },
     "execution_count": 84,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(train), len(val), len(test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "from mangaki.algo.als import MangakiALS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "als = MangakiALS()\n",
    "als.set_parameters(config['USER_NUM'], config['ITEM_NUM'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "\n",
    "X_train = np.array(train[['encoded_user_id', 'encoded_work_id']])\n",
    "y_train = np.array(train['rating'])\n",
    "X_val = np.array(val[['encoded_user_id', 'encoded_work_id']])\n",
    "y_val = np.array(val['rating'])\n",
    "X_test = np.array(test[['encoded_user_id', 'encoded_work_id']])\n",
    "y_test = np.array(test['rating'])\n",
    "als.fit(X_train, y_train, X_test, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Train RMSE=0.929643\n",
      "Test RMSE=1.148684\n"
     ]
    }
   ],
   "source": [
    "als.compute_all_errors(X_train, y_train, X_val, y_val)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [],
   "source": [
    "titles = dict(Work.objects.values_list('id', 'title'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "#titles"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [],
   "source": [
    "top = [475, 1400, 1649, 1420, 1249, 1468, 2158, 1711, 985, 122, 361, 2182, 3361, 383, 310, 1242, 366, 1844, 1726, 812, 686, 2672, 195, 767, 1417, 1458, 452, 655, 86, 1207, 1408, 563, 3, 664, 568, 809, 1250, 159, 444, 104, 513, 1383, 413, 1763, 1422, 2064, 1384, 1320, 33, 423, 354, 1533, 1257, 1412, 1746, 234, 309, 116, 103, 806, 777, 1427, 319, 545, 860, 272, 386, 999, 994, 1834, 714, 470, 1385, 3305, 516, 2575, 51, 133, 1268, 1484, 1410, 728, 45, 225, 162, 3295, 554, 1344, 1781, 794, 277, 1155, 503, 1039, 549, 1933, 799, 311, 1414, 1392, 1216, 378, 722, 442, 1034, 804, 2457, 456, 52, 3388, 348, 970, 1054, 510, 364, 144, 675, 622, 1352, 267, 243, 1652, 389, 957, 63, 194, 836, 1382, 2481, 322, 748, 518, 208, 3043, 57, 586, 191, 3204, 142, 250, 3334, 1448, 330, 580, 218, 94, 658, 187, 776, 801, 150, 1980, 3392, 22, 1481, 85, 370, 1948, 178, 1407, 949, 449, 1632, 476, 865, 101, 566, 665, 76, 29, 404, 1374, 737, 53, 2229, 254, 2021, 3286, 179, 285, 154, 854, 506, 2655, 1429, 232, 77, 410, 1454, 182, 175, 3344, 197, 602, 2067, 344, 323, 169, 1868, 190, 3367, 368, 646, 687, 2110, 62, 1455, 457, 1139, 300, 261, 136, 174, 418, 167, 1026, 1413, 213, 640, 613, 207, 324, 491, 307, 7, 1105, 1124, 2830, 110, 265, 497, 1870, 555, 1411, 16, 128, 196, 132, 35, 37, 1245, 1488, 241, 775, 441, 1379, 8, 3400, 572, 557, 1432, 203, 282, 488, 770, 3355, 2856, 2017, 55, 540, 760, 3030, 450, 360, 807, 60, 1504, 1786, 291, 126, 242, 1451, 993, 216, 26, 1873, 25, 525, 381, 1943, 335, 2605, 3368, 1035, 1107, 329, 653, 158, 3042, 591, 263, 743, 1476, 553, 486, 99, 635, 2066, 2205, 824, 467, 346, 694, 345, 1332, 56, 332, 1893, 1554, 201, 259, 937, 629, 186, 36, 1487, 376, 1979, 2272, 721, 281, 2016, 1097, 365, 139, 424, 885, 3401, 501, 2495, 3315, 3195, 180, 815, 247, 573, 526, 877, 46, 121, 204, 280, 487, 356, 156, 305, 1867, 328, 102, 855, 1084, 1935, 231, 429, 130, 111, 107, 271, 321, 342, 88, 3356, 183, 546, 1409, 1251, 685, 331, 10, 351, 727, 32, 547, 1969, 3357, 3369, 428, 31, 1146, 2318, 725, 97, 397, 761, 1, 914, 1340, 202, 802, 2146, 1280, 2035, 2512, 927, 20, 83, 477, 940, 215, 1179, 1768, 1461, 2274, 1079, 3307, 135, 416, 620, 205, 2100, 1011, 283, 935, 955, 4, 1881, 717, 1584, 1766, 463, 326, 358, 3380, 1225, 6, 333, 607, 228, 42, 1278, 484, 1475, 27, 308, 149, 435, 125, 1758, 320, 2710, 1614, 859, 74, 315, 472, 251, 11, 726, 690, 115, 1416, 304, 1936, 23, 1145, 1585, 3376, 705, 340, 1820, 997, 2023, 783, 230, 161, 1160, 100, 58, 206, 2089, 1955, 399, 108, 1462, 670, 2121, 2888, 958, 1133, 47, 612, 755, 44, 2513, 615, 87, 845, 2585, 512, 627, 1760, 3323, 124, 54, 2020, 1194, 313, 712, 245, 2019, 1201, 790, 1490, 1894, 357, 273, 238, 3026, 455, 1070, 2562, 98, 284, 334, 338, 779, 28, 474, 1581, 2065, 210, 1308, 509, 791, 377, 1378, 211, 105, 1491, 70, 209, 1561, 1515, 41, 2076, 430, 146, 192, 264, 2106, 172, 1635, 185, 226, 438, 803, 224, 853, 3003, 733, 112, 682, 1128, 64, 21, 127, 253, 1956, 991, 1078, 117, 13, 240, 43, 92, 548, 2835, 5, 1848, 293, 923, 84, 1782, 3364, 585, 95, 177, 0, 148, 2108, 113, 1631, 66, 395, 17, 385, 405, 387, 252, 114, 636, 454, 837, 48, 339, 165, 2968, 34, 82, 523, 237, 278, 564, 163, 30, 19, 171, 160, 236, 244, 188, 93, 118, 384, 288, 65, 170, 12, 59, 72, 9, 155]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [],
   "source": [
    "decode_work = {v: k for k, v in encode_work.items()}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [],
   "source": [
    "#[(id_, titles[decode_work[id_]]) for id_ in top]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [],
   "source": [
    "train['title'] = train['work_id'].map(titles)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>work_id</th>\n",
       "      <th>choice</th>\n",
       "      <th>rating</th>\n",
       "      <th>count_user</th>\n",
       "      <th>count_work</th>\n",
       "      <th>encoded_user_id</th>\n",
       "      <th>encoded_work_id</th>\n",
       "      <th>title</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>24043</th>\n",
       "      <td>314</td>\n",
       "      <td>1</td>\n",
       "      <td>like</td>\n",
       "      <td>4</td>\n",
       "      <td>444</td>\n",
       "      <td>1663</td>\n",
       "      <td>245</td>\n",
       "      <td>0</td>\n",
       "      <td>Death Note</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>89389</th>\n",
       "      <td>700</td>\n",
       "      <td>1</td>\n",
       "      <td>like</td>\n",
       "      <td>4</td>\n",
       "      <td>129</td>\n",
       "      <td>1663</td>\n",
       "      <td>582</td>\n",
       "      <td>0</td>\n",
       "      <td>Death Note</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>280499</th>\n",
       "      <td>2012</td>\n",
       "      <td>1</td>\n",
       "      <td>like</td>\n",
       "      <td>4</td>\n",
       "      <td>285</td>\n",
       "      <td>1663</td>\n",
       "      <td>1595</td>\n",
       "      <td>0</td>\n",
       "      <td>Death Note</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17739</th>\n",
       "      <td>20</td>\n",
       "      <td>1</td>\n",
       "      <td>neutral</td>\n",
       "      <td>2</td>\n",
       "      <td>273</td>\n",
       "      <td>1663</td>\n",
       "      <td>212</td>\n",
       "      <td>0</td>\n",
       "      <td>Death Note</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>272889</th>\n",
       "      <td>1933</td>\n",
       "      <td>1</td>\n",
       "      <td>like</td>\n",
       "      <td>4</td>\n",
       "      <td>74</td>\n",
       "      <td>1663</td>\n",
       "      <td>1537</td>\n",
       "      <td>0</td>\n",
       "      <td>Death Note</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        user_id  work_id   choice  rating  count_user  count_work  \\\n",
       "24043       314        1     like       4         444        1663   \n",
       "89389       700        1     like       4         129        1663   \n",
       "280499     2012        1     like       4         285        1663   \n",
       "17739        20        1  neutral       2         273        1663   \n",
       "272889     1933        1     like       4          74        1663   \n",
       "\n",
       "        encoded_user_id  encoded_work_id       title  \n",
       "24043               245                0  Death Note  \n",
       "89389               582                0  Death Note  \n",
       "280499             1595                0  Death Note  \n",
       "17739               212                0  Death Note  \n",
       "272889             1537                0  Death Note  "
      ]
     },
     "execution_count": 82,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.sort_values('count_work', ascending=False).head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Django Shell-Plus",
   "language": "python",
   "name": "django_extensions"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}