EDA.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import json\n",
    "from scipy.sparse import csr_matrix"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# [Spotify MPD] Item to Item Collabortive Filtering\n",
    "\n",
    "## Building the Similarity Matrix\n",
    "\n",
    "We will primarily use the Item to Item Collaborative Filtering (CF) Algorithim to generate reccomendations. At the high level, the Item to Item CF simply creates a large matrix (S) that holds the pairwise similairties between each item of a data. In our case each item is a song, identified by it's unique track id. So if there a total of 34443 songs in the dataset, S will be a 34443 x 34443 matrix where $S_{ij}$ holds the pairwise similarity betewen song $i$ and song $j$. We calculate similarity using the [Jaccard Index](https://en.wikipedia.org/wiki/Jaccard_index):\n",
    "$$ S_{ij} = \\frac{|\\{ p_i\\} \\cap \\{p_j\\}|}{|\\{p_i\\} \\cup \\{p_j\\}|}$$\n",
    "\n",
    "$\\{p_k\\}$ here is a **set** of all playlist ids that contain song k. \n",
    "\n",
    "Let's start building our similarity matrix (S). We'll begin by loading in the first 1000 playlist splice of our dataset. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Read in json splice and transform into a playlist level dataframe\n",
    "start = 0\n",
    "end = 1000\n",
    "path = 'data/mpd.slice.' + str(start) + \"-\" + str(end-1) + '.json'\n",
    "d = json.load(open(path, 'r'))\n",
    "thisSlice = pd.DataFrame.from_dict(d['playlists'], orient='columns')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(67503, 4)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>trackid</th>\n",
       "      <th>artist_name</th>\n",
       "      <th>track_name</th>\n",
       "      <th>pid</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>spotify:track:0UaMYEvWZi0ZqiDOoHU3YI</td>\n",
       "      <td>Missy Elliott</td>\n",
       "      <td>Lose Control (feat. Ciara &amp; Fat Man Scoop)</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>spotify:track:6I9VzXrHxO9rA9A5euc8Ak</td>\n",
       "      <td>Britney Spears</td>\n",
       "      <td>Toxic</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>spotify:track:0WqIKmW4BTrj3eJFmnCKMv</td>\n",
       "      <td>Beyoncé</td>\n",
       "      <td>Crazy In Love</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>spotify:track:1AWQoqb9bSvzTjaLralEkT</td>\n",
       "      <td>Justin Timberlake</td>\n",
       "      <td>Rock Your Body</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>spotify:track:1lzr43nnXAijIGYnCT8M8H</td>\n",
       "      <td>Shaggy</td>\n",
       "      <td>It Wasn't Me</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>spotify:track:0XUfyU2QviPAs6bxSpXYG4</td>\n",
       "      <td>Usher</td>\n",
       "      <td>Yeah!</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>spotify:track:68vgtRHr7iZHpzGpon6Jlo</td>\n",
       "      <td>Usher</td>\n",
       "      <td>My Boo</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>spotify:track:3BxWKCI06eQ5Od8TY2JBeA</td>\n",
       "      <td>The Pussycat Dolls</td>\n",
       "      <td>Buttons</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>spotify:track:7H6ev70Weq6DdpZyyTmUXk</td>\n",
       "      <td>Destiny's Child</td>\n",
       "      <td>Say My Name</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>spotify:track:2PpruBYCo4H7WOBJ7Q2EwM</td>\n",
       "      <td>OutKast</td>\n",
       "      <td>Hey Ya! - Radio Mix / Club Mix</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                trackid         artist_name  \\\n",
       "0  spotify:track:0UaMYEvWZi0ZqiDOoHU3YI       Missy Elliott   \n",
       "1  spotify:track:6I9VzXrHxO9rA9A5euc8Ak      Britney Spears   \n",
       "2  spotify:track:0WqIKmW4BTrj3eJFmnCKMv             Beyoncé   \n",
       "3  spotify:track:1AWQoqb9bSvzTjaLralEkT   Justin Timberlake   \n",
       "4  spotify:track:1lzr43nnXAijIGYnCT8M8H              Shaggy   \n",
       "5  spotify:track:0XUfyU2QviPAs6bxSpXYG4               Usher   \n",
       "6  spotify:track:68vgtRHr7iZHpzGpon6Jlo               Usher   \n",
       "7  spotify:track:3BxWKCI06eQ5Od8TY2JBeA  The Pussycat Dolls   \n",
       "8  spotify:track:7H6ev70Weq6DdpZyyTmUXk     Destiny's Child   \n",
       "9  spotify:track:2PpruBYCo4H7WOBJ7Q2EwM             OutKast   \n",
       "\n",
       "                                   track_name  pid  \n",
       "0  Lose Control (feat. Ciara & Fat Man Scoop)    0  \n",
       "1                                       Toxic    0  \n",
       "2                               Crazy In Love    0  \n",
       "3                              Rock Your Body    0  \n",
       "4                                It Wasn't Me    0  \n",
       "5                                       Yeah!    0  \n",
       "6                                      My Boo    0  \n",
       "7                                     Buttons    0  \n",
       "8                                 Say My Name    0  \n",
       "9              Hey Ya! - Radio Mix / Club Mix    0  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Turn playlist level dataframe into song level dataframe\n",
    "songPlaylistArray = []\n",
    "for index, row in thisSlice.iterrows():\n",
    "    for track in row['tracks']:\n",
    "        songPlaylistArray.append([track['track_uri'], track['artist_name'], track['track_name'], row['pid']])\n",
    "songPlaylist = pd.DataFrame(songPlaylistArray, columns=['trackid', 'artist_name', 'track_name', 'pid'])\n",
    "\n",
    "print(songPlaylist.shape)\n",
    "songPlaylist.head(10)   #is a df of all track ids, cooresponding artist names, track names and playlist ids"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now in order to create S, we could naively look up the playlists that contain song $i$ and song $j$ for each entry in the matrix but that would use a nested for loop and take hours to run. Instead, we'll do some matrix manipulation to obtain the same result. We will start off by building a simlple [co-occurance matrix](https://blogs.msdn.microsoft.com/carlnol/2012/06/23/co-occurrence-approach-to-an-item-based-recommender/). \n",
    "\n",
    "Each entry in the co-occurance matrix (C) will hold the number of playlists in which song $i$ and song $j$ are both present. Note, this is exactly the numerator of our similarity metric above. So matrix C will hold all the numerators required to create S, our final similarity matrix. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1000, 34443)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>trackid</th>\n",
       "      <th>spotify:track:000mA0etY38nKdvf1N04af</th>\n",
       "      <th>spotify:track:000xQL6tZNLJzIrtIgxqSl</th>\n",
       "      <th>spotify:track:006AVH7fq061voGXkUiII4</th>\n",
       "      <th>spotify:track:006PJvsr6CyV3JdBf7wiNF</th>\n",
       "      <th>spotify:track:006yrnQMCZpiUgkR612gC8</th>\n",
       "      <th>spotify:track:00BuKLSAFkaEkaVAgIMbeA</th>\n",
       "      <th>spotify:track:00Ci0EXS4fNPnkTbS6wkOh</th>\n",
       "      <th>spotify:track:00CmjeeHvAVKvx3tcIiZTy</th>\n",
       "      <th>spotify:track:00DYRuYJQzfI6dH4Adkimo</th>\n",
       "      <th>spotify:track:00DlEKhhlQNtjnJk7xqB9O</th>\n",
       "      <th>...</th>\n",
       "      <th>spotify:track:7znZvX0Mt6NBmaI8VCPurT</th>\n",
       "      <th>spotify:track:7zo8XAMYBG6nGpqGiIudBc</th>\n",
       "      <th>spotify:track:7zprfY9KDG8g3S5BIOB7xZ</th>\n",
       "      <th>spotify:track:7zqLBFKCBkk5IfbgKgH4VZ</th>\n",
       "      <th>spotify:track:7zscdQe9CjzXnqT3P1Ey7K</th>\n",
       "      <th>spotify:track:7zsw78LtXUD7JfEwH64HK2</th>\n",
       "      <th>spotify:track:7zuwaenG5AF0vG7o7kMduX</th>\n",
       "      <th>spotify:track:7zxRMhXxJMQCeDDg0rKAVo</th>\n",
       "      <th>spotify:track:7zz1drChhd4hQBiGSnLRBZ</th>\n",
       "      <th>spotify:track:7zzBEZBTJejWeL6EqWmCD9</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>pid</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 34443 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "trackid  spotify:track:000mA0etY38nKdvf1N04af  \\\n",
       "pid                                             \n",
       "0                                           0   \n",
       "1                                           0   \n",
       "2                                           0   \n",
       "3                                           0   \n",
       "4                                           0   \n",
       "\n",
       "trackid  spotify:track:000xQL6tZNLJzIrtIgxqSl  \\\n",
       "pid                                             \n",
       "0                                           0   \n",
       "1                                           0   \n",
       "2                                           0   \n",
       "3                                           0   \n",
       "4                                           0   \n",
       "\n",
       "trackid  spotify:track:006AVH7fq061voGXkUiII4  \\\n",
       "pid                                             \n",
       "0                                           0   \n",
       "1                                           0   \n",
       "2                                           0   \n",
       "3                                           0   \n",
       "4                                           0   \n",
       "\n",
       "trackid  spotify:track:006PJvsr6CyV3JdBf7wiNF  \\\n",
       "pid                                             \n",
       "0                                           0   \n",
       "1                                           0   \n",
       "2                                           0   \n",
       "3                                           0   \n",
       "4                                           0   \n",
       "\n",
       "trackid  spotify:track:006yrnQMCZpiUgkR612gC8  \\\n",
       "pid                                             \n",
       "0                                           0   \n",
       "1                                           0   \n",
       "2                                           0   \n",
       "3                                           0   \n",
       "4                                           0   \n",
       "\n",
       "trackid  spotify:track:00BuKLSAFkaEkaVAgIMbeA  \\\n",
       "pid                                             \n",
       "0                                           0   \n",
       "1                                           0   \n",
       "2                                           0   \n",
       "3                                           0   \n",
       "4                                           0   \n",
       "\n",
       "trackid  spotify:track:00Ci0EXS4fNPnkTbS6wkOh  \\\n",
       "pid                                             \n",
       "0                                           0   \n",
       "1                                           0   \n",
       "2                                           0   \n",
       "3                                           0   \n",
       "4                                           0   \n",
       "\n",
       "trackid  spotify:track:00CmjeeHvAVKvx3tcIiZTy  \\\n",
       "pid                                             \n",
       "0                                           0   \n",
       "1                                           0   \n",
       "2                                           0   \n",
       "3                                           0   \n",
       "4                                           0   \n",
       "\n",
       "trackid  spotify:track:00DYRuYJQzfI6dH4Adkimo  \\\n",
       "pid                                             \n",
       "0                                           0   \n",
       "1                                           0   \n",
       "2                                           0   \n",
       "3                                           0   \n",
       "4                                           0   \n",
       "\n",
       "trackid  spotify:track:00DlEKhhlQNtjnJk7xqB9O  \\\n",
       "pid                                             \n",
       "0                                           0   \n",
       "1                                           0   \n",
       "2                                           0   \n",
       "3                                           0   \n",
       "4                                           0   \n",
       "\n",
       "trackid                  ...                   \\\n",
       "pid                      ...                    \n",
       "0                        ...                    \n",
       "1                        ...                    \n",
       "2                        ...                    \n",
       "3                        ...                    \n",
       "4                        ...                    \n",
       "\n",
       "trackid  spotify:track:7znZvX0Mt6NBmaI8VCPurT  \\\n",
       "pid                                             \n",
       "0                                           0   \n",
       "1                                           0   \n",
       "2                                           0   \n",
       "3                                           0   \n",
       "4                                           0   \n",
       "\n",
       "trackid  spotify:track:7zo8XAMYBG6nGpqGiIudBc  \\\n",
       "pid                                             \n",
       "0                                           0   \n",
       "1                                           0   \n",
       "2                                           0   \n",
       "3                                           0   \n",
       "4                                           0   \n",
       "\n",
       "trackid  spotify:track:7zprfY9KDG8g3S5BIOB7xZ  \\\n",
       "pid                                             \n",
       "0                                           0   \n",
       "1                                           0   \n",
       "2                                           0   \n",
       "3                                           0   \n",
       "4                                           0   \n",
       "\n",
       "trackid  spotify:track:7zqLBFKCBkk5IfbgKgH4VZ  \\\n",
       "pid                                             \n",
       "0                                           0   \n",
       "1                                           0   \n",
       "2                                           0   \n",
       "3                                           0   \n",
       "4                                           0   \n",
       "\n",
       "trackid  spotify:track:7zscdQe9CjzXnqT3P1Ey7K  \\\n",
       "pid                                             \n",
       "0                                           0   \n",
       "1                                           0   \n",
       "2                                           0   \n",
       "3                                           0   \n",
       "4                                           0   \n",
       "\n",
       "trackid  spotify:track:7zsw78LtXUD7JfEwH64HK2  \\\n",
       "pid                                             \n",
       "0                                           0   \n",
       "1                                           0   \n",
       "2                                           0   \n",
       "3                                           0   \n",
       "4                                           0   \n",
       "\n",
       "trackid  spotify:track:7zuwaenG5AF0vG7o7kMduX  \\\n",
       "pid                                             \n",
       "0                                           0   \n",
       "1                                           0   \n",
       "2                                           0   \n",
       "3                                           0   \n",
       "4                                           0   \n",
       "\n",
       "trackid  spotify:track:7zxRMhXxJMQCeDDg0rKAVo  \\\n",
       "pid                                             \n",
       "0                                           0   \n",
       "1                                           0   \n",
       "2                                           0   \n",
       "3                                           0   \n",
       "4                                           0   \n",
       "\n",
       "trackid  spotify:track:7zz1drChhd4hQBiGSnLRBZ  \\\n",
       "pid                                             \n",
       "0                                           0   \n",
       "1                                           0   \n",
       "2                                           0   \n",
       "3                                           0   \n",
       "4                                           0   \n",
       "\n",
       "trackid  spotify:track:7zzBEZBTJejWeL6EqWmCD9  \n",
       "pid                                            \n",
       "0                                           0  \n",
       "1                                           0  \n",
       "2                                           0  \n",
       "3                                           0  \n",
       "4                                           0  \n",
       "\n",
       "[5 rows x 34443 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## index: pids, columns: unique songs, 0/1 whether its in the playlist\n",
    "A = songPlaylist.drop(['artist_name', 'track_name'], axis=1).pivot_table(index=[\"pid\"], columns=\"trackid\", aggfunc=lambda x: 1, fill_value=0)\n",
    "print(A.shape)\n",
    "A.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We can compute the co-occurance matrix (C) using smone matrix manipulation. Note that C is simply $ C = A^T A $."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(34443, 34443)\n",
      "CPU times: user 1.13 s, sys: 43.7 ms, total: 1.17 s\n",
      "Wall time: 1.35 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "A = csr_matrix(A) #scipy's sparse library will compress our array's and speed up computation time\n",
    "\n",
    "C = A.T * A\n",
    "print(C.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[1, 0, 0, ..., 0, 0, 0],\n",
       "       [0, 2, 0, ..., 0, 0, 0],\n",
       "       [0, 0, 1, ..., 0, 0, 0],\n",
       "       ...,\n",
       "       [0, 0, 0, ..., 4, 0, 0],\n",
       "       [0, 0, 0, ..., 0, 1, 0],\n",
       "       [0, 0, 0, ..., 0, 0, 1]], dtype=int64)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "C.toarray()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Great, now C holds all the numerators we need to create S.\n",
    "\n",
    "We will create a seperate matrix D to hold our denominator values. Each $D_{ij}$ should hold the length of the union of $\\{p_i\\}$ and $\\{p_j\\}$. Se theory tells us $A \\cup B = A + B - (A \\cap B)$. We can apply length function to both sides of this equation to obtain $|A \\cup B| = |A| + |B| - |(A \\cap B)|$. Note that $|(A \\cap B)|$ is just our co-occurance matrix C. \n",
    "\n",
    "So to build D, we will compute D = $(\\{p_i\\} + \\{p_j\\}) - C$ for all $i$, $j$."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "song_freq_vec = np.sum(A, axis=0)\n",
    "song_freq_vec = song_freq_vec.reshape(34443, 1)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 59.1 s, sys: 45.7 s, total: 1min 44s\n",
      "Wall time: 2min 24s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "C1 = C.toarray()\n",
    "D  = (song_freq_vec + song_freq_vec.T)-C1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Finally, to build S we have C which holds all numerators and D which holds all denominators. To obtain S, sll we need to do is a pairwise division of S by D."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 56.6 s, sys: 29.9 s, total: 1min 26s\n",
      "Wall time: 1min 40s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "S1 = np.divide(C1, D)\n",
    "print(S.shape)\n",
    "S"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Generating Similar Reccomendations\n",
    "\n",
    "With our similarity matrix (S) at hand, we will generate reccomendations based on which ever songs are most similar to the songs in our testing playlist. For each test playlist, we will splice S to create a new matrix that contians only the songs in the testing playlist in the row index and contians all songs in the column index. Then to compute the similarity a song has to the test playlist all we need to do is add along the columns (or collapse rows). Then for each song will have a similarity metric to the whole test playlist. To generate reccomendations we select the songs that are most similary or have largest sum. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>trackid</th>\n",
       "      <th>spotify:track:000mA0etY38nKdvf1N04af</th>\n",
       "      <th>spotify:track:000xQL6tZNLJzIrtIgxqSl</th>\n",
       "      <th>spotify:track:006AVH7fq061voGXkUiII4</th>\n",
       "      <th>spotify:track:006PJvsr6CyV3JdBf7wiNF</th>\n",
       "      <th>spotify:track:006yrnQMCZpiUgkR612gC8</th>\n",
       "      <th>spotify:track:00BuKLSAFkaEkaVAgIMbeA</th>\n",
       "      <th>spotify:track:00Ci0EXS4fNPnkTbS6wkOh</th>\n",
       "      <th>spotify:track:00CmjeeHvAVKvx3tcIiZTy</th>\n",
       "      <th>spotify:track:00DYRuYJQzfI6dH4Adkimo</th>\n",
       "      <th>spotify:track:00DlEKhhlQNtjnJk7xqB9O</th>\n",
       "      <th>...</th>\n",
       "      <th>spotify:track:7znZvX0Mt6NBmaI8VCPurT</th>\n",
       "      <th>spotify:track:7zo8XAMYBG6nGpqGiIudBc</th>\n",
       "      <th>spotify:track:7zprfY9KDG8g3S5BIOB7xZ</th>\n",
       "      <th>spotify:track:7zqLBFKCBkk5IfbgKgH4VZ</th>\n",
       "      <th>spotify:track:7zscdQe9CjzXnqT3P1Ey7K</th>\n",
       "      <th>spotify:track:7zsw78LtXUD7JfEwH64HK2</th>\n",
       "      <th>spotify:track:7zuwaenG5AF0vG7o7kMduX</th>\n",
       "      <th>spotify:track:7zxRMhXxJMQCeDDg0rKAVo</th>\n",
       "      <th>spotify:track:7zz1drChhd4hQBiGSnLRBZ</th>\n",
       "      <th>spotify:track:7zzBEZBTJejWeL6EqWmCD9</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>trackid</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>spotify:track:000xQL6tZNLJzIrtIgxqSl</th>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>spotify:track:006yrnQMCZpiUgkR612gC8</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>spotify:track:00EhzXcoejkD5RWoS8L4IX</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>spotify:track:00W5ThRIn0d1bQNOp0S0af</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>spotify:track:02WDuAXXKumemrvsmrVBvH</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>spotify:track:01PLa3LP9WfuAKr3mBRnY4</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>spotify:track:0DIekYoiwwr1kkTUGmQlSL</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>7 rows × 34443 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "trackid                               spotify:track:000mA0etY38nKdvf1N04af  \\\n",
       "trackid                                                                      \n",
       "spotify:track:000xQL6tZNLJzIrtIgxqSl                                   0.0   \n",
       "spotify:track:006yrnQMCZpiUgkR612gC8                                   0.0   \n",
       "spotify:track:00EhzXcoejkD5RWoS8L4IX                                   0.0   \n",
       "spotify:track:00W5ThRIn0d1bQNOp0S0af                                   0.0   \n",
       "spotify:track:02WDuAXXKumemrvsmrVBvH                                   0.0   \n",
       "spotify:track:01PLa3LP9WfuAKr3mBRnY4                                   0.0   \n",
       "spotify:track:0DIekYoiwwr1kkTUGmQlSL                                   0.0   \n",
       "\n",
       "trackid                               spotify:track:000xQL6tZNLJzIrtIgxqSl  \\\n",
       "trackid                                                                      \n",
       "spotify:track:000xQL6tZNLJzIrtIgxqSl                                   1.0   \n",
       "spotify:track:006yrnQMCZpiUgkR612gC8                                   0.0   \n",
       "spotify:track:00EhzXcoejkD5RWoS8L4IX                                   0.0   \n",
       "spotify:track:00W5ThRIn0d1bQNOp0S0af                                   0.0   \n",
       "spotify:track:02WDuAXXKumemrvsmrVBvH                                   0.0   \n",
       "spotify:track:01PLa3LP9WfuAKr3mBRnY4                                   0.0   \n",
       "spotify:track:0DIekYoiwwr1kkTUGmQlSL                                   0.0   \n",
       "\n",
       "trackid                               spotify:track:006AVH7fq061voGXkUiII4  \\\n",
       "trackid                                                                      \n",
       "spotify:track:000xQL6tZNLJzIrtIgxqSl                                   0.0   \n",
       "spotify:track:006yrnQMCZpiUgkR612gC8                                   0.0   \n",
       "spotify:track:00EhzXcoejkD5RWoS8L4IX                                   0.0   \n",
       "spotify:track:00W5ThRIn0d1bQNOp0S0af                                   0.0   \n",
       "spotify:track:02WDuAXXKumemrvsmrVBvH                                   0.0   \n",
       "spotify:track:01PLa3LP9WfuAKr3mBRnY4                                   0.0   \n",
       "spotify:track:0DIekYoiwwr1kkTUGmQlSL                                   0.0   \n",
       "\n",
       "trackid                               spotify:track:006PJvsr6CyV3JdBf7wiNF  \\\n",
       "trackid                                                                      \n",
       "spotify:track:000xQL6tZNLJzIrtIgxqSl                                   0.0   \n",
       "spotify:track:006yrnQMCZpiUgkR612gC8                                   0.0   \n",
       "spotify:track:00EhzXcoejkD5RWoS8L4IX                                   0.0   \n",
       "spotify:track:00W5ThRIn0d1bQNOp0S0af                                   0.0   \n",
       "spotify:track:02WDuAXXKumemrvsmrVBvH                                   0.0   \n",
       "spotify:track:01PLa3LP9WfuAKr3mBRnY4                                   0.0   \n",
       "spotify:track:0DIekYoiwwr1kkTUGmQlSL                                   0.0   \n",
       "\n",
       "trackid                               spotify:track:006yrnQMCZpiUgkR612gC8  \\\n",
       "trackid                                                                      \n",
       "spotify:track:000xQL6tZNLJzIrtIgxqSl                                   0.0   \n",
       "spotify:track:006yrnQMCZpiUgkR612gC8                                   1.0   \n",
       "spotify:track:00EhzXcoejkD5RWoS8L4IX                                   0.0   \n",
       "spotify:track:00W5ThRIn0d1bQNOp0S0af                                   0.0   \n",
       "spotify:track:02WDuAXXKumemrvsmrVBvH                                   0.0   \n",
       "spotify:track:01PLa3LP9WfuAKr3mBRnY4                                   0.0   \n",
       "spotify:track:0DIekYoiwwr1kkTUGmQlSL                                   0.0   \n",
       "\n",
       "trackid                               spotify:track:00BuKLSAFkaEkaVAgIMbeA  \\\n",
       "trackid                                                                      \n",
       "spotify:track:000xQL6tZNLJzIrtIgxqSl                                   0.0   \n",
       "spotify:track:006yrnQMCZpiUgkR612gC8                                   0.0   \n",
       "spotify:track:00EhzXcoejkD5RWoS8L4IX                                   0.0   \n",
       "spotify:track:00W5ThRIn0d1bQNOp0S0af                                   0.0   \n",
       "spotify:track:02WDuAXXKumemrvsmrVBvH                                   0.0   \n",
       "spotify:track:01PLa3LP9WfuAKr3mBRnY4                                   0.0   \n",
       "spotify:track:0DIekYoiwwr1kkTUGmQlSL                                   0.0   \n",
       "\n",
       "trackid                               spotify:track:00Ci0EXS4fNPnkTbS6wkOh  \\\n",
       "trackid                                                                      \n",
       "spotify:track:000xQL6tZNLJzIrtIgxqSl                                   0.0   \n",
       "spotify:track:006yrnQMCZpiUgkR612gC8                                   0.0   \n",
       "spotify:track:00EhzXcoejkD5RWoS8L4IX                                   0.0   \n",
       "spotify:track:00W5ThRIn0d1bQNOp0S0af                                   0.0   \n",
       "spotify:track:02WDuAXXKumemrvsmrVBvH                                   0.0   \n",
       "spotify:track:01PLa3LP9WfuAKr3mBRnY4                                   0.0   \n",
       "spotify:track:0DIekYoiwwr1kkTUGmQlSL                                   0.0   \n",
       "\n",
       "trackid                               spotify:track:00CmjeeHvAVKvx3tcIiZTy  \\\n",
       "trackid                                                                      \n",
       "spotify:track:000xQL6tZNLJzIrtIgxqSl                                   0.0   \n",
       "spotify:track:006yrnQMCZpiUgkR612gC8                                   0.0   \n",
       "spotify:track:00EhzXcoejkD5RWoS8L4IX                                   0.0   \n",
       "spotify:track:00W5ThRIn0d1bQNOp0S0af                                   0.0   \n",
       "spotify:track:02WDuAXXKumemrvsmrVBvH                                   0.0   \n",
       "spotify:track:01PLa3LP9WfuAKr3mBRnY4                                   0.0   \n",
       "spotify:track:0DIekYoiwwr1kkTUGmQlSL                                   0.0   \n",
       "\n",
       "trackid                               spotify:track:00DYRuYJQzfI6dH4Adkimo  \\\n",
       "trackid                                                                      \n",
       "spotify:track:000xQL6tZNLJzIrtIgxqSl                                   0.0   \n",
       "spotify:track:006yrnQMCZpiUgkR612gC8                                   0.0   \n",
       "spotify:track:00EhzXcoejkD5RWoS8L4IX                                   0.0   \n",
       "spotify:track:00W5ThRIn0d1bQNOp0S0af                                   0.0   \n",
       "spotify:track:02WDuAXXKumemrvsmrVBvH                                   0.0   \n",
       "spotify:track:01PLa3LP9WfuAKr3mBRnY4                                   0.0   \n",
       "spotify:track:0DIekYoiwwr1kkTUGmQlSL                                   0.0   \n",
       "\n",
       "trackid                               spotify:track:00DlEKhhlQNtjnJk7xqB9O  \\\n",
       "trackid                                                                      \n",
       "spotify:track:000xQL6tZNLJzIrtIgxqSl                                   0.0   \n",
       "spotify:track:006yrnQMCZpiUgkR612gC8                                   0.0   \n",
       "spotify:track:00EhzXcoejkD5RWoS8L4IX                                   0.0   \n",
       "spotify:track:00W5ThRIn0d1bQNOp0S0af                                   0.0   \n",
       "spotify:track:02WDuAXXKumemrvsmrVBvH                                   0.0   \n",
       "spotify:track:01PLa3LP9WfuAKr3mBRnY4                                   0.0   \n",
       "spotify:track:0DIekYoiwwr1kkTUGmQlSL                                   0.0   \n",
       "\n",
       "trackid                                               ...                   \\\n",
       "trackid                                               ...                    \n",
       "spotify:track:000xQL6tZNLJzIrtIgxqSl                  ...                    \n",
       "spotify:track:006yrnQMCZpiUgkR612gC8                  ...                    \n",
       "spotify:track:00EhzXcoejkD5RWoS8L4IX                  ...                    \n",
       "spotify:track:00W5ThRIn0d1bQNOp0S0af                  ...                    \n",
       "spotify:track:02WDuAXXKumemrvsmrVBvH                  ...                    \n",
       "spotify:track:01PLa3LP9WfuAKr3mBRnY4                  ...                    \n",
       "spotify:track:0DIekYoiwwr1kkTUGmQlSL                  ...                    \n",
       "\n",
       "trackid                               spotify:track:7znZvX0Mt6NBmaI8VCPurT  \\\n",
       "trackid                                                                      \n",
       "spotify:track:000xQL6tZNLJzIrtIgxqSl                                   0.0   \n",
       "spotify:track:006yrnQMCZpiUgkR612gC8                                   0.0   \n",
       "spotify:track:00EhzXcoejkD5RWoS8L4IX                                   0.0   \n",
       "spotify:track:00W5ThRIn0d1bQNOp0S0af                                   0.0   \n",
       "spotify:track:02WDuAXXKumemrvsmrVBvH                                   0.0   \n",
       "spotify:track:01PLa3LP9WfuAKr3mBRnY4                                   0.0   \n",
       "spotify:track:0DIekYoiwwr1kkTUGmQlSL                                   0.0   \n",
       "\n",
       "trackid                               spotify:track:7zo8XAMYBG6nGpqGiIudBc  \\\n",
       "trackid                                                                      \n",
       "spotify:track:000xQL6tZNLJzIrtIgxqSl                                   0.0   \n",
       "spotify:track:006yrnQMCZpiUgkR612gC8                                   0.0   \n",
       "spotify:track:00EhzXcoejkD5RWoS8L4IX                                   0.0   \n",
       "spotify:track:00W5ThRIn0d1bQNOp0S0af                                   0.0   \n",
       "spotify:track:02WDuAXXKumemrvsmrVBvH                                   0.0   \n",
       "spotify:track:01PLa3LP9WfuAKr3mBRnY4                                   0.0   \n",
       "spotify:track:0DIekYoiwwr1kkTUGmQlSL                                   0.0   \n",
       "\n",
       "trackid                               spotify:track:7zprfY9KDG8g3S5BIOB7xZ  \\\n",
       "trackid                                                                      \n",
       "spotify:track:000xQL6tZNLJzIrtIgxqSl                                   0.0   \n",
       "spotify:track:006yrnQMCZpiUgkR612gC8                                   0.0   \n",
       "spotify:track:00EhzXcoejkD5RWoS8L4IX                                   0.0   \n",
       "spotify:track:00W5ThRIn0d1bQNOp0S0af                                   0.0   \n",
       "spotify:track:02WDuAXXKumemrvsmrVBvH                                   0.0   \n",
       "spotify:track:01PLa3LP9WfuAKr3mBRnY4                                   0.0   \n",
       "spotify:track:0DIekYoiwwr1kkTUGmQlSL                                   0.0   \n",
       "\n",
       "trackid                               spotify:track:7zqLBFKCBkk5IfbgKgH4VZ  \\\n",
       "trackid                                                                      \n",
       "spotify:track:000xQL6tZNLJzIrtIgxqSl                                   0.0   \n",
       "spotify:track:006yrnQMCZpiUgkR612gC8                                   0.0   \n",
       "spotify:track:00EhzXcoejkD5RWoS8L4IX                                   0.0   \n",
       "spotify:track:00W5ThRIn0d1bQNOp0S0af                                   0.0   \n",
       "spotify:track:02WDuAXXKumemrvsmrVBvH                                   0.0   \n",
       "spotify:track:01PLa3LP9WfuAKr3mBRnY4                                   0.0   \n",
       "spotify:track:0DIekYoiwwr1kkTUGmQlSL                                   0.0   \n",
       "\n",
       "trackid                               spotify:track:7zscdQe9CjzXnqT3P1Ey7K  \\\n",
       "trackid                                                                      \n",
       "spotify:track:000xQL6tZNLJzIrtIgxqSl                                   0.0   \n",
       "spotify:track:006yrnQMCZpiUgkR612gC8                                   0.0   \n",
       "spotify:track:00EhzXcoejkD5RWoS8L4IX                                   0.0   \n",
       "spotify:track:00W5ThRIn0d1bQNOp0S0af                                   0.0   \n",
       "spotify:track:02WDuAXXKumemrvsmrVBvH                                   0.0   \n",
       "spotify:track:01PLa3LP9WfuAKr3mBRnY4                                   0.0   \n",
       "spotify:track:0DIekYoiwwr1kkTUGmQlSL                                   0.0   \n",
       "\n",
       "trackid                               spotify:track:7zsw78LtXUD7JfEwH64HK2  \\\n",
       "trackid                                                                      \n",
       "spotify:track:000xQL6tZNLJzIrtIgxqSl                                   0.0   \n",
       "spotify:track:006yrnQMCZpiUgkR612gC8                                   0.0   \n",
       "spotify:track:00EhzXcoejkD5RWoS8L4IX                                   0.0   \n",
       "spotify:track:00W5ThRIn0d1bQNOp0S0af                                   0.0   \n",
       "spotify:track:02WDuAXXKumemrvsmrVBvH                                   0.0   \n",
       "spotify:track:01PLa3LP9WfuAKr3mBRnY4                                   0.0   \n",
       "spotify:track:0DIekYoiwwr1kkTUGmQlSL                                   0.0   \n",
       "\n",
       "trackid                               spotify:track:7zuwaenG5AF0vG7o7kMduX  \\\n",
       "trackid                                                                      \n",
       "spotify:track:000xQL6tZNLJzIrtIgxqSl                                   0.0   \n",
       "spotify:track:006yrnQMCZpiUgkR612gC8                                   0.0   \n",
       "spotify:track:00EhzXcoejkD5RWoS8L4IX                                   0.0   \n",
       "spotify:track:00W5ThRIn0d1bQNOp0S0af                                   0.0   \n",
       "spotify:track:02WDuAXXKumemrvsmrVBvH                                   0.0   \n",
       "spotify:track:01PLa3LP9WfuAKr3mBRnY4                                   0.0   \n",
       "spotify:track:0DIekYoiwwr1kkTUGmQlSL                                   0.0   \n",
       "\n",
       "trackid                               spotify:track:7zxRMhXxJMQCeDDg0rKAVo  \\\n",
       "trackid                                                                      \n",
       "spotify:track:000xQL6tZNLJzIrtIgxqSl                                   0.0   \n",
       "spotify:track:006yrnQMCZpiUgkR612gC8                                   0.0   \n",
       "spotify:track:00EhzXcoejkD5RWoS8L4IX                                   0.0   \n",
       "spotify:track:00W5ThRIn0d1bQNOp0S0af                                   0.0   \n",
       "spotify:track:02WDuAXXKumemrvsmrVBvH                                   0.0   \n",
       "spotify:track:01PLa3LP9WfuAKr3mBRnY4                                   0.0   \n",
       "spotify:track:0DIekYoiwwr1kkTUGmQlSL                                   0.0   \n",
       "\n",
       "trackid                               spotify:track:7zz1drChhd4hQBiGSnLRBZ  \\\n",
       "trackid                                                                      \n",
       "spotify:track:000xQL6tZNLJzIrtIgxqSl                                   0.0   \n",
       "spotify:track:006yrnQMCZpiUgkR612gC8                                   0.0   \n",
       "spotify:track:00EhzXcoejkD5RWoS8L4IX                                   0.0   \n",
       "spotify:track:00W5ThRIn0d1bQNOp0S0af                                   0.0   \n",
       "spotify:track:02WDuAXXKumemrvsmrVBvH                                   0.0   \n",
       "spotify:track:01PLa3LP9WfuAKr3mBRnY4                                   0.0   \n",
       "spotify:track:0DIekYoiwwr1kkTUGmQlSL                                   0.0   \n",
       "\n",
       "trackid                               spotify:track:7zzBEZBTJejWeL6EqWmCD9  \n",
       "trackid                                                                     \n",
       "spotify:track:000xQL6tZNLJzIrtIgxqSl                                   0.0  \n",
       "spotify:track:006yrnQMCZpiUgkR612gC8                                   0.0  \n",
       "spotify:track:00EhzXcoejkD5RWoS8L4IX                                   0.0  \n",
       "spotify:track:00W5ThRIn0d1bQNOp0S0af                                   0.0  \n",
       "spotify:track:02WDuAXXKumemrvsmrVBvH                                   0.0  \n",
       "spotify:track:01PLa3LP9WfuAKr3mBRnY4                                   0.0  \n",
       "spotify:track:0DIekYoiwwr1kkTUGmQlSL                                   0.0  \n",
       "\n",
       "[7 rows x 34443 columns]"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test = [1, 4, 10, 22, 188, 97, 998]\n",
    "cooc_2.iloc[test]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count    34443.000000\n",
       "mean         0.015455\n",
       "std          0.108496\n",
       "min          0.000000\n",
       "25%          0.000000\n",
       "50%          0.000000\n",
       "75%          0.000000\n",
       "max          1.000000\n",
       "dtype: float64"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.sum(cooc_2.iloc[test], axis=0).describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"\"\n",
    "def f(df):\n",
    "    df = df.drop(['artist_name', 'track_name'], axis=1)\n",
    "    keys,values=df.sort_values('trackid').values.T\n",
    "    ukeys,index=np.unique(keys,True)\n",
    "    arrays=np.split(values,index[1:])\n",
    "    df2=pd.DataFrame({'trackid':ukeys,'pid':[list(a) for a in arrays]})\n",
    "    return df2\n",
    "f(songPlaylist)\n",
    "\"\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # LOADING ALL 1,000,000 playlists\n",
    "# songPlaylistArray = []\n",
    "# start = 0\n",
    "# end = 1000\n",
    "# while end != 1000000:\n",
    "#     path = 'data/mpd.slice.' + str(start) + \"-\" + str(end-1) + '.json'\n",
    "#     start += 1000\n",
    "#     end += 1000\n",
    "#     d = json.load(open(path, 'r'))\n",
    "#     thisSlice = pd.DataFrame.from_dict(d['playlists'], orient='columns')\n",
    "#     for index, row in thisSlice.iterrows():\n",
    "#         for track in row['tracks']:\n",
    "#             songPlaylistArray.append([track['track_uri'], track['artist_name'], track['track_name'], row['pid']])\n",
    "# songPlaylist = pd.DataFrame(songPlaylistArray, columns=['trackid', 'artist_name', 'track_name', 'pid'])"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}