Spark_ALS.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "kvD4HBMi0ohY"
   },
   "source": [
    "# Install Java, Spark, and Findspark\n",
    "This installs Apache Spark 2.3.2, Java 8, and [Findspark](https://github.com/minrk/findspark), a library that makes it easy for Python to find Spark."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "fUhBhrGmyAvs"
   },
   "outputs": [],
   "source": [
    "!apt-get install openjdk-8-jdk-headless -qq > /dev/null\n",
    "!wget -q http://www-eu.apache.org/dist/spark/spark-2.4.4/spark-2.4.4-bin-hadoop2.7.tgz\n",
    "!tar xf spark-2.4.4-bin-hadoop2.7.tgz\n",
    "!pip install -q findspark"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 68
    },
    "colab_type": "code",
    "id": "5hhXcL2PLMGu",
    "outputId": "0771bcc4-a25d-46e1-eae4-25f9fb263e29"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: py4j in /usr/local/lib/python3.6/dist-packages (0.10.7)\n",
      "Requirement already satisfied: pyspark in /usr/local/lib/python3.6/dist-packages (2.4.4)\n",
      "Requirement already satisfied: py4j==0.10.7 in /usr/local/lib/python3.6/dist-packages (from pyspark) (0.10.7)\n"
     ]
    }
   ],
   "source": [
    "!pip install py4j\n",
    "!pip install pyspark"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "b4Kjvk_h1AHl"
   },
   "source": [
    "# Set Environment Variables\n",
    "Set the locations where Spark and Java are installed."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "8Xnb_ePUyQIL"
   },
   "outputs": [],
   "source": [
    "import os\n",
    "os.environ[\"JAVA_HOME\"] = \"/usr/lib/jvm/java-8-openjdk-amd64\"\n",
    "os.environ[\"SPARK_HOME\"] = \"/content/spark-2.4.4-bin-hadoop2.7\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "NwU28K5f1H3P"
   },
   "source": [
    "# Start a SparkSession\n",
    "This will start a local Spark session."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "lsMpgjgaAbuI"
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "b3xI5hrK_p4l"
   },
   "outputs": [],
   "source": [
    "# spark imports\n",
    "from pyspark.sql import SparkSession\n",
    "from pyspark.sql.functions import UserDefinedFunction, explode, desc\n",
    "from pyspark.sql.types import StringType, ArrayType\n",
    "from pyspark.mllib.recommendation import ALS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "q-m8p5V9AeSp"
   },
   "outputs": [],
   "source": [
    "# data science imports\n",
    "import math\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "# visualization imports\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "zgReRGl0y23D"
   },
   "outputs": [],
   "source": [
    "import findspark\n",
    "findspark.init(\"spark-2.4.4-bin-hadoop2.7\")# SPARK_HOME\n",
    "\n",
    "from pyspark.sql import SparkSession\n",
    "spark = SparkSession.builder.master(\"local[*]\").getOrCreate()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "7kI3tH14nyTG",
    "outputId": "30862113-2b9f-4788-ecb8-fb0da28fd64f"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Drive already mounted at /content/gdrive; to attempt to forcibly remount, call drive.mount(\"/content/gdrive\", force_remount=True).\n"
     ]
    }
   ],
   "source": [
    "from google.colab import drive\n",
    "drive.mount('/content/gdrive')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "PacPhfuan_6r"
   },
   "outputs": [],
   "source": [
    "!cp /content/gdrive/My\\ Drive/MscsDs/msdchallenge/kaggle_visible_evaluation_triplets.txt ."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "3q9GWx_2oFLS"
   },
   "outputs": [],
   "source": [
    "!cp -r /content/gdrive/My\\ Drive/MscsDs/MSongsDB-master/PythonSrc ."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "pyiYETN8oQbG"
   },
   "outputs": [],
   "source": [
    "# ! cd /content/gdrive/My\\ Drive/MscsDs/ && tar -zxvf millionsongsubset_full.tar.gz"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 204
    },
    "colab_type": "code",
    "id": "MDD5vViroUvk",
    "outputId": "5455583c-d3f8-4596-cef6-9c592942a26f"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>song_ids</th>\n",
       "      <th>play_count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>fd50c4007b68a3737fe052d5a4f78ce8aa117f3d</td>\n",
       "      <td>SOBONKR12A58A7A7E0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>fd50c4007b68a3737fe052d5a4f78ce8aa117f3d</td>\n",
       "      <td>SOEGIYH12A6D4FC0E3</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>fd50c4007b68a3737fe052d5a4f78ce8aa117f3d</td>\n",
       "      <td>SOFLJQZ12A6D4FADA6</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>fd50c4007b68a3737fe052d5a4f78ce8aa117f3d</td>\n",
       "      <td>SOHTKMO12AB01843B0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>fd50c4007b68a3737fe052d5a4f78ce8aa117f3d</td>\n",
       "      <td>SODQZCY12A6D4F9D11</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                    user_id            song_ids  play_count\n",
       "0  fd50c4007b68a3737fe052d5a4f78ce8aa117f3d  SOBONKR12A58A7A7E0           1\n",
       "1  fd50c4007b68a3737fe052d5a4f78ce8aa117f3d  SOEGIYH12A6D4FC0E3           1\n",
       "2  fd50c4007b68a3737fe052d5a4f78ce8aa117f3d  SOFLJQZ12A6D4FADA6           1\n",
       "3  fd50c4007b68a3737fe052d5a4f78ce8aa117f3d  SOHTKMO12AB01843B0           1\n",
       "4  fd50c4007b68a3737fe052d5a4f78ce8aa117f3d  SODQZCY12A6D4F9D11           1"
      ]
     },
     "execution_count": 112,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "users = pd.read_table('./kaggle_visible_evaluation_triplets.txt', sep='\\t', names= ['user_id','song_ids','play_count'])\n",
    "users.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 255
    },
    "colab_type": "code",
    "id": "hwFRfTFypgyK",
    "outputId": "edf18945-b6ec-4fb7-a854-61cca652a23a"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                                          user_id  ... play_count\n",
      "0        fd50c4007b68a3737fe052d5a4f78ce8aa117f3d  ...          1\n",
      "1        fd50c4007b68a3737fe052d5a4f78ce8aa117f3d  ...          1\n",
      "2        fd50c4007b68a3737fe052d5a4f78ce8aa117f3d  ...          1\n",
      "3        fd50c4007b68a3737fe052d5a4f78ce8aa117f3d  ...          1\n",
      "4        fd50c4007b68a3737fe052d5a4f78ce8aa117f3d  ...          1\n",
      "...                                           ...  ...        ...\n",
      "1450928  5e650759ebf89012044c6d52121eeada8b0ec814  ...          1\n",
      "1450929  5e650759ebf89012044c6d52121eeada8b0ec814  ...          2\n",
      "1450930  5e650759ebf89012044c6d52121eeada8b0ec814  ...          2\n",
      "1450931  5e650759ebf89012044c6d52121eeada8b0ec814  ...          2\n",
      "1450932  5e650759ebf89012044c6d52121eeada8b0ec814  ...          3\n",
      "\n",
      "[1450933 rows x 3 columns]\n"
     ]
    }
   ],
   "source": [
    "from pandas import DataFrame\n",
    "\n",
    "temp = DataFrame(users, columns= ['user_id', 'song_ids','play_count'])\n",
    "export_csv = temp.to_csv (r'./triplets.csv', index = None, header=True)\n",
    "print(temp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 390
    },
    "colab_type": "code",
    "id": "nWV9yDHGyOUl",
    "outputId": "756718d4-1075-4346-f270-823bcdaba313"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>song_ids</th>\n",
       "      <th>song_id_code</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>SOBONKR12A58A7A7E0</td>\n",
       "      <td>10546</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>SOEGIYH12A6D4FC0E3</td>\n",
       "      <td>28684</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>SOFLJQZ12A6D4FADA6</td>\n",
       "      <td>36622</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>SOHTKMO12AB01843B0</td>\n",
       "      <td>51861</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>SODQZCY12A6D4F9D11</td>\n",
       "      <td>24663</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>SOXLOQG12AF72A2D55</td>\n",
       "      <td>148312</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>SOUVUHC12A67020E3B</td>\n",
       "      <td>133075</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>SOUQERE12A58A75633</td>\n",
       "      <td>131771</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>SOIPJAX12A8C141A2D</td>\n",
       "      <td>57332</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>SOEFCDJ12AB0185FA0</td>\n",
       "      <td>28367</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>SOATCSU12A8C13393A</td>\n",
       "      <td>4975</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              song_ids  song_id_code\n",
       "0   SOBONKR12A58A7A7E0         10546\n",
       "1   SOEGIYH12A6D4FC0E3         28684\n",
       "2   SOFLJQZ12A6D4FADA6         36622\n",
       "3   SOHTKMO12AB01843B0         51861\n",
       "4   SODQZCY12A6D4F9D11         24663\n",
       "5   SOXLOQG12AF72A2D55        148312\n",
       "6   SOUVUHC12A67020E3B        133075\n",
       "7   SOUQERE12A58A75633        131771\n",
       "8   SOIPJAX12A8C141A2D         57332\n",
       "9   SOEFCDJ12AB0185FA0         28367\n",
       "10  SOATCSU12A8C13393A          4975"
      ]
     },
     "execution_count": 114,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.preprocessing import LabelEncoder\n",
    "\n",
    "frame = temp\n",
    "lb_make = LabelEncoder()\n",
    "frame[\"user_id_code\"] = lb_make.fit_transform(frame[\"user_id\"])\n",
    "frame[[\"user_id\", \"user_id_code\"]].head(11)\n",
    "\n",
    "lb_make2 = LabelEncoder()\n",
    "frame[\"song_id_code\"] = lb_make2.fit_transform(frame[\"song_ids\"])\n",
    "frame[[\"song_ids\", \"song_id_code\"]].head(11)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 419
    },
    "colab_type": "code",
    "id": "urnaD1IpyZZ6",
    "outputId": "6103c07a-bb7c-4268-9234-f11dcf71c2f1"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>song_ids</th>\n",
       "      <th>play_count</th>\n",
       "      <th>user_id_code</th>\n",
       "      <th>song_id_code</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>fd50c4007b68a3737fe052d5a4f78ce8aa117f3d</td>\n",
       "      <td>SOBONKR12A58A7A7E0</td>\n",
       "      <td>1</td>\n",
       "      <td>108811</td>\n",
       "      <td>10546</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>fd50c4007b68a3737fe052d5a4f78ce8aa117f3d</td>\n",
       "      <td>SOEGIYH12A6D4FC0E3</td>\n",
       "      <td>1</td>\n",
       "      <td>108811</td>\n",
       "      <td>28684</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>fd50c4007b68a3737fe052d5a4f78ce8aa117f3d</td>\n",
       "      <td>SOFLJQZ12A6D4FADA6</td>\n",
       "      <td>1</td>\n",
       "      <td>108811</td>\n",
       "      <td>36622</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>fd50c4007b68a3737fe052d5a4f78ce8aa117f3d</td>\n",
       "      <td>SOHTKMO12AB01843B0</td>\n",
       "      <td>1</td>\n",
       "      <td>108811</td>\n",
       "      <td>51861</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>fd50c4007b68a3737fe052d5a4f78ce8aa117f3d</td>\n",
       "      <td>SODQZCY12A6D4F9D11</td>\n",
       "      <td>1</td>\n",
       "      <td>108811</td>\n",
       "      <td>24663</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1450928</th>\n",
       "      <td>5e650759ebf89012044c6d52121eeada8b0ec814</td>\n",
       "      <td>SOVLNXV12A6D4F706E</td>\n",
       "      <td>1</td>\n",
       "      <td>40490</td>\n",
       "      <td>136574</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1450929</th>\n",
       "      <td>5e650759ebf89012044c6d52121eeada8b0ec814</td>\n",
       "      <td>SOVDSJC12A58A7A271</td>\n",
       "      <td>2</td>\n",
       "      <td>40490</td>\n",
       "      <td>134757</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1450930</th>\n",
       "      <td>5e650759ebf89012044c6d52121eeada8b0ec814</td>\n",
       "      <td>SOBRHVR12A8C133F35</td>\n",
       "      <td>2</td>\n",
       "      <td>40490</td>\n",
       "      <td>11310</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1450931</th>\n",
       "      <td>5e650759ebf89012044c6d52121eeada8b0ec814</td>\n",
       "      <td>SOMGVYU12A8C1314FF</td>\n",
       "      <td>2</td>\n",
       "      <td>40490</td>\n",
       "      <td>80823</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1450932</th>\n",
       "      <td>5e650759ebf89012044c6d52121eeada8b0ec814</td>\n",
       "      <td>SOTCMDJ12A6D4F8528</td>\n",
       "      <td>3</td>\n",
       "      <td>40490</td>\n",
       "      <td>122750</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1450933 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                          user_id  ... song_id_code\n",
       "0        fd50c4007b68a3737fe052d5a4f78ce8aa117f3d  ...        10546\n",
       "1        fd50c4007b68a3737fe052d5a4f78ce8aa117f3d  ...        28684\n",
       "2        fd50c4007b68a3737fe052d5a4f78ce8aa117f3d  ...        36622\n",
       "3        fd50c4007b68a3737fe052d5a4f78ce8aa117f3d  ...        51861\n",
       "4        fd50c4007b68a3737fe052d5a4f78ce8aa117f3d  ...        24663\n",
       "...                                           ...  ...          ...\n",
       "1450928  5e650759ebf89012044c6d52121eeada8b0ec814  ...       136574\n",
       "1450929  5e650759ebf89012044c6d52121eeada8b0ec814  ...       134757\n",
       "1450930  5e650759ebf89012044c6d52121eeada8b0ec814  ...        11310\n",
       "1450931  5e650759ebf89012044c6d52121eeada8b0ec814  ...        80823\n",
       "1450932  5e650759ebf89012044c6d52121eeada8b0ec814  ...       122750\n",
       "\n",
       "[1450933 rows x 5 columns]"
      ]
     },
     "execution_count": 116,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "frame"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "CQaWupmeyiWr"
   },
   "outputs": [],
   "source": [
    "export_csv = frame.to_csv (r'./triplets.csv', index = None, header=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "r-8o3ceGpgiM"
   },
   "outputs": [],
   "source": [
    "# spark imports\n",
    "from pyspark.ml.feature import StringIndexer\n",
    "from pyspark.ml import Pipeline\n",
    "from pyspark.sql.functions import col\n",
    "from pyspark.ml.evaluation import RegressionEvaluator\n",
    "from pyspark.ml.recommendation import ALS\n",
    "from pyspark.sql import SparkSession\n",
    "from pyspark.sql.functions import UserDefinedFunction, explode, desc\n",
    "from pyspark.sql.types import StringType, ArrayType"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "-d4BjZ-ipoVm"
   },
   "outputs": [],
   "source": [
    "spark = SparkSession \\\n",
    "    .builder \\\n",
    "    .appName(\"Music Recommender System\") \\\n",
    "    .config(\"spark.driver.maxResultSize\", \"96g\") \\\n",
    "    .config(\"spark.driver.memory\", \"96g\") \\\n",
    "    .config(\"spark.executor.memory\", \"8g\") \\\n",
    "    .config(\"spark.master\", \"local[12]\") \\\n",
    "    .getOrCreate()\n",
    "# get spark context\n",
    "sc = spark.sparkContext"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 289
    },
    "colab_type": "code",
    "id": "85GstFIJpqHy",
    "outputId": "595f1bfc-ab43-4c00-c7c7-626a5abd8ef1"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+----------------------------------------+------------------+----------+------------+------------+\n",
      "|user_id                                 |song_ids          |play_count|user_id_code|song_id_code|\n",
      "+----------------------------------------+------------------+----------+------------+------------+\n",
      "|fd50c4007b68a3737fe052d5a4f78ce8aa117f3d|SOBONKR12A58A7A7E0|1         |108811      |10546       |\n",
      "|fd50c4007b68a3737fe052d5a4f78ce8aa117f3d|SOEGIYH12A6D4FC0E3|1         |108811      |28684       |\n",
      "|fd50c4007b68a3737fe052d5a4f78ce8aa117f3d|SOFLJQZ12A6D4FADA6|1         |108811      |36622       |\n",
      "|fd50c4007b68a3737fe052d5a4f78ce8aa117f3d|SOHTKMO12AB01843B0|1         |108811      |51861       |\n",
      "|fd50c4007b68a3737fe052d5a4f78ce8aa117f3d|SODQZCY12A6D4F9D11|1         |108811      |24663       |\n",
      "|fd50c4007b68a3737fe052d5a4f78ce8aa117f3d|SOXLOQG12AF72A2D55|1         |108811      |148312      |\n",
      "|d7083f5e1d50c264277d624340edaaf3dc16095b|SOUVUHC12A67020E3B|1         |92390       |133075      |\n",
      "|d7083f5e1d50c264277d624340edaaf3dc16095b|SOUQERE12A58A75633|1         |92390       |131771      |\n",
      "|d7083f5e1d50c264277d624340edaaf3dc16095b|SOIPJAX12A8C141A2D|1         |92390       |57332       |\n",
      "|d7083f5e1d50c264277d624340edaaf3dc16095b|SOEFCDJ12AB0185FA0|2         |92390       |28367       |\n",
      "+----------------------------------------+------------------+----------+------------+------------+\n",
      "only showing top 10 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "df = spark.read.load('./triplets.csv', format='csv', header=True, inferSchema=True)\n",
    "df.show(10, truncate=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "-w67_jL8qQTo"
   },
   "outputs": [],
   "source": [
    "# indexer = [StringIndexer(inputCol=column, outputCol=column+\"_index\") for column in list(set(df.columns)-set(['overall'])) ]\n",
    "# pipeline = Pipeline(stages=indexer)\n",
    "# transformed = pipeline.fit(df).transform(df)\n",
    "# transformed.show(truncate=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "zvofnE_gR8UM"
   },
   "outputs": [],
   "source": [
    "# user = list(transformed.select('user_id_index').distinct().sort(transformed[\"user_id_index\"])) # Get our unique customers\n",
    "# song = list(transformed.select('song_ids_index').distinct().sort(transformed[\"song_ids_index\"])) # Get our unique products that were purchased\n",
    "# playCount = list(transformed.select('play_count')) # All of our purchases\n",
    "\n",
    "# # rows = transformed.select('user_id_index').astype('category', categories = user).cat.codes \n",
    "# # # Get the associated row indices\n",
    "# # cols = transformed.song_ids_index.astype('category', categories = song).cat.codes \n",
    "# # Get the associated column indices\n",
    "# # playCount_sparse = sparse.csr_matrix((playCount, (user, song)), shape=(len(user), len(song)))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "OV2o-FcW_1Td"
   },
   "outputs": [],
   "source": [
    "(training,test)=df.randomSplit([0.8, 0.2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "dVzUftH1UwtU"
   },
   "outputs": [],
   "source": [
    "als=ALS(maxIter=5, regParam=0.09,rank=25,userCol=\"user_id_code\",itemCol=\"song_id_code\",ratingCol=\"play_count\",coldStartStrategy=\"drop\",nonnegative=True)\n",
    "model=als.fit(training)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 969
    },
    "colab_type": "code",
    "id": "AUHd8hwJV3OD",
    "outputId": "9fc3b00e-1c28-4dea-addc-b9f9f22bdacf"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+----------------------------------------+------------------+----------+------------+------------+----------+\n",
      "|user_id                                 |song_ids          |play_count|user_id_code|song_id_code|prediction|\n",
      "+----------------------------------------+------------------+----------+------------+------------+----------+\n",
      "|8982c3890ef7a2978292a3c986e61cb914723261|SOAAOQV12A58A7FDFA|1         |59044       |148         |1.0275742 |\n",
      "|19faa693156a63d3956f9bea38bd1ef594249144|SOABVCK12A6D4F8A92|2         |11215       |471         |0.5101284 |\n",
      "|ba6186342cce189527238d6961d782a78982ae8f|SOABVCK12A6D4F8A92|2         |80079       |471         |0.30069754|\n",
      "|cae426aea0ddb4132d197c6ca6ff8281d75a4954|SOADGAX12A8151B896|2         |87115       |833         |1.0532596 |\n",
      "|cab90d247ebe4fa99117b686e40acbfd58ca2ca0|SOAKBZR12AC468D1DD|1         |87050       |2659        |1.1064874 |\n",
      "|8439c1a5fbd468d956fceb15a7495c08bc51c54c|SOAOGOV12A8C13ACED|7         |56813       |3749        |0.71650064|\n",
      "|55b972d3bd087312c570bf501c411412c6b49665|SOAOGOV12A8C13ACED|1         |36784       |3749        |0.38686666|\n",
      "|9a336136b35549aeca2ecc0d68e5cf5cbc32455f|SOAOGOV12A8C13ACED|1         |66134       |3749        |1.3442959 |\n",
      "|dc7a1da974ed898e356d3277970ac9d4c8bc478d|SOAOGOV12A8C13ACED|1         |94737       |3749        |1.3043659 |\n",
      "|56e27943491cedc2eb4a505a080def6a5068fdf0|SOAOGOV12A8C13ACED|1         |37259       |3749        |0.29489112|\n",
      "|ab428a3ce64b66fc4ab37801b126c90de5b46b4b|SOAOGOV12A8C13ACED|3         |73494       |3749        |3.8756835 |\n",
      "|9d9e0e02d02ccdbe747d760d4840d4ae02519902|SOAOGOV12A8C13ACED|1         |67657       |3749        |2.662067  |\n",
      "|b1da390ea4cfeb03d9e80abcc28c0b0631c6b975|SOAOGOV12A8C13ACED|2         |76345       |3749        |2.692673  |\n",
      "|6615f72ac5a32426a6a528cd22685a08115a9d18|SOAOGOV12A8C13ACED|1         |43813       |3749        |1.3359375 |\n",
      "|f6b6d11dc93a20d976c195f670781952ad505d27|SOARGYR12A6D4F90C7|1         |105899      |4519        |0.6676656 |\n",
      "|07c48483fe7d14ce1ca64f38f413dc6dd4a76022|SOARGYR12A6D4F90C7|3         |3374        |4519        |0.87244946|\n",
      "|950fbfeccbc60ade6ed5964b8d324d3fed9be7c8|SOARGYR12A6D4F90C7|5         |63946       |4519        |1.4610394 |\n",
      "|7fba40fcf9a21f7bb0143f947d935b5947a51759|SOARGYR12A6D4F90C7|1         |54810       |4519        |1.7686548 |\n",
      "|8d06f67ebf98f3685726346b182812e01baccbf0|SOARGYR12A6D4F90C7|5         |60544       |4519        |2.030398  |\n",
      "|1ed19063a4f3f3ecaa4874ea4eafb63c49783e81|SOAVBMC12A6D4F745A|1         |13258       |5518        |1.7134204 |\n",
      "|0d7b286cd669f1078da7da82b650686e398791fd|SOAYQAJ12A67AE123E|1         |5830        |6466        |0.33060303|\n",
      "|b39dfda8466831031ccf4f48ed8857a858f2f17c|SOBBSNN12AF72A9C42|1         |77139       |7253        |1.0974667 |\n",
      "|388f7b7150bd020ce59b82603f7dd29f590ff8ed|SOBCAQT12A6702123E|1         |24383       |7340        |1.00141   |\n",
      "|82436cfc62dd21607c1d60a00ad7db7e4135f32f|SOBDROL12A6D4F9EED|1         |55924       |7754        |2.3457708 |\n",
      "|b0e27370b66a67cf83d5562d293dabd4311add5f|SOBFARY12A6310E34F|1         |75943       |8086        |0.89488906|\n",
      "|64aa07e24cab5618c80bdc076916190024d83f39|SOBHFYO12A6D4FAC8D|1         |43231       |8638        |1.4804353 |\n",
      "|dbfee1e9a3d1cc23a2453d56d0ff3e9162ea5b38|SOBMFOX12A6D4F5B86|1         |94550       |9900        |0.8885433 |\n",
      "|360d6dff96604c2cce95c3383d9b70c2a066ba8d|SOBNHMD12A67ADADA3|1         |23311       |10206       |1.1336254 |\n",
      "|254f817243dbd2539256bf456aa9036f5379b654|SOBNHMD12A67ADADA3|1         |16064       |10206       |2.2536728 |\n",
      "|3069c3d629d106d80a222cedb4a20361761ef132|SOBNHMD12A67ADADA3|1         |20929       |10206       |0.88243246|\n",
      "|9b52e895436a0aa8f1290cf863a0994ac58de52e|SOBNHMD12A67ADADA3|1         |66644       |10206       |1.2767642 |\n",
      "|e57715de18deb3245af1b1e751b06929f0d58d70|SOBNHMD12A67ADADA3|1         |98575       |10206       |1.2492512 |\n",
      "|39ee1eff10a5c48be44f103f592374d53c692dfa|SOBNHMD12A67ADADA3|1         |25031       |10206       |1.0494484 |\n",
      "|a0044a2450e744509e42a696e90c5b75f5f36af3|SOBNHMD12A67ADADA3|1         |68699       |10206       |0.7490436 |\n",
      "|6e93172f31b8240b848a980036b721e5d4e56a02|SOBOUPA12A6D4F81F1|9         |47501       |10623       |0.38634884|\n",
      "|61338bbdf424e8f8f986ef8f1315aa4df5476a1d|SOBOUPA12A6D4F81F1|1         |41704       |10623       |0.18352605|\n",
      "|99a61c348645b8153dcc9982ec3853d5a08da0e1|SOBOUPA12A6D4F81F1|1         |65905       |10623       |4.766694  |\n",
      "|10d63816e42097e44b2b05e2f62ef3b3acebe0ee|SOBOUPA12A6D4F81F1|1         |7281        |10623       |2.4408543 |\n",
      "|346e890cd7f82b0386ec8d971004b9252c083455|SOBOUPA12A6D4F81F1|1         |22609       |10623       |4.8452806 |\n",
      "|053b73c5c8b38f195ecab4a6858c832f18e52876|SOBOUPA12A6D4F81F1|3         |2259        |10623       |3.497225  |\n",
      "|327712b9a51091c064b3ded8c63569722e808f0b|SOBOUPA12A6D4F81F1|1         |21769       |10623       |0.91443443|\n",
      "|2fe09ed614ae7b3e5e1e991b0c42636a93fa5aca|SOBOUPA12A6D4F81F1|38        |20671       |10623       |2.1283727 |\n",
      "|4ae4df63d5d70b291dd507da656acf7676105bd4|SOBOUPA12A6D4F81F1|3         |32199       |10623       |1.3279399 |\n",
      "|92e22578b1dd365da69bf87cd197bfdfb3841a9e|SOBOUPA12A6D4F81F1|1         |63033       |10623       |0.66111124|\n",
      "|cf7f599565576f0d0d64fba61227d32966601365|SOBOUPA12A6D4F81F1|2         |89109       |10623       |1.5132858 |\n",
      "|f7b899ab523efde466a1774c4961852938d5f1af|SOBOUPA12A6D4F81F1|3         |106347      |10623       |3.5346136 |\n",
      "|1fe40010162258712013a9a466324f491a3ad288|SOBOUPA12A6D4F81F1|5         |13701       |10623       |2.0195777 |\n",
      "|d0f781e47e4777b4bac0c77dd6c170eb833bf068|SOBOUPA12A6D4F81F1|2         |89772       |10623       |1.8618262 |\n",
      "|81c301ce91b185cea96b633e3105887ae4a72fe1|SOBOUPA12A6D4F81F1|9         |55704       |10623       |3.842008  |\n",
      "|9d8e4741ca9d8e78acd7b3081d022a4bbacd955c|SOBOUPA12A6D4F81F1|1         |67628       |10623       |1.1754149 |\n",
      "+----------------------------------------+------------------+----------+------------+------------+----------+\n",
      "only showing top 50 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "evaluator=RegressionEvaluator(metricName=\"rmse\",labelCol=\"play_count\",predictionCol=\"prediction\")\n",
    "predictions=model.transform(test)\n",
    "predictions.show(50, truncate = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "ELdFbYzQN6mO"
   },
   "outputs": [],
   "source": [
    "# rmse = evaluator.evaluate(predictions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "uIG74SmqaVmo"
   },
   "outputs": [],
   "source": [
    "# print(\"Root-mean-square error = \" + str(rmse))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 309
    },
    "colab_type": "code",
    "id": "ZkcnWBalB84v",
    "outputId": "efe636ea-059e-4ec8-a37b-ea0f7b5a6be0"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+\n",
      "|user_id_index|recommendations                                                                                                                                                                                              |\n",
      "+-------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+\n",
      "|148          |[[132559, 128.63263], [147680, 83.976524], [66419, 82.88117], [125845, 76.72285], [82226, 76.19292], [39411, 76.14007], [162509, 75.64509], [99909, 63.91176], [98323, 63.563046], [117480, 62.751663]]      |\n",
      "|463          |[[50694, 81.836334], [39411, 64.48308], [147680, 64.23534], [122151, 61.286102], [91642, 56.230762], [18266, 45.621635], [132559, 45.549026], [162509, 43.640633], [125845, 39.744972], [28504, 38.03746]]   |\n",
      "|471          |[[91642, 166.9629], [90521, 82.83951], [118079, 81.80899], [90425, 70.81573], [28504, 69.14364], [125845, 65.72644], [130593, 63.165348], [147680, 60.90121], [123867, 60.728004], [116217, 54.02317]]       |\n",
      "|496          |[[98323, 561.9435], [159673, 237.96635], [29924, 204.07669], [130289, 202.01738], [162509, 188.40092], [64530, 180.07327], [96723, 176.96783], [47412, 176.5399], [117683, 176.40698], [94070, 173.90211]]   |\n",
      "|833          |[[125845, 261.29282], [162509, 211.07877], [91642, 209.26482], [90521, 200.1382], [39411, 183.05717], [132559, 174.52623], [50694, 170.21997], [147680, 165.52065], [82226, 135.1852], [93177, 130.17105]]   |\n",
      "|1088         |[[91642, 477.74738], [132559, 301.17938], [98323, 163.98624], [123867, 163.49843], [130593, 150.79263], [147680, 148.13669], [28504, 145.4549], [90521, 139.62958], [162509, 136.77386], [122151, 136.32388]]|\n",
      "|1238         |[[18266, 157.90025], [91642, 144.36684], [50694, 119.0692], [122151, 78.154465], [11315, 73.731895], [62883, 68.069824], [36371, 66.961914], [154520, 66.59154], [9336, 64.87565], [63740, 64.10445]]        |\n",
      "|1342         |[[132559, 215.22742], [32380, 146.98671], [123867, 143.82867], [40945, 137.36111], [117480, 121.496414], [36371, 117.13508], [91642, 107.41648], [21660, 107.31862], [98323, 104.51557], [46713, 104.25622]] |\n",
      "|1580         |[[132559, 347.78375], [98323, 195.82375], [159673, 132.96431], [51399, 103.85994], [64530, 102.61894], [162509, 90.60711], [90425, 89.49118], [23038, 87.83676], [36371, 78.12712], [36489, 73.4358]]        |\n",
      "|1591         |[[50694, 125.0975], [122151, 99.453705], [88869, 85.341286], [132559, 70.60342], [91642, 66.271194], [80202, 65.54762], [147680, 63.61119], [162509, 47.968006], [66419, 47.93394], [47290, 46.066273]]      |\n",
      "+-------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+\n",
      "only showing top 10 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# user_recs=model.recommendForAllUsers(10).show(10, truncate = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "skM3poAyKgfu"
   },
   "outputs": [],
   "source": [
    "recs=model.recommendForAllUsers(10).toPandas()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 419
    },
    "colab_type": "code",
    "id": "7AiEvI4Bmnh3",
    "outputId": "2e860d9a-f71f-4dbc-c8d1-c56cb64b77ee"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id_code</th>\n",
       "      <th>recommendations</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>148</td>\n",
       "      <td>[(86930, 164.18919372558594), (130976, 143.074...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>463</td>\n",
       "      <td>[(140084, 59.36147689819336), (20335, 54.54523...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>471</td>\n",
       "      <td>[(140084, 346.9989013671875), (20335, 337.6782...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>496</td>\n",
       "      <td>[(86930, 102.5612564086914), (20335, 101.52572...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>833</td>\n",
       "      <td>[(39155, 280.4085388183594), (130976, 217.9136...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>109984</th>\n",
       "      <td>108934</td>\n",
       "      <td>[(86930, 162.45037841796875), (3934, 160.69415...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>109985</th>\n",
       "      <td>108946</td>\n",
       "      <td>[(140084, 271.0825500488281), (20335, 264.0805...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>109986</th>\n",
       "      <td>108961</td>\n",
       "      <td>[(140084, 114.94924926757812), (20335, 99.6657...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>109987</th>\n",
       "      <td>109144</td>\n",
       "      <td>[(20335, 165.97198486328125), (86930, 129.8025...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>109988</th>\n",
       "      <td>109297</td>\n",
       "      <td>[(20335, 122.2970199584961), (86930, 58.789989...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>109989 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        user_id_code                                    recommendations\n",
       "0                148  [(86930, 164.18919372558594), (130976, 143.074...\n",
       "1                463  [(140084, 59.36147689819336), (20335, 54.54523...\n",
       "2                471  [(140084, 346.9989013671875), (20335, 337.6782...\n",
       "3                496  [(86930, 102.5612564086914), (20335, 101.52572...\n",
       "4                833  [(39155, 280.4085388183594), (130976, 217.9136...\n",
       "...              ...                                                ...\n",
       "109984        108934  [(86930, 162.45037841796875), (3934, 160.69415...\n",
       "109985        108946  [(140084, 271.0825500488281), (20335, 264.0805...\n",
       "109986        108961  [(140084, 114.94924926757812), (20335, 99.6657...\n",
       "109987        109144  [(20335, 165.97198486328125), (86930, 129.8025...\n",
       "109988        109297  [(20335, 122.2970199584961), (86930, 58.789989...\n",
       "\n",
       "[109989 rows x 2 columns]"
      ]
     },
     "execution_count": 98,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "recs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "P8-lZYvERY3i",
    "outputId": "6313c723-56c7-4dc6-dab7-b8997a58a221"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(109989, 2)"
      ]
     },
     "execution_count": 35,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "recs.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "FZ7pGPqH6KWM"
   },
   "outputs": [],
   "source": [
    "song_df = pd.read_csv('/content/gdrive/My Drive/MscsDs/MillionSongSubset/msd.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 615
    },
    "colab_type": "code",
    "id": "eTlNUFt46Pe8",
    "outputId": "a51c53a8-2c1d-4e32-e8ff-d1b505dd92ab"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>Unnamed: 0.1</th>\n",
       "      <th>song_ids</th>\n",
       "      <th>song_titles</th>\n",
       "      <th>song_durations</th>\n",
       "      <th>song_realease_years</th>\n",
       "      <th>artist_names</th>\n",
       "      <th>song_hotness</th>\n",
       "      <th>song_tempo</th>\n",
       "      <th>song_energies</th>\n",
       "      <th>song_danceability</th>\n",
       "      <th>song_bars</th>\n",
       "      <th>song_beats</th>\n",
       "      <th>song_time_signatures</th>\n",
       "      <th>song_tatum</th>\n",
       "      <th>song_modes</th>\n",
       "      <th>song_keys</th>\n",
       "      <th>artist_familiarty</th>\n",
       "      <th>artist_hotness</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>SONHTWP12A8C142744</td>\n",
       "      <td>Vals '89</td>\n",
       "      <td>162.58567</td>\n",
       "      <td>0</td>\n",
       "      <td>Manolis Famellos &amp; I Podilates</td>\n",
       "      <td>NaN</td>\n",
       "      <td>94.301</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>[  1.45502   3.39383   5.32826   7.25953   9.1...</td>\n",
       "      <td>[  0.16441   0.80395   1.45502   2.09102   2.7...</td>\n",
       "      <td>3</td>\n",
       "      <td>[  0.16441   0.48418   0.80395   1.13522   1.4...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.228730</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>SOPIHMB12AC468E0DD</td>\n",
       "      <td>Ghost of the Ocean (Live)</td>\n",
       "      <td>214.25587</td>\n",
       "      <td>0</td>\n",
       "      <td>Uriah Heep</td>\n",
       "      <td>0.2502</td>\n",
       "      <td>149.921</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>[  1.20218   2.80457   4.39266   5.98514   7.5...</td>\n",
       "      <td>[  0.38752   0.79653   1.20218   1.60244   2.0...</td>\n",
       "      <td>4</td>\n",
       "      <td>[  0.2523    0.38752   0.52273 ... 212.97102 2...</td>\n",
       "      <td>1</td>\n",
       "      <td>7</td>\n",
       "      <td>0.673239</td>\n",
       "      <td>0.479205</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>SODRPJT12AC468DEF1</td>\n",
       "      <td>The frog song</td>\n",
       "      <td>169.87383</td>\n",
       "      <td>0</td>\n",
       "      <td>Alain-François</td>\n",
       "      <td>NaN</td>\n",
       "      <td>136.044</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>[  1.37397   4.48312   7.56804  10.65675  13.7...</td>\n",
       "      <td>[  0.48609   0.93058   1.37397   1.81958   2.2...</td>\n",
       "      <td>7</td>\n",
       "      <td>[  0.26328   0.48609   0.70778   0.93058   1.1...</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>0.380317</td>\n",
       "      <td>0.300918</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>SOLLOTO12AB01804C6</td>\n",
       "      <td>Deep Blue Sea (Daniel Rossen home recording)</td>\n",
       "      <td>351.50322</td>\n",
       "      <td>2008</td>\n",
       "      <td>Grizzly Bear</td>\n",
       "      <td>NaN</td>\n",
       "      <td>85.003</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>[  0.72449   3.55798   6.37712   9.20176  12.0...</td>\n",
       "      <td>[  0.72449   1.43405   2.1445    2.85228   3.5...</td>\n",
       "      <td>4</td>\n",
       "      <td>[1.8653000e-01 3.6881000e-01 5.4665000e-01 ......</td>\n",
       "      <td>1</td>\n",
       "      <td>9</td>\n",
       "      <td>0.760636</td>\n",
       "      <td>0.547244</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>SOCNYGS12AB01832B8</td>\n",
       "      <td>La vérité</td>\n",
       "      <td>252.44689</td>\n",
       "      <td>2007</td>\n",
       "      <td>Annie Blanchard</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>113.287</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>[2.2334000e-01 2.3324800e+00 4.4484000e+00 6.5...</td>\n",
       "      <td>[2.2334000e-01 7.5000000e-01 1.2775800e+00 1.8...</td>\n",
       "      <td>4</td>\n",
       "      <td>[2.2334000e-01 3.9890000e-01 5.7445000e-01 ......</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>0.479735</td>\n",
       "      <td>0.352460</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Unnamed: 0  Unnamed: 0.1  ... artist_familiarty artist_hotness\n",
       "0           0             0  ...          0.228730       0.000000\n",
       "1           1             1  ...          0.673239       0.479205\n",
       "2           2             2  ...          0.380317       0.300918\n",
       "3           3             3  ...          0.760636       0.547244\n",
       "4           4             4  ...          0.479735       0.352460\n",
       "\n",
       "[5 rows x 19 columns]"
      ]
     },
     "execution_count": 46,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "song_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 119
    },
    "colab_type": "code",
    "id": "yhmzMcDr9syW",
    "outputId": "6d141ba2-5c07-4906-d7a1-92aa0af09438"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " Recommendation to user  148  : \n",
      "   Song:  Seed Will Grow\n",
      "   Artist:  Ms. Dynamite / Kymani Marley\n",
      " Recommendation to user  1088  : \n",
      "   Song:  Till There Was You (John Creamer & Stephane K Remix)\n",
      "   Artist:  Rachael Starr\n"
     ]
    }
   ],
   "source": [
    "count = 0\n",
    "song_id_found = set()\n",
    "for recValue in recs.values:\n",
    "    for item in recValue[1]:\n",
    "      row = frame.loc[frame['song_id_code'] == item.song_id_code]\n",
    "      song_id = row.song_ids.values[0]\n",
    "      if ( song_id not in song_id_found) and (song_df.loc[ song_df['song_ids'] == song_id].shape[0] == 1):\n",
    "        rowDetails = song_df.loc[ song_df['song_ids'] == song_id]\n",
    "        print(\" Recommendation to user \", recValue[0], \" : \")\n",
    "        print(\"   Song: \", rowDetails.song_titles.values[0])  \n",
    "        print(\"   Artist: \",  rowDetails.artist_names.values[0])\n",
    "        song_id_found.add(song_id)\n",
    "        count += 1\n",
    "        \n",
    "    if count > 1:\n",
    "      break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 408
    },
    "colab_type": "code",
    "id": "WyW00de2BjBT",
    "outputId": "b1a5fc1b-b9e8-44b3-8954-55c47111051a"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " Recommendation to user 148 : \n",
      "\n",
      "   Song ID:  SONGFCZ12AB01859AD\n",
      "\n",
      "   Song ID:  SOUMPJA12AAF3B3FD0\n",
      "\n",
      "   Song ID:  SOUEXOI12AB0189BA4\n",
      "\n",
      "   Song ID:  SODALYQ12AB017EED4\n",
      "\n",
      "   Song ID:  SOWAWSK12A8C1383B2\n",
      "\n",
      "   Song ID:  SOAOXTG12A67021AA7\n",
      "     Song:  Seed Will Grow\n",
      "     Artist:  Ms. Dynamite / Kymani Marley\n",
      "\n",
      "   Song ID:  SODJYMH12A8AE46EA5\n",
      "\n",
      "   Song ID:  SOTFIZA12A8C13D80F\n",
      "\n",
      "   Song ID:  SORJSQI12A6701D62D\n",
      "\n",
      "   Song ID:  SOXTUWG12AB018A2E2\n"
     ]
    }
   ],
   "source": [
    "print(\" Recommendation to user 148 : \")\n",
    "for recom in recs.query('user_id_code == 148').recommendations:\n",
    "  for item in recom:\n",
    "    row = frame.loc[frame['song_id_code'] == item.song_id_code]\n",
    "    print(\"\\n   Song ID: \", row.song_ids.values[0])\n",
    "    song_id = row.song_ids.values[0]\n",
    "    if song_df.loc[ song_df['song_ids'] == song_id].shape[0] == 1:\n",
    "      rowDetails = song_df.loc[ song_df['song_ids'] == song_id]\n",
    "      print(\"     Song: \", rowDetails.song_titles.values[0])  \n",
    "      print(\"     Artist: \",  rowDetails.artist_names.values[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 374
    },
    "colab_type": "code",
    "id": "KgKOnwXg7xXH",
    "outputId": "d1e0ad70-bee1-4b3b-c349-fcd20fc8c8da"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " Recommendation to user 108811 : \n",
      "\n",
      "   Song ID:  SONGFCZ12AB01859AD\n",
      "\n",
      "   Song ID:  SODALYQ12AB017EED4\n",
      "\n",
      "   Song ID:  SOUMPJA12AAF3B3FD0\n",
      "\n",
      "   Song ID:  SOFVLYV12A8C145D8F\n",
      "\n",
      "   Song ID:  SOTFIZA12A8C13D80F\n",
      "\n",
      "   Song ID:  SOUEXOI12AB0189BA4\n",
      "\n",
      "   Song ID:  SORJSQI12A6701D62D\n",
      "\n",
      "   Song ID:  SOWAWSK12A8C1383B2\n",
      "\n",
      "   Song ID:  SOJLDIQ12A58A7E506\n",
      "\n",
      "   Song ID:  SOLLPYO12A8C13B38C\n"
     ]
    }
   ],
   "source": [
    "print(\" Recommendation to user 108811 : \")\n",
    "for recom in recs.query('user_id_code == 108811').recommendations:\n",
    "  for item in recom:\n",
    "    row = frame.loc[frame['song_id_code'] == item.song_id_code]\n",
    "    print(\"\\n   Song ID: \", row.song_ids.values[0])\n",
    "    song_id = row.song_ids.values[0]\n",
    "    if song_df.loc[ song_df['song_ids'] == song_id].shape[0] == 1:\n",
    "      rowDetails = song_df.loc[ song_df['song_ids'] == song_id]\n",
    "      print(\"     Song: \", rowDetails.song_titles.values[0])  \n",
    "      print(\"     Artist: \",  rowDetails.artist_names.values[0])"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "name": "Spark_ALS.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}