diff --git a/HW4_solutions.ipynb b/HW4_solutions.ipynb index d739837..9cf570b 100644 --- a/HW4_solutions.ipynb +++ b/HW4_solutions.ipynb @@ -3687,6 +3687,129 @@ " \n", "Note that by default mrjob does an alphanumeric sort on sim, which is not what we want. Indeed it is complicated, but possible to have this addition run locally (on mac/linux) you would need to specify the sort binary as `sort -nr`. On Hadoop, the parameters outlined [here](http://pythonhosted.org/mrjob/job.html#mrjob.job.MRJob.jobconf) can be used. Of-course, you could do the final sorting on a front end machine anyway.\n", "\n", + "UPDATE: Brandon suggests an even simpler solution that will work both locally and on EMR, without having to use any hadoop sorting specifics. \n", + "We change to:\n", + "\n", + "MAP STEP:\n", + " \n", + " def ranking_mapper(self, restaurants, values):\n", + " sim, n_common = values\n", + " rest1, rest2 = restaurants\n", + " if int(n_common) > 0:\n", + " yield (rest1), (sim, rest2, n_common)\n", + "\n", + "\n", + "REDUCE STEP:\n", + " \n", + " def top_similar_collector(self, key, values):\n", + " rest1 = key\n", + " for sim, rest2, n_common in sorted(values, reverse=True):\n", + " yield None, (rest1, rest2, sim, n_common)\n", + " \n", + "Full code:\n" + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "thecode = open(\"computesim2.py\").read()\n", + "thehtml=highlight(thecode, PythonLexer(), HtmlFormatter())\n", + "HTML(thehtml)" + ], + "language": "python", + "metadata": {}, + "outputs": [ + { + "html": [ + "
import numpy as np\n",
+        "\n",
+        "from mrjob.job import MRJob\n",
+        "from itertools import combinations, permutations\n",
+        "from math import sqrt\n",
+        "import mrjob\n",
+        "\n",
+        "from scipy.stats.stats import pearsonr\n",
+        "\n",
+        "class RestaurantSimilarities(MRJob):\n",
+        "\n",
+        "    def steps(self):\n",
+        "        thesteps = [\n",
+        "            self.mr(mapper=self.line_mapper, reducer=self.users_items_collector),\n",
+        "            self.mr(mapper=self.pair_items_mapper, reducer=self.calc_sim_collector),\n",
+        "            self.mr(mapper=self.ranking_mapper, reducer=self.top_similar_collector)\n",
+        "        ]\n",
+        "        return thesteps\n",
+        "\n",
+        "    def line_mapper(self,_,line):\n",
+        "        user_id,business_id,stars,business_avg,user_avg=line.split(',')\n",
+        "        yield user_id, (business_id,stars,business_avg,user_avg)\n",
+        "\n",
+        "    def users_items_collector(self, user_id, values):\n",
+        "        ratings=[]\n",
+        "        for business_id,stars,business_avg,user_avg in values:\n",
+        "            ratings.append((business_id,(stars, user_avg)))\n",
+        "        yield user_id, ratings\n",
+        "\n",
+        "    def pair_items_mapper(self, user_id, values):\n",
+        "        ratings = values\n",
+        "        for biz1tuple, biz2tuple in combinations(ratings, 2):\n",
+        "            biz1, biz1r=biz1tuple\n",
+        "            biz2, biz2r=biz2tuple\n",
+        "            if biz1 <= biz2 :\n",
+        "                yield (biz1, biz2), (biz1r, biz2r)\n",
+        "            else:\n",
+        "                yield (biz2, biz1), (biz2r, biz1r)\n",
+        "\n",
+        "    def calc_sim_collector(self, key, values):\n",
+        "        (rest1, rest2), common_ratings = key, values\n",
+        "        diff1=[]\n",
+        "        diff2=[]\n",
+        "        n_common=0\n",
+        "\n",
+        "\n",
+        "        for rt1, rt2 in common_ratings:\n",
+        "            diff1.append(float(rt1[0])-float(rt1[1]))\n",
+        "            diff2.append(float(rt2[0])-float(rt2[1]))\n",
+        "            n_common=n_common+1\n",
+        "        if n_common==0:\n",
+        "            rho=0.\n",
+        "        else:\n",
+        "            rho=pearsonr(diff1, diff2)[0]\n",
+        "            if np.isnan(rho):\n",
+        "                rho=0.\n",
+        "        yield (rest1, rest2), (rho, n_common)\n",
+        "\n",
+        "    def ranking_mapper(self, restaurants, values):\n",
+        "        sim, n_common = values\n",
+        "        rest1, rest2 = restaurants\n",
+        "        if int(n_common) > 0:\n",
+        "            yield (rest1), (sim, rest2, n_common)\n",
+        "\n",
+        "    def top_similar_collector(self, key, values):\n",
+        "        rest1 = key\n",
+        "        for sim, rest2, n_common in sorted(values, reverse=True):\n",
+        "            yield None, (rest1, rest2, sim, n_common)\n",
+        "\n",
+        "#Below MUST be there for things to work!\n",
+        "if __name__ == '__main__':\n",
+        "    RestaurantSimilarities.run()\n",
+        "
\n" + ], + "metadata": {}, + "output_type": "pyout", + "prompt_number": 4, + "text": [ + "" + ] + } + ], + "prompt_number": 4 + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ "To complete the recommender to the level of 1.7, we can now take the user's top restaurants, and repeat the process with the output of `top_similar_collector`, which could be stored in a hash table with restaurant keys and arrays of nearest neighbors." ] }, diff --git a/computesim.py b/computesim.py new file mode 100644 index 0000000..af9c29d --- /dev/null +++ b/computesim.py @@ -0,0 +1,60 @@ +import numpy as np + +from mrjob.job import MRJob +from itertools import combinations, permutations +from math import sqrt + +from scipy.stats.stats import pearsonr + +class RestaurantSimilarities(MRJob): + + def steps(self): + thesteps = [ + self.mr(mapper=self.line_mapper, reducer=self.users_items_collector), + self.mr(mapper=self.pair_items_mapper, reducer=self.calc_sim_collector) + ] + return thesteps + + def line_mapper(self,_,line): + user_id,business_id,stars,business_avg,user_avg=line.split(',') + yield user_id, (business_id,stars,business_avg,user_avg) + + def users_items_collector(self, user_id, values): + ratings=[] + for business_id,stars,business_avg,user_avg in values: + ratings.append((business_id,(stars, user_avg))) + yield user_id, ratings + + def pair_items_mapper(self, user_id, values): + ratings = values + for biz1tuple, biz2tuple in combinations(ratings, 2): + biz1, biz1r=biz1tuple + biz2, biz2r=biz2tuple + if biz1 <= biz2 : + yield (biz1, biz2), (biz1r, biz2r) + else: + yield (biz2, biz1), (biz2r, biz1r) + + def calc_sim_collector(self, key, values): + (rest1, rest2), common_ratings = key, values + diff1=[] + diff2=[] + n_common=0 + + + for rt1, rt2 in common_ratings: + diff1.append(float(rt1[0])-float(rt1[1])) + diff2.append(float(rt2[0])-float(rt2[1])) + n_common=n_common+1 + if n_common==0: + rho=0. + else: + rho=pearsonr(diff1, diff2)[0] + if np.isnan(rho): + rho=0. + yield (rest1, rest2), (rho, n_common) + + +#Below MUST be there for things to work! +if __name__ == '__main__': + RestaurantSimilarities.run() diff --git a/computesim2.py b/computesim2.py new file mode 100644 index 0000000..a756f9e --- /dev/null +++ b/computesim2.py @@ -0,0 +1,72 @@ +import numpy as np + +from mrjob.job import MRJob +from itertools import combinations, permutations +from math import sqrt +import mrjob + +from scipy.stats.stats import pearsonr + +class RestaurantSimilarities(MRJob): + + def steps(self): + thesteps = [ + self.mr(mapper=self.line_mapper, reducer=self.users_items_collector), + self.mr(mapper=self.pair_items_mapper, reducer=self.calc_sim_collector), + self.mr(mapper=self.ranking_mapper, reducer=self.top_similar_collector) + ] + return thesteps + + def line_mapper(self,_,line): + user_id,business_id,stars,business_avg,user_avg=line.split(',') + yield user_id, (business_id,stars,business_avg,user_avg) + + def users_items_collector(self, user_id, values): + ratings=[] + for business_id,stars,business_avg,user_avg in values: + ratings.append((business_id,(stars, user_avg))) + yield user_id, ratings + + def pair_items_mapper(self, user_id, values): + ratings = values + for biz1tuple, biz2tuple in combinations(ratings, 2): + biz1, biz1r=biz1tuple + biz2, biz2r=biz2tuple + if biz1 <= biz2 : + yield (biz1, biz2), (biz1r, biz2r) + else: + yield (biz2, biz1), (biz2r, biz1r) + + def calc_sim_collector(self, key, values): + (rest1, rest2), common_ratings = key, values + diff1=[] + diff2=[] + n_common=0 + + + for rt1, rt2 in common_ratings: + diff1.append(float(rt1[0])-float(rt1[1])) + diff2.append(float(rt2[0])-float(rt2[1])) + n_common=n_common+1 + if n_common==0: + rho=0. + else: + rho=pearsonr(diff1, diff2)[0] + if np.isnan(rho): + rho=0. + yield (rest1, rest2), (rho, n_common) + + def ranking_mapper(self, restaurants, values): + sim, n_common = values + rest1, rest2 = restaurants + if int(n_common) > 0: + yield (rest1), (sim, rest2, n_common) + + def top_similar_collector(self, key, values): + rest1 = key + for sim, rest2, n_common in sorted(values, reverse=True): + yield None, (rest1, rest2, sim, n_common) + +#Below MUST be there for things to work! +if __name__ == '__main__': + RestaurantSimilarities.run()