pytorch-mnist.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "from torch.autograd import Variable\n",
    "import numpy as np\n",
    "from nbutil import imshow_multi\n",
    "import random"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from read_mnist import read_images, read_labels\n",
    "\n",
    "def load_dataset(name):\n",
    "    def featurize(image):\n",
    "        return image.astype(float) / 255.0\n",
    "    images = np.array([featurize(image) for image in read_images(name + '-images-idx3-ubyte')])\n",
    "    labels = np.array([label for label in read_labels(name + '-labels-idx1-ubyte')])\n",
    "    return images, labels\n",
    "\n",
    "import os\n",
    "test_in, test_out = load_dataset('../data/t10k')\n",
    "train_in, train_out = load_dataset('../data/train')\n",
    "nb_classes = 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAeUAAABkCAYAAAC4qyirAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAAPYQAAD2EBqD+naQAAIABJREFUeJztnddyG1uStVfBFby3BGhAilat02ZOT0df9PW8w7zpXMzd\nRExMT0SPTkuiBS0IXwAKpuDNf6E/UwWKFCkJBEBwfxGI06EmCLDc2pk7c6U0Ho8hEAgEAoFg/hjm\n/QUEAoFAIBB8RoiyQCAQCAQLghBlgUAgEAgWBCHKAoFAIBAsCEKUBQKBQCBYEIQoCwQCgUCwIAhR\nFggEAoFgQRCiLBAIBALBgmB6yg9JkhQA8G8ArgB0nvMLLSlWABsA/mM8Hpe/983i+P804vjPH3EO\n5os4/vPl6cd/PB4/+gLw7wDG4vXTr39/yvEWx18c/yV+iXMgjv9rfj16/J+avr564s8Jvs3VjN8n\nmORqxu8TfM3VjN8nmORqxu8TTHL12A88VZRFumI6/OhxFMd/OojjP3/EOZgv4vjPl0ePoyj0EggE\nAoFgQRCiLBAIBALBgiBEWSAQCASCBeFJLVGC14fRaITJZILRaAQASJIEADAYDJAkCQaDAQaDAUaj\nkf8/ABiPxxgMBhgOhxiNRhOv4XCI4XA4l79HIBAIXgJClAX34na74ff74fF4JkRYlmXIsgybzQaX\nywWn0wlZlvl97XYbtVoNqqpC0zS02210Oh00Gg3U63XU63VqsRAIBALBHYQoC+7F4/EgkUggHo/D\naDTCYDDAbDazEHu9XkQiEYTDYbhcLn6fqqpIp9O4vb2Foiio1Wqo1WooFAoYj8doNBpClAUCgeAB\nhCgLGIvFAqvVCqvVikQigTdv3mBzc5NF2WKxwO12cxQdi8UQi8Xgdrv5d1QqFYTDYQQCARQKBZTL\nZZTLZZhMJrTbbZRKJfT7fb0pgeAZoSyH0WiEzWaD3W6H2WxGt9tFr9dDv9/n12g0mvfXfTHQcbVY\nLLBYLDCbzTCbzTCZTJAkibdr+v0+Op0Out0uhsOhuO4FjyJEWcAEAgGOjre2trC1tYW1tbWJ/WOr\n1QqbzQaHwwGPxwOz2TzxO2RZRiAQwHg8htvtRigUgqqqMBgMaDQayOVyaLVa6Pf7GAwGc/pLXw9m\nsxkWiwVOp5PPKS2YCoUCisUiKpUKqtUqWq3WvL/ui4DuB5vNhnA4jEgkgmAwCJ/PB5/PB4PBgE6n\ng06ng2KxiNvbW9ze3vJ13+/35/0nCBYYIcoCxu/3Y2dnB7///e+xvr6O9fV1xONxSJLEL6PRyC+K\nEPQrf4vFgkAgAKfTiVAohHq9jkajgV6vh3w+D6fTidFoxAVhgudDkiSYzWbY7Xb4/X68ffsWf/vb\n35BMJnF8fIyjoyOcnZ3BaDSi3W4LUX4iFCXbbDasrKxgb28Pm5ubWF1dxerqKsxmM+r1Omq1Gs7O\nzvD+/Xs0m02+7oUoC76FEOVXjizLsNvtsNlsSCaT2N3dxbt37xCNRhGNRhEOh/lnx+Mxp+VGoxEG\ngwHa7fZnv1ZdNG02m2G1WmG32zmiLhQKiEajCAaDAIDRaIROR5gEPTcWiwUOhwM+nw+JRAI7OzvY\n3d1Ft9tFpVJBoVCA1WrlKnvBt6GFqdlshsPhQDQaxfb2Ng4ODrC6uoq1tTWYzWbUajXU63UYDAaU\ny2Xc3t7yFkGn0xEp7CdAzxOTyQS73Q673Q6TyYRGo4Fmszm15wd9hslk+qprZB7bDUKUXzk+n48f\nJgcHB9jd3UUikYDH44HNZpv42fF4jF6vh263i3a7zTfHYDDgvTW73Q6XywW32w2j0QhZliFJEnw+\nHyKRCNbX1yFJEvr9Pmq12pz+6teBJEmQZZkL8+ihRgsqSrGK/eTvw2Qy8XENhUIcIft8PpjNZhiN\nRtjtdkiShGAwiEgkglgsxnv4ogPhaciyDKvVCqfTibW1Nayvr8PtduP4+BgnJyfI5XJT+RybzQan\n0wm73Y5ut8sv2mqYdRunEOVXjtfrxfb2Nv70pz9ha2sLyWSSU3Am0+TlQaKsaRpqtRpKpRJKpRK6\n3S6vZP1+PyRJgsPhgMVigSzLMJvNXBi2vr6OTqcjBHkGSJLE+8lerxc2mw0mk4lTqJ1OB+12W4jy\nd0CRsl6U19fX+Z4xm82QJAk2mw2yLLMor6ysQNM0NBqNib5+wcPQtRsMBrG/v49ff/0VkUgEZrMZ\npVJpaqJstVq5HqDZbKLZbHI75zy8FYQov0IMBgMbgwQCAayurmJvbw8rKysIh8NwOp2cxtE0jS/U\nZrPJ0bGqqlAUBaVSCb1ej1Pg0WgUg8GAI2ZKC9GK1+v1wul0wmKxzPswLD16UabMh8FgwGAwQKvV\ngqqqqFQq0DRN7O8/gr7a2u/3IxwOI5lMYmVlBX6/f6ItEAAvaPWFkVarlauzBY8jyzI8Hg/C4TDi\n8TjW19cRiUQQCARgtVqn9jk2mw0+nw8rKytotVpotVpoNptQFAXlcnnmNQBClF8h+j2aUCiEaDSK\nRCIBn8/HKWuKpJrNJi4vL3FxccEVpK1Wi1f99Xqd09eyLGN9fZ0rrylisFqtvBCgyNlgEA6vs0Cf\nvrbZbJAkCb1eD/V6HcViEblcjgvxBA9D+44OhwOJRAK7u7vY39/H2toaHA7HvL/eUmKz2eD3+xGN\nRuH1ern2YdqLGpvNhkAggHg8zjUz7XYbqVQK3W4XjUZjqp/3GEKUXyFUpOL1ehEKhRCLxRCPx2Gz\n2bjFqd/vo9VqoVqt4uTkBP/zP/+DT58+cc9lp9Ph/eXRaMSRd7lchtvtxtraGux2O7uASZIEk8kE\ni8UCk8kkRHkG3I2UaXHU6/XY0CWXywn70yegL+5KJBL43e9+h19++QXRaBR2u33eX28poQg2EolM\nXL9k9TvNz/H7/UgkEjCZTDCbzej1emi321NLkX8PMxNlipLIoIIiKABfVbqNx+OJB/9d72Rqwhf8\nGLQn5nA44HA4OPVsNBoxGAzQ7XaRz+eRzWZxc3ODw8NDpFIpXF9fc7HKYDBgj2uKIMh4hFqlSHwl\nSeKUaa1Wg6Zpoi3kGaGojiKAlZUVXnTROVBVFc1mE91ud95fd2Gx2Wz8cjqdcLlcCIVC2NnZwebm\nJlZWVuB2u7+5FUOmLW63Gz6fD+FwmI873UNUUCS2EL5ALWf6SJkW99NmPB6zrtACdjwec3RuMBhm\nWoU9M1G2WCzweDzw+XwIBoP8Aj5HZb1ej1tuRqMRKpUKyuUyVFWdcB2iSE1cwD8OWWaSgFJKSF+R\ne3V1hU+fPuHw8BDX19fI5XLQNI0vXv1CymKxIBgMIhaL8cPK5/NxsZckSeh2u1BVFblcDpVKBe12\ne96HYWmhhS9Zoa6vryOZTGIwGKBer6NcLqNSqQhBfgSXy4VwOMwOdYFAANFoFDs7O4jH4/D5fJBl\n+ZvtZGazGU6nE4FAAJqmYTwew2g0QtM0tFqtr7oYBGBPBCocjcViLMrPIYy9Xg/NZhPVapWzhdTS\nSc9H/fPuuZmpKPt8PsTjcaytrSGZTGJjYwMAeGiBPhJOp9NIp9MwmUwcNdODnFaYgh/jrihT8Qml\nrOv1Oi4vL/GPf/wDf//737nAi3qSAXxlGBIMBrG5uYmtrS3E43H4/X44HA4WfCHKs0O/PRGNRlmU\nM5kMMpkM0um0EOUn4Ha7EY/HeaEZj8cRi8W4h9/r9XIB2EOQKAeDQQwGA85gULaiVqvxvadp2gz/\nusWEBNlgMLAoR6NRuN1uyLL8LN4GJMqqqiIUCrGdsN1uhyzLMJlMGAwGM4uWZybKVJm7u7uLjY0N\nrK6uIpFIAMDE3iSlqb1eL8LhMLfQ6HtjG43G1E+O3hhjMBhw1TG1jCxT2wg9ACqVCq6vr+H1emEw\nGNDtdrnK+tOnT7i5uUG5XJ44P/dhNpt5waWPIPSVppQior6/ZTmWiwi5eLndbh4gYrVa0ev1oCgK\nbm9vhSg/AvXWJ5NJ/PLLL5zZCwQCXMl+t2XwPiwWC7xeL/r9PrfeRKNRNhep1Wq4urqC2WyeyAi+\n1j1+ChZo64WO+Wg04ixPrVabemEiLQbIbMflcn0VKc/qnMxMlO12O1ZWVvD27Vusr68jFAohEAhw\ni4berH00GiEajXK/WK/XY2Gg8X/PYQlIn9NqtZDNZpHL5VAqlaBpGn+PZaDX60FVVfR6PUiSBE3T\ncHt7i16vx+nrdDqNTCaDTqeDwWDwTRE1m83weDxYWVlBLBZjT2y60AWzQ2+tqX+wSJKEVquFYrGI\nTCaDarUqRPkRfD4ftra28Mc//pG7FagW5q7n+0PIsgyv18sLVwowKGXdaDRgt9sxGo34eUfbRK8R\nq9UKj8fDQVkoFILP54OiKFAUhQOFaQZl+s4Qqh+gSJlEeTgczuxZNtNImUR5bW0NTqcTTqeTUz93\n0wJ6gaYiiG63y6MAp53+HI/HvMdTq9VwfHwMo9HIE42WyRKSJgTRsby9vcWHDx84kqW9e3o9xrdE\nWTB7zGYzz7umFJzBYBCi/J34/X5sbm7iD3/4A/8bXdNPvbZp246Kh+g5R22FJMC1Wg35fB4AeHH8\nGqHe5EgkglAoxIM+SqXShChP89olUbZYLA+K8rQrvr/FzES53W4jn8/j+PgYjUaD02rkLkT7LVSS\nTi86KPT/OZ1OLlnXV2JTS863DhzdEHr/ZhquQP2btIptt9tot9vodrsYj8ecyl4W9FXutKelr3B/\nrBqUzoXL5ZowUXA6nSwCw+GQsw/VapVfzWZzabIOiwjthW5vbyMYDMJkMqHVaqHRaAjDkEegQlSq\nsg6FQhNDV56yp6gvWKXioNFoNOHlTD3jZrMZyWSSXfHIQnLWvbHzhNrNzGYzVlZWsLOzg+3tbWxs\nbMDhcPBi8vLyEqenp8jn81PNlFLrVTQahc/ng9VqnRjCM2tmJsqapiGdTuO3335DLpfjubz9fp8F\nkPYSaLXicDg4ona5XBPWjaPRiCO+8XjM7VbfqoQkISLB6ff7LPxms5n3k8nRhYwy6KJYRobD4cSs\nV3qIPLbvazabueH+zZs3XHGtT5eSJWej0eC5yuVyWZhVPCOSJMHj8fDwiVAoxNW+9XodqqpCVVW0\nWi0hyvcQDAZxcHCA/f197O/vIxAITNRFPOUhTRPQaFuOXmazmWstyOXOZDJhfX0dDocD8XgcJpMJ\niqLg+vr6uf/UhYG2W+x2O1ZXV/H27Vv86U9/gt/vh9VqRb1eRz6fx8XFBU5OTlAul6caIOlbB6kN\nap7MVJRvbm7Q7Xbh9/vh9Xrh9XrR6XS44IHSBm63G36/Hz6fjy3tyFyfBrVTOq7VamE8HvNJfWiv\nRy84etNx2iOiEyFJEtrtNnufNhoNlEqlpbWFHI1GHM1+DyTKyWQSb968mai4pgb/wWCAZrOJSqXC\nlnXlcvlV75k9N5Ikwe12Y3V1Fbu7uywCzWYT9XqdsxW0ABVMQqL8t7/9DSsrK9y2eZ8wPxQ1U8bp\nbk8/tU/Rf/V+AVTQqigKPnz4MJs/dkGgzIHH42FR/utf/8qZS1VVJ0R52t03drudAwwqUp0nMxNl\nekDT7FZVVWG327kN51uRstfr5X0ZipyNRiMXSozHY7hcLrhcLhbP+/aoKZ1E7VWtVgvr6+vY2NhA\nIpHgvQPgi6OVpmnfrDx+TVBDPw1339rawv7+PnZ2dhCJRHhhQ+nvarWKm5sbXF1d4eLiAoqicJpc\nmL9MF0qLyrIMt9vN9wxx13xHnIMvUPbNarUiEAjA6/XC7XZPjLTU7wffd9yorYa6Q+h/07ShwWAA\nh8MBt9sNj8cz8dLjcrmwsrKCZDLJv2PZ95edTie3yq6trcHn88FkMiGfz+P29hbX19dIpVKoVCpT\nu3ZpsIjBYODRpqFQiDVknuZGMxPlfr/PaUtVVXkPgVbsg8GA9xZoX1k/DpAEmi5qk8nEvX7kvuL1\neh+MaPUpWUpPa5qGf/3Xf4XZbEY4HObPphQ3CTe5ir126AL2+/1YXV3F1tYWDg4O8ObNm4lRj3Q+\nSZQ/fvyIVCoFRVH4WApBmC6UDqWqaxJlygjpBVmI8iTUl+rxeBAMBuH1euFyuTi9/JggA59FuVwu\nI5fLoVgsskFLq9Xi4km3280mJDTu0e128++QJIlFeXNzE7lcjg19lhkS5YODA6yvr8Pn88FoNKJS\nqeDk5AS//fYbLi4uUKlUptYvTMVdVBtDokya8ypEmR7UP9IgTweKXIq8Xi9MJhOn4sbjMae7H9oP\n0EcKelF2Op3Y2NjAYDDgjX3aZ6X9UDI2ee2QKAeDQcTjcWxsbGB7exvJZHIiZT0YDNBut1GtVpFO\np3F0dIR0Oo1yufxoe5Xgx9BP4qItII/Hg0ajwX2vtAAW1/IXJEni50o0GuUWHJfLxdazekHWi4He\nFlhfjJROp1EoFJDP59lStt/vs81mOBzmoS2j0WiiopuK9La2tjAej3noi/6zlgUyCaG/eW9vj2e5\nGwwGVKtVpFIp/OMf/0C1WoWqqlO7do1GI2sKLWL9fv9ERnVex/tFDKSgtigAaDQaGA6HXLzS6XQw\nHo9Rq9W4mOKh36GvgpRlGXa7nYe/m81mjEYjtNttnqBDDkjU0/vaoQcYpUcdDsdX/ciUCVEUBel0\nGtlsFsViEaqqTjiCCaYLtZKEQiF2PwK+bCX0ej2xINJBrnYmkwnRaBR7e3vY29vD9vY24vH4RDvM\nXSgdTYWMzWYThUIBJycnODk54WeGqqpsCTwcDrlHWdM0eDwexONxdLvdiQ4Qv9+Pra0tGAwG7oe2\n2+1sNrIs9xBNL6MBH/F4nL3Egc/PecqEVqvVqfvlu1wuRCIRRCIRJBIJuFwujMdjLjoulUqo1Wqc\n6Zil4dGLEGV9ZTAJJ3k104micVsPVV/rV5qU2qOCMxIXSlerqopSqYRsNotMJoN6vS5EGeDFDIky\nLWb0PXzU/pTNZnF7e4tcLodCocALm2V4oCwidF70oqx3USNxEMf/M/pIKRaLYW9vD3/9618RDocR\nDAbZIvbu84Qqq2msaalUQqlUwvX1NQ4PD3F0dIR8Ps8Pd326lfqSa7UaIpEIVFVFp9PhbgWj0Qi/\n3w+j0Qi3282OeAaDAZlMhs/jMmw9yLLMKeNEIsEeB+SSRoWJJMqUbZgWLpcLiUSCF2Eul4u1hZ7/\nFEiQKC/dQIqfQT/F46ET8xTRJOHweDxwu91IJBIIhUJwOp0wmUxsqFEsFpHP55HP51EsFqd+Qbw0\naP/f5XLB7/cjFArxw5/28On8ULX69fU1bm5ukM/neW9N8HyQKAeDwYmCR4rmKMoSFdefMRgMbBZB\nXQS//PLLRNsl8OWZQdtfVBtTq9VQLpd58Xl1dYWTkxOcnp5CUZR7p9lRtwi1+CiKwl0nNNqUill9\nPt9EZmM0GkFVVe5ffunCTKJM1ry0daDvxqGU9TR7timrp0+Zx+Nx9szQNA2KoiCXy6FarXLr4CyP\n94sQ5WlAjftGoxGhUAhv3rzB73//e2xubsLn82E8HkNVVVxfX+Ps7AyZTAa1Wo1TVS/5BvgZjEYj\notEoYrEYEokE3rx5gzdv3mBjYwOhUAiyLPODql6v4/r6GsfHx/j06RPOz895H1nwvOiHUNjtdphM\nJoxGI9RqNaTT6WcxXXjJ6PdxqcCU/Arui44pi0bX+PX1NTKZDMrlMhRFQbFYnJikdt9DnGpVAPDC\n9ejoCLFYDLFYjA1FKIoPh8MYDAbc/lkoFFCr1Sb8GV4q5AMej8cRDofhcrnY5KZQKPC+/DSvV30h\nMQ26WF9fRzAYhM1mw3A4ZA04Pj5GNptFs9mc+QLoVYky3XihUAjb29v49ddf2cZNL8pUmKRPub7W\nvTij0YhIJIJ3797h7du3PEgkEonwoANqf8rlcjg/P8fJyQk+fvyIbDbLe/2C54W6FDwez0SNhKqq\nuLm5wenpKXK5nBBlHRQ10cOaerrvTn0im13ql/306RP++c9/IpVK8Z4ypaZbrdaD1zv1Lg8GAzYI\noWIvp9PJfgzkyRAOh+FwOGCz2VAsFnF+fo58Ps8p9Jf8TJJlGX6/H4lEAuFwmNP15PyYSqWQz+fR\nbDan9pnUF04Lglgsho2NDbjdbthsNoxGI1SrVVxdXbEo07hNIcrPABVMOJ1ORCIRrK2tYXt7m53A\nhsMh6vU6crkcrq+veRDFaxUUiiTMZjNnFt69e4dIJIJwOAyPx8MPkGaziXK5jJubG5yfn+Pi4gKX\nl5eoVqui2vcZeajXkvr4u90uqtUqMpkMLi8voSjKUlnF/iz6MYH6KAqYrEHp9/s8dvTy8hInJyf4\n8OEDzs7O2HjnKdc4iSkAqKqKbDbLhZORSAT9fp/PJ6Wy3W43JEni80qR/Ev2ladFB1W8BwIB3sOn\nSDmVSk09UqZsErW+hUIhRCIRPu/6+4V8Fcicapa8GlG22+2IRCKchiULN4PBwNXdVLwh2qC+mFHQ\nYAOfz4dAIACn08mFKVR412g0kMvlcHp6ivPzcxSLxSdNlxL8HLQnSmNRV1dXkUwm4fF4eNVfLpdR\nLBZRLBafZeTdMkODIZrNJi4vL3F8fIzT01PumaWo90ce2p1OB5VKBbIsIx6PQ1EUVKtVPp/L6CCo\nXwDRcyUQCHBhIk2so0hZUZSpzpi22+0Ih8NfjZjVG0pVq1Uu3ms2m3OpJXp1ory1tYVEIsEnhPZ/\ner0e22uSE89rF2V66OtFmaYOkSj3+300m03kcjmcnZ3h/PycI7JZViy+RiwWC1wuFzweD6LRKBKJ\nBJLJJC+WqtUqFEVBqVTihZIQ5adD88XL5TKurq7w8eNHfPz4EYqisCj/6NZWp9Nhj4V8Po9SqYRq\ntYrRaMQTi5YNyuxQTz3ZKetb+JrNJosyGd9MC5vNhlAohM3NTfbql2UZrVaLxwLTQrZUKk3dzvOp\nLLUo64u7qKhgZ2cHiUSCDUharRZXUubzeR6Y8NorVSkdGovFOMXkcrkmXM/ooaUoClvi0b4ljbwU\nPB/6NqhQKMTzZ2nmuKqqPJ6zXq/PdFD7MlCv19nm8eTkBGdnZ7i8vOR2p585lmTjK0kSGo0Ge+1T\nwZEkSXz/kL2tx+OBz+fj1p2X1hFCkTLZwdrtdhbk4XCIZrPJU8wURZn659+XMqeCSKr6ptc0I/Tv\nZalF2WQy8aAK8pPd39/HysoKe86WSiWkUimkUil8/PiR+5Jfe/qaRtft7e1hd3cXwWCQzRSoR7xW\nq3GlJKVH6WEh0tbPDxmGRKNRHptJYwY7nQ4ajQbbxIqsxePoK7Ipgn3//j3+7//+Dzc3N8hms7zg\nnPb1rf/suzObLRYLgsEg3rx5g3a7jYuLCzZOemmQ8x8JMxVYUbHctFPWeqjQy+l0wmazsdFUv9/n\n/vFFCMaWWpTNZjP3/JGf7N7eHjweD9txlkolHB0d4e9//zvS6TRub29ffVRBhSUHBwf4y1/+gng8\nzr6w9NAgUaY+Tb0DzkvvoXwp6AfCkyhbLJYJUW6328Jv/AncFUIALMr/+Z//yS1Rz2Hecfez6R6j\niVRmsxnBYBBbW1u8x53JZKb2+bNCX1hHKXqbzcaFdJTGf64OAb0VLfmaA1+yFkKUZ4DZbIbb7Wa/\nWUrzmUwmTpdQv+Dp6SkqlcqrLoahQSCyLHOF+tbWFrxeL5xOJwwGA/dt0010eXmJi4sLFAoFaJo2\n9wv6NUF7ylQsQ1ONaH+sWCyiXq+j2+0KQb4DVeLq9zTvVjSTfebNzc2zfD61sNGseCrwutuSBUwa\nKL3kBRZVtFPQQ66MZAVLgRB5jk9r+ITeIjgYDMLj8fA+dq/XQ6PRQLVanVtxl56lFmWa/hKJRHgP\nwWAw8Jg1TdNQLBZ51q+maa9WkIHPhRA0yCASifDEHL0ZRbvd5mKMy8tLHB0d4ezsDMVi8VUfu3lw\n1zCEUtetVgulUgmZTAaVSuVFpjmfGyr62djYwMrKClwuF4AvU6CeW/T0vbL6e81ms3HNBn2PXq+H\nUqmE8/NzvtemWQA1K0iMB4MB16OoqsrFX7RAId9xEu2fORf6aVBUf5FIJNhKVZIkXsQqisKTDOfJ\n0osy7bnd7YWjk1AsFlEqldh56jVHelarlZ1uotEoG6vQRU1jL8kohJrsU6kUp/YEs4MMQ3w+H/u3\n660Cb29vhSg/gNVqRTgcRjKZnBBlPc8pzHSvxeNxFmUaWXh3hnO/34eiKEilUpzRe4n3GkX7kiSh\n0+lA0zSoqsp99STKlDGgvfuf2b/Xm0a5XC6EQiGsrq5ykReAr0RZRMrPAKWhqNqO9txo3q8+0stk\nMqhWqzxt6qWmhX4U/T6Px+PBysoKtra2sLKyAq/Xy2k9aoGiKsVyucwzY2mk2mvdg58ldK70FbnU\nqgaADfULhYKIlL8BRarkf09R07S5a1Cit/ol21q616gNivaTKcXb6/U4kKjVai+2CPW+v6nRaPBg\nENo229zchKIoXAn9FFcvOr4mk4kNWPRtnVarFclkkl3SKHUNfJmktijeCksnyvqbgB5aelGmMY83\nNzf48OEDbm5uoKrqUs4rfQr6i9nv92NtbQ0HBwdIJBLsJqQvPOl2u2zIr2ka93O/5H2ulwSt/PVF\njKFQCHa7HaPRiBdMJMrNZlOI8j1YLBZ4vV7EYjFe1DyXKOtrNaxWK6xWKxKJBLa2tvD27VvEYrF7\nI3X9hC/9pKKXep/Rd9eLoKZpcLvdcDqd8Hg8WF9fR7vdhtVqRS6XQy6XQ7FYfPR3k9ERHV9yaiQz\nFrvdjp2dHUQikQf37PXfcZ4snSgDX4SGhpeTKFut1glR/vjxI/L5PGq12txXR/OCHhoWiwU+n49F\nmaZA6S/gu6JM7TavOeU/a8gOUj+cPRQKTbirVSoV5PN5ZDKZV91F8C3uE+XnQJ8+dTgccLlcPKFu\na2sL+/v7cDgccDgcE+8j8er1eveK8ryF40eh52y/3+f6lMFgwEW5q6urkGUZ0WgUZ2dnSKVSE1Ht\nQ1D9EAk8HVOHwwG73Q6Hw4FwOHyvKM+qjuCpLJ0o609OIpHgvVGbzcZVw4qioFAo8MCE1xxJ2Gw2\n+Hw+FmTtBsf1AAASN0lEQVQao+ZyubhtjIazq6qK29tb9rYulUqv+tjNA1po+nw+hMNh3k/Wz+qt\n1+u8YBLcjz69+Zx+0rR3HAgEEAwG+bW/v4+1tTVOW1M9AC2uqLhLURRcXV3xgBxKsS6KgPworVYL\nxWIRFxcXE9XRw+Fwot/earUiEAg8+vvIk4IiY4qayaeCjEru26Zot9u8kFVVde779UsnylTAEY/H\nsbW1hXg8jkAgALPZzPttVNylKMqrtx50OByIxWJYX19HMplELBaD3++HLMs8aUhf3HV+fo7j42Oc\nnJygWCyKAQczhqqGE4kEYrEYvF4vrFYru0KVy2U0m81XfU0vEna7HbFYDMlkkiesUbAQiUQgyzLv\ngQLAYDDg7pCbmxucnZ3h9PQUZ2dn/LxahH3Pn6XVaiGbzcJsNvPfQ90EtPA0GAzw+XxIJpNP+p1U\na6EfLkIGJVarlf97V5Q1TUOhUMDV1RVUVZ37M20pRZmmGulFmbxmyZyfRPm174U6nU7EYjHs7u5y\nJarf7+d95OFwCE3TUCqVkE6ncX5+jqOjI5ycnPCEHMHs0LfyxGIx9u+lViiyiRXnZTGgYSG7u7vY\n3d3F1tYWtre32R1PP7uZpkh1Oh3UajVcXV3h/fv3+PjxIwqFAnvKL8PzStM0Ho04HA5hNBp5AWOz\n2TgbBOBJGQxK9ZNRDkF7+bT4ue93UWvs1dUV90zPk6UQZdo/lmWZnbv29/eRTCbh8/lgMBgmhiZQ\nKkjstX2OlKPRKHZ2dhCPx+HxeCYeFKPRCOVyGefn5zg8PMT19TXK5TJXgL70FftLQ2/mT73Jdyed\nCZvT+WK32xEIBOD3+5FMJrG7u4v19XVEIhH4fD7uRb4L9SSTu1SlUkGhUEChUEC9XucBGMsAWVuO\nx2Pc3t7CZDKh0+kgEokgGo0iHA5zWp/sML9Fr9dDvV7nYUIEmUfR6EtKcevFWV9QtwjPtKUQZZPJ\nxEUv5EK1v7+PeDwOl8uF0WiEWq2GdDqNw8ND3NzcoFarzftrLwSUvt7Z2UEwGPyqCnQ4HEJRFJye\nnuL9+/fI5XKo1Wq8r7UsD4mXAvn32u32CatAMmWgwrt5P1heM06nkwMDqtNYWVlBMBj8ZpU39SRT\nbUClUuHWw263u1QFlcPhkBf25CmezWYn9t711dSP0W63OQOqTz9TFnBnZwfRaBShUOir37dobmlL\nJcrhcBirq6ssyn6/n11hqEjp8PAQuVwOqqrO+2svBHpRttlsX1UmjkYjFuXffvttKhNyBD+OXpRl\nWb5XlBdhtf+acblcSCaT+Mtf/oK1tTW43W64XC6uCv5WOlYfKdMYwUqlMsNvPxtIBMmjPZvNAgCn\nrcnalxy+HqPRaOD29hbpdBqNRoP/fXt7G41Gg/ftbTbbV4Vj+v7pReBFizJt6rtcLiQSCezv72N/\nfx+xWAxWqxWDwQCNRoM9bMlSk/yABZ+hvm79hBo9ZFVHRRlPvXjpYr+vjYM+h4oz9J+r7zUnJx59\nNfhDnwWArftoe2LZonkqgqG9fzLE6XQ6qFQqyOVybIYj+DaP7VVSZ0I0GuV/o35+v98Pl8vF+8L6\neycSieDg4ADJZPKriM9kMn0zUi4Wi/j48SM+fPiA8/PzV5PRo3uUWi6pwLRerz+pJYpqhmhgCEFb\nOs1mkyfY3X0ekA+52+3mPeV5ZiVerChTfy15mq6uruLt27fY29vjqkbaZygWi8jn8yzKojr1+9BP\ndaGWjacUX1D0po/caPINvfQuPASdW6PRCIfDgXg8zvvdD0E3Gtl/PtdEn3lDLnVUJU+iTG0d2WxW\niPJ3oL8W745NtNvt8Pv9iMVi/PM2mw1v3rxhJy6LxcJFW4TT6UQoFOJ0NV3L9N+HFr+j0QiFQgEf\nPnzAf//3f0NRlFcjygQ9l3u9Hj/f9cf2IQaDwb0TnobDIbrdLlqt1oMzqKnq2+12o9VqccHdvHjR\nokwmCl6vF6urqzg4OMD29jYXwFDUpHeGqVQqQpC/AzrONpuN/WJNJtOTKhQpnaqf50uCThEyPdT0\nxRz0mWTwkEwmsbOzg3A4/OBnUUScSqXQ6XRQKBS4+GlZUrmSJHFlajQanRhB2ul0+FpfhF7LlwCl\nLO9buJEoB4NBJBIJ/neXy4U//OEP+Jd/+Rdsb29zT6y+cOuu6D5mTkHX7mAwQKFQwOHhIf73f/93\nmn/qi2Ha8weo57vVanE72V3o+Ub1R/O+d16sKNtsNqytrWFtbQ27u7vY3t5GMBjkebLtdhuKouD6\n+hqHh4e4vLxEtVpdmgf0rDAajUgkEvjzn/8Mn8/H6Z2npLDb7TabWfR6PU4LUdRAk2Foz43QR9DU\nJkEidB/6h54kSbzvTYuwl74nR4tP8mvWm/Y/JYoQfE273UapVMLl5SXG4zFisRicTueEmIbDYbx9\n+5azEcDnTMXm5iaCwSDv6d+Neul6JGta/b/dR6PRQKPRYHvU55on/Np5KEOhL64T85R/EIoY1tbW\n8Ouvv/L+TSAQgCzLPLSaRPnTp0+4urpCpVIRovydmEwmJBIJWK1W7OzscHTxlJRwrVbjvXwai9nt\ndjkKtlgsiEQiiEQiCIVCE++lfWZKLd01kdej/y5GoxGdTgedTgcWiwWDweDFi7LFYmFvYBJlq9Uq\nRPknIFG+uLhgC8xoNPqVKJvNZqyurvI1RtOMKEtB1+m37ofH6hoajQZyuRxubm6EKM8B0gvqKhGi\n/J3oJ+TE43G8e/cO796944o9Gl1HgnB9fY3T01MUCgXuixN8QW8QTylkfQW20WjkUY7fS7lcxu3t\nLTKZDBqNxoRYkjk/ORzp9+2+9V3vFo/dXWRpmoZYLIZKpYJGo/EkM/tFh6xjQ6EQFxjp26Gow0C/\nfy+u82/T6XQ4UqaZ68PhkK99SZLg8/ng9/v5Pd97TPVRsv6c0D1H56tYLOL6+hqpVAr5fB6apk3p\nrxTcx91oWT8cYxF4UaKs32t0uVwcOXg8HthsNhiNRtTrdWQyGVxdXeHo6Ai3t7fsb31f5d1rhzyt\nc7kcR2FOp3Mqv1uWZX6o0fHv9/ucujabzRPFSsDkg+/uuaI0k6ZpaLfb6Ha76HQ6E8KczWZxdXW1\nVFXIsizD5/MhHo/zcHaDwYB+v8979pVKheeD3zVQEHxNt9tFtVpFNptFLBab8MC/67T1M+iLHWkL\np9fr8bkqFovI5XLI5/PIZrNftfQIps99GrBIuvAiRZk25d1uN7xeLzweD1fptdttZLNZfPr0CcfH\nx8hkMrxXsCh9aIuEXpSBz+nqaYqy1+uFzWbjCO5uoRdFzMCXSPihfbher4darQZFUVCtVtnBR190\npigKMpkMm8sviyhTG5RelLvdLjRNQ7PZZFEulUrC/vQJ0HUvSRISiQRqtRq63S6L8bRFmYqNWq0W\nNE3D2dkZjo+PcXx8DFVVUavV+CVE+Xl4Ke2RL0KUaYOexntRm0IoFGJDfv0+8s3NDY6Pj3F+fs5p\n63n7mS4qZMZ+cXHBeylP6QvU9xLrB7jrU99kkedyuSbSdXfTed1ulx2L6EUCro+CW60W8vk88vk8\nFEWBqqqoVqsT51bvhLQsokwFXnpRliQJvV4PjUYDiqKgXC6jWq2iVqu9mIfPPKF2Sap4ppfb7eb5\nu5TN+RmBHgwGaLVaE/3z1WoVR0dHeP/+Pd6/f89T2MjhSgQPz8N9FfF3//ci8CJEmW4Op9OJZDKJ\n7e1t7O3tYXNzE06nE91uly3WTk9PkUqlcHt7C0VRoGmaKO76Bqqq4uzsDIPBgNvGyuXyV32bd6Eo\n12KxwG63c+bCYrHc+/NUiV2r1TiNpxdTmgVcr9e5j/yutWC32+Xf0Ww2OfLQ/wwZBTSbTWia9mLT\nuPpFj97EgtL9VGVO+/aKonCPpeBxqFUGAAqFAo6Pj3lMoNfrhdfrhd/v5+2xH6XdbvNsa0pT53I5\npNNppNNpNJvNiY4GUQ8wXegeooCBggZ9Vm7RjvnCizIZSVitVng8HmxubuLPf/4zfve73yEajcLp\ndKLT6SCfz+P09BSHh4dIpVJIp9Mol8tL1af6HFCkmcvlUCgUUKlUUK/XHxVlqop2Op0sGNSmcx9U\n7ZrJZHhPuNPp8M1Axgk0Dec+UaWCNPJ3ppf+htL/+yJUUv4M9DCx2+3w+XyIxWIIBALsn0yinE6n\nhSh/J2QqQde+xWJBu93mkYrRaBTr6+uwWCw/JcqtVgu5XA7Hx8e4uLjA1dUVLi8vedQmBQ36rJA4\nh9OFxFjvHKi31hSi/J3Q8GuPx4NwOIy1tTVsb2+zV7PVauU5snTBZ7NZnisr+DYUbRaLxQk/WrqA\n73phE+Q37nK5EAgE2M70of3oQqEwER2Qww4xHA45iigWixztLkP6+UfQL4poUep0OtlVrdVqsa3m\n5eUlt50JnoZ+CIGqqjAYDDz6kvymKZNjMBh4Pq/eZYpGm9LijyJeqpuQJAm3t7e4vr7G+fk5UqkU\nLi8vcXFxIVLUM+LuVDWTycSmQhQYLNqo0xchym63m0cy0mB3m83GY+to5m+lUkG1WkWr1RLR8Q/Q\naDSQyWTQ7XYfjZRpMILVaoXD4eCCu4f2o+v1OqrVKiqVCkco+hthNBqhXq9zocuyTcX5XvSpNXrw\n93o99vBVVRU3Nze4uLjA6ekp8vm8KBD6QbrdLur1Ors5qaqKbDbLoxPT6TR8Ph+nsmkxajKZ2PiD\nBkiQ/zLw+dlVLBZxeXnJ2QzRljlbZFlGMBjk0Zl2ux2DwQDVapU9FBZtauDCi7LBYGBRTiaTiEaj\nLMqUkiBRphVuq9USK9EfgFaMiqLwvz0kyvq9GqPRyFaZD0XWFEXctdwkaGwduX4t0tSWeUHCTKLc\n7/fRbDY5MqMI7PT0FJqmCdOJH6TX62E0GvFih3zeS6USstksbm5ukEgkeAzjaDSCLMuQJIl74cn8\ngzJBRKPRQKlUQqlUgqqqYothxlCdAIkyDSqqVCrcG35zc4N6vT7vr8osrCjrbRaDwSDW1tawubmJ\nSCQCp9M54ZWsT0e0222+yQTfB7XSiIhr/ugjZU3TkM/nkUqlYDKZ0O120ev1cHZ2hnQ6jVKp9F3T\nuwSTUNHX3RTmYDCApmncskTZnnA4jHA4DFmWOdrSF2/R/SNJ0kThIfXWC1GeHTRMR5ZlDIdDVKtV\n9Ho9nJ+f4+TkhNtmF+mZt7CibLVaucAlkUggmUxic3MT4XB4wmxCIFhW6OFdLpdxdHSEdrsNg8HA\nkTIV51FmQTzsp0u73eZCSKq7OD8/f3L6GvicIaKWv3mPBHyNUEtatVqFqqpQFIXtly8vL3F1dbVw\n9UcLK8qyLHNxVzweRzKZxNbWFk9lEQiWHYqWy+UyWq0WLi4u2Gd5PB5zb+vdCnTBdKCsG41/vVvs\nJUnSV5X+d7s9KNtBL3GeZgu5AFarVTSbTVxcXODi4gLZbJanB1KL5qKw0KLsdrsRDAYRCATg9/vh\n9Xp5D5P22qg4Q/T5CZYVirQEs4WqsxepMlfwfbRaLWSzWRwdHaHRaOD6+pqj40qlslAFXsTCijK1\nQYVCIfa2NplM3Kqj3wfSNI29lUmYBQKBQPC6qdfrSKVSvJ9PHSCkGYvIwooypa+DwSDcbjesViv3\nBgJfiruoz7bT6XBlr4iWBQKBQFCv19HpdJBOp9mHXG/ju4gsrChT9bXFYplo1qdiCU3TkMvlkM1m\ncX5+jsvLS1SrVbTbbTENSiAQCAQswi+pXXBhRfk+KGVNlZBHR0f4+PEjt4bQ8AnREiUQCASCl8iL\nEmWqOG00GigUCjg6OsJ//dd/4fT0lG0bqYpORMoCgUAgeGksrCg3Gg3k83lYLBYMBgOoqopwOMzz\nY/P5PI6OjpDL5VCr1bgdQYixQCAQCF4qCyvKtVoN6XQa9XodmUwGnz59gsPh4IprEmYxCUogEAgE\ny8LCinK9XucRgvqRW4R+DqaIjgUCgUCwDDxVlK3P+i3ugYT27uCCF86PHseZH/8lRRz/+SPOwXwR\nx3++PHoc7x/p8zUbP/c9BP+fjRm/TzDJxozfJ/iajRm/TzDJxozfJ5hk47EfkJ6S+pUkKQDg3wBc\nAVhMG5TFxorPJ+M/xuNx+XvfLI7/TyOO//wR52C+iOM/X558/J8kygKBQCAQCJ6fp6avBQKBQCAQ\nPDNClAUCgUAgWBCEKAsEAoFAsCAIURYIBAKBYEEQoiwQCAQCwYIgRFkgEAgEggVBiLJAIBAIBAvC\n/wOKw9XKlVKVjAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f9c6d4c5250>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "imshow_multi(test_in[0:5])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# two conv layers:\n",
    "# 28x28x1 -> 14x14x16 -> 7x7x16\n",
    "\n",
    "class Net(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(Net, self).__init__()\n",
    "        self.conv1 = nn.Conv2d(1, 16, 3, padding=1)\n",
    "        self.conv2 = nn.Conv2d(16, 16, 3, padding=1)\n",
    "        self.fc1 = nn.Linear(7 * 7 * 16, 256)\n",
    "        self.fc2 = nn.Linear(256, 10)\n",
    "    \n",
    "    def forward(self, x):\n",
    "        x = x.view(-1, 1, 28, 28)\n",
    "        x = F.relu(self.conv1(x))\n",
    "        x = F.max_pool2d(x, (2,2))\n",
    "        x = F.relu(self.conv2(x))\n",
    "        x = F.max_pool2d(x, (2,2))\n",
    "        x = x.view(-1, 7*7*16)\n",
    "        x = F.relu(self.fc1(x))\n",
    "        x = self.fc2(x)\n",
    "        return x\n",
    "\n",
    "net = Net()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "batch_size = 16\n",
    "def minibatch(inputs, labels):\n",
    "    idx = [random.randint(0, len(inputs)-1) for _ in xrange(batch_size)]\n",
    "    img, label = np.array([inputs[i] for i in idx]), np.array([labels[i] for i in idx])\n",
    "    return torch.from_numpy(img).float(), torch.from_numpy(label).long()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.FloatTensor\n"
     ]
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([2, 5, 2, 5])"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x = torch.rand(10, 10)\n",
    "x.view(-1, 5, 2, 5).size()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "crit = nn.CrossEntropyLoss()\n",
    "opt = torch.optim.SGD(net.parameters(), lr=0.001, momentum=0.9)\n",
    "\n",
    "def train_step():\n",
    "    opt.zero_grad()\n",
    "    \n",
    "    images, labels = minibatch(train_in, train_out)\n",
    "    images = Variable(images)\n",
    "    labels = Variable(labels)\n",
    "    out_labels = net.forward(images)\n",
    "    loss = crit(out_labels, labels)\n",
    "    loss.backward()\n",
    "    opt.step()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.103125"
      ]
     },
     "execution_count": 97,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def compute_acc(n=20):\n",
    "    # accuracy on 20 minibatches:\n",
    "    correct = 0\n",
    "    incorrect = 0\n",
    "    for _ in xrange(n):\n",
    "        images, labels = minibatch(test_in, test_out)\n",
    "        images = Variable(images)\n",
    "        labels = Variable(labels)\n",
    "        predictions = F.softmax(net.forward(images)).data.numpy().argmax(axis=1)\n",
    "        for a, b in zip(labels.data.numpy(), predictions):\n",
    "            if a == b:\n",
    "                correct += 1\n",
    "            else:\n",
    "                incorrect += 1\n",
    "    score = correct / float(correct + incorrect)\n",
    "    return score\n",
    "\n",
    "compute_acc()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "13% correct\n",
      "30% correct\n",
      "26% correct\n",
      "28% correct\n",
      "35% correct\n",
      "39% correct\n",
      "39% correct\n",
      "44% correct\n",
      "45% correct\n",
      "43% correct\n",
      "58% correct\n",
      "43% correct\n",
      "51% correct\n",
      "59% correct\n",
      "61% correct\n",
      "67% correct\n",
      "75% correct\n",
      "73% correct\n",
      "69% correct\n",
      "79% correct\n",
      "79% correct\n",
      "79% correct\n",
      "80% correct\n",
      "81% correct\n",
      "81% correct\n",
      "78% correct\n",
      "85% correct\n",
      "76% correct\n",
      "84% correct\n",
      "83% correct\n",
      "85% correct\n",
      "83% correct\n",
      "82% correct\n",
      "85% correct\n",
      "83% correct\n",
      "88% correct\n",
      "86% correct\n",
      "82% correct\n",
      "85% correct\n",
      "85% correct\n",
      "87% correct\n",
      "90% correct\n",
      "88% correct\n",
      "86% correct\n",
      "89% correct\n",
      "87% correct\n",
      "88% correct\n",
      "86% correct\n",
      "87% correct\n",
      "82% correct\n"
     ]
    }
   ],
   "source": [
    "for i in xrange(1000):\n",
    "    train_step()\n",
    "    if i % 20 == 0:\n",
    "        score = compute_acc()\n",
    "        print \"{}% correct\".format(int(score * 100))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12+"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}