From 82f73ab61e726ccbb79e6da054a021ac504f3607 Mon Sep 17 00:00:00 2001 From: Yacine Mahdid Date: Mon, 17 Aug 2020 12:24:21 -0400 Subject: [PATCH] Add Adam --- ...t Optimization Algorithms-checkpoint.ipynb | 2141 +---------------- .../.ipynb_checkpoints/README-checkpoint.md | 8 + ...ient Descent Optimization Algorithms.ipynb | 2141 +---------------- .../AlexNet-checkpoint.ipynb | 6 + Image Recognition/AlexNet.ipynb | 32 + 5 files changed, 270 insertions(+), 4058 deletions(-) create mode 100644 Deep Learning from Scratch in Python/.ipynb_checkpoints/README-checkpoint.md create mode 100644 Image Recognition/.ipynb_checkpoints/AlexNet-checkpoint.ipynb create mode 100644 Image Recognition/AlexNet.ipynb diff --git a/Deep Learning from Scratch in Python/.ipynb_checkpoints/Gradient Descent Optimization Algorithms-checkpoint.ipynb b/Deep Learning from Scratch in Python/.ipynb_checkpoints/Gradient Descent Optimization Algorithms-checkpoint.ipynb index 0e7600b..d8dc501 100644 --- a/Deep Learning from Scratch in Python/.ipynb_checkpoints/Gradient Descent Optimization Algorithms-checkpoint.ipynb +++ b/Deep Learning from Scratch in Python/.ipynb_checkpoints/Gradient Descent Optimization Algorithms-checkpoint.ipynb @@ -10,7 +10,8 @@ "- [Stochastic Gradient Descent + Momentum](https://youtu.be/7EuiXb6hFAM)\n", "- [Adagrad](https://youtu.be/EGt-UOIIdDk)\n", "- [RMSprop](https://youtu.be/nLCuzsQaAKE)\n", - "- AdaDelta\n", + "- [AdaDelta](https://youtu.be/6gvh0IySNMs)\n", + "- Adam\n", "\n", "## Tests\n", "In order to demonstrate the algorithms capabilities to optimize a function we used these simple test setup:\n", @@ -19,7 +20,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 26, "metadata": {}, "outputs": [], "source": [ @@ -292,702 +293,87 @@ " \n", " if i % 100 == 0:\n", " print(f\"Iteration {i}\")\n", - " print(model)" + " print(model)\n", + " \n", + "\n", + "def adam(model, xs, ys, learning_rate = 0.1, b1 = 0.9, b2 = 0.999, epsilon = 0.00000001, max_iteration = 1000):\n", + " \"\"\"\n", + " Adam: This is the adam optimizer that build upong adadelta and RMSProp\n", + " model: The model we want to optimize the parameter on\n", + " xs: the feature of my dataset\n", + " ys: the continous value (target)\n", + " learning_rate: the amount of learning we want to happen at each time step (default is 0.1 and will be updated by the optimizer)\n", + " b1: this is the first decaying average with proposed default value of 0.9 (deep learning purposes)\n", + " b2: this is the second decaying average with proposed default value of 0.999 (deep learning purposes)\n", + " epsilon: a variable for numerical stability during the division\n", + " max_iteration: the number of sgd round we want to do before stopping the optimization\n", + " \"\"\"\n", + " \n", + " \n", + " # Variable Initialization\n", + " m = [0, 0] # two m for each parameter\n", + " v = [0, 0] # two v for each parameter\n", + " g = [0, 0] # two gradient\n", + " t = 1 # time steps\n", + " \n", + " for i in range(max_iteration):\n", + " # Calculate the gradients \n", + " x, y = stochastic_sample(xs, ys)\n", + " g[0] = model.dx_w0(x, y)\n", + " g[1] = model.dx_w1(x, y)\n", + "\n", + " # Update the m and v parameter\n", + " m = [b1*m_i + (1 - b1)*g_i for m_i, g_i in zip(m, g)]\n", + " v = [b2*v_i + (1 - b2)*(g_i**2) for v_i, g_i in zip(v, g)]\n", + "\n", + " # Bias correction for m and v\n", + " m_cor = [m_i / (1 - (b1**t)) for m_i in m]\n", + " v_cor = [v_i / (1 - (b2**t)) for v_i in v]\n", + "\n", + " # Update the parameter\n", + " model.w0 = model.w0 - (learning_rate / (np.sqrt(v_cor[0]) + epsilon))*m_cor[0]\n", + " model.w1 = model.w1 - (learning_rate / (np.sqrt(v_cor[1]) + epsilon))*m_cor[1]\n", + "\n", + " t = t + 1\n", + " \n", + " if i % 100 == 0:\n", + " print(f\"Iteration {i}\")\n", + " print(model)\n", + " \n", + " " ] }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 27, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "Gradient Descent: \n", + "Adam\n", "Iteration 0\n", - "y = 0.15080809254765393 + 0.6790908370317389*x\n", + "y = 0.8575880576520295 + 0.8405747654640998*x\n", "Iteration 100\n", - "y = 0.14070772861001737 + 0.9716919751062303*x\n", + "y = -0.001590081073315357 + 0.9984342651063493*x\n", "Iteration 200\n", - "y = 0.09514343174678791 + 0.980858744143106*x\n", + "y = 2.70122392056152e-05 + 1.0001344335031852*x\n", "Iteration 300\n", - "y = 0.06433386917675844 + 0.9870571091711977*x\n", + "y = -1.4876611515003004e-06 + 1.0000033575642253*x\n", "Iteration 400\n", - "y = 0.04350112926625649 + 0.9912483055313238*x\n", + "y = 1.030624925582267e-05 + 0.9999970769162957*x\n", "Iteration 500\n", - "y = 0.029414494599109095 + 0.9940822991489187*x\n", + "y = 2.567876097403007e-06 + 0.9999995341571982*x\n", "Iteration 600\n", - "y = 0.01988942602444484 + 0.9959985825044247*x\n", + "y = 3.001062476619204e-08 + 0.9999999996624379*x\n", "Iteration 700\n", - "y = 0.013448786830212731 + 0.9972943306231896*x\n", + "y = 3.5509149827671966e-09 + 0.9999999957864142*x\n", "Iteration 800\n", - "y = 0.009093770075728168 + 0.9981704866376216*x\n", + "y = 2.013987394989773e-08 + 0.9999999952565595*x\n", "Iteration 900\n", - "y = 0.006149004756654367 + 0.9987629238177406*x\n", - "y = 0.00417412143896754 + 0.9991602370760215*x\n", - "Stochastic Gradient Descent: \n", - "Iteration 0\n", - "y = 0.15993686766783655 + 0.1833518650965968*x\n", - "Iteration 100\n", - "y = 0.21828157996824554 + 0.9594516741177225*x\n", - "Iteration 200\n", - "y = 0.1451844200607905 + 0.9741557959554868*x\n", - "Iteration 300\n", - "y = 0.1023440101887112 + 0.9800172156010936*x\n", - "Iteration 400\n", - "y = 0.07450363961602721 + 0.9893311113895694*x\n", - "Iteration 500\n", - "y = 0.046350587935699336 + 0.9881459700038466*x\n", - "Iteration 600\n", - "y = 0.030476212073656663 + 0.9947271999445939*x\n", - "Iteration 700\n", - "y = 0.021391053052246013 + 0.9959265131392894*x\n", - "Iteration 800\n", - "y = 0.014571200279042585 + 0.9967328328159811*x\n", - "Iteration 900\n", - "y = 0.009519640905117088 + 0.997649803991943*x\n", - "y = 0.006666932714079679 + 0.998500306222465*x\n", - "SGD + Momentum: \n", - "Iteration 0\n", - "y = 0.45244291469084846 + 0.9353652979013074*x\n", - "Iteration 100\n", - "y = -0.2248285078215096 + 2.6361555442512574*x\n", - "Iteration 200\n", - "y = -12.384110603326564 + -0.9941162210048384*x\n", - "Iteration 300\n", - "y = -0.09392179857751858 + 1.0244628660411053*x\n", - "Iteration 400\n", - "y = -0.13879271342536312 + 0.9636770795356304*x\n", - "Iteration 500\n", - "y = -0.44087896381090375 + 0.8087858583453921*x\n", - "Iteration 600\n", - "y = 0.011663448270001958 + 1.0025566762581413*x\n", - "Iteration 700\n", - "y = 0.0007482973122936055 + 0.9998250344477012*x\n", - "Iteration 800\n", - "y = -7.183493853088314e-05 + 1.0000396032465118*x\n", - "Iteration 900\n", - "y = -2.1474874379620237e-05 + 1.0001801674854078*x\n", - "y = 4.0014297998085525e-05 + 0.9996092661868773*x\n", - "Adagrad\n", - "Iteration 0\n", - "y = 0.14957133873587877 + 0.349770642623898*x\n", - "Iteration 100\n", - "y = 0.5789747422257289 + 0.883780412031906*x\n", - "Iteration 200\n", - "y = 0.476060907959348 + 0.9021321576916074*x\n", - "Iteration 300\n", - "y = 0.3883109597254469 + 0.9167686675736807*x\n", - "Iteration 400\n", - "y = 0.32213311654349985 + 0.931919861161982*x\n", - "Iteration 500\n", - "y = 0.26703558629401253 + 0.9468055802251013*x\n", - "Iteration 600\n", - "y = 0.22881588002466632 + 0.9522323308080116*x\n", - "Iteration 700\n", - "y = 0.18778860232158315 + 0.9582100069795217*x\n", - "Iteration 800\n", - "y = 0.15554079368406587 + 0.9661025574708333*x\n", - "Iteration 900\n", - "y = 0.13264118785089948 + 0.9731507500316178*x\n", - "Iteration 1000\n", - "y = 0.1070410549333178 + 0.9749049605055626*x\n", - "Iteration 1100\n", - "y = 0.09185365717182185 + 0.9807915717798473*x\n", - "Iteration 1200\n", - "y = 0.08111295608842777 + 0.9850047591169898*x\n", - "Iteration 1300\n", - "y = 0.06211168920968319 + 0.9838182610782438*x\n", - "Iteration 1400\n", - "y = 0.05380151228795558 + 0.9883694454256858*x\n", - "Iteration 1500\n", - "y = 0.04424068877458216 + 0.9903814386731185*x\n", - "Iteration 1600\n", - "y = 0.03729834508521443 + 0.9916615367006102*x\n", - "Iteration 1700\n", - "y = 0.031939592115124336 + 0.9933359574629548*x\n", - "Iteration 1800\n", - "y = 0.02765181756924617 + 0.9947914270963896*x\n", - "Iteration 1900\n", - "y = 0.022584337990350704 + 0.9952052508861745*x\n", - "Iteration 2000\n", - "y = 0.018842544645614277 + 0.9960323037855057*x\n", - "Iteration 2100\n", - "y = 0.015706324717892087 + 0.9964034044711639*x\n", - "Iteration 2200\n", - "y = 0.013376815996403906 + 0.9970421549139848*x\n", - "Iteration 2300\n", - "y = 0.011140802753997708 + 0.9975800313527812*x\n", - "Iteration 2400\n", - "y = 0.009180596246359064 + 0.998101061323726*x\n", - "Iteration 2500\n", - "y = 0.007884923400885832 + 0.9985257724925584*x\n", - "Iteration 2600\n", - "y = 0.0064791120032062 + 0.9985724869754915*x\n", - "Iteration 2700\n", - "y = 0.005665873552865111 + 0.9988468038704624*x\n", - "Iteration 2800\n", - "y = 0.0048065607643679445 + 0.9990486498541105*x\n", - "Iteration 2900\n", - "y = 0.003968266097986841 + 0.9991154584760444*x\n", - "Iteration 3000\n", - "y = 0.0033298137947775286 + 0.999309958328142*x\n", - "Iteration 3100\n", - "y = 0.0029504826077003834 + 0.9994302759246211*x\n", - "Iteration 3200\n", - "y = 0.002380617673238137 + 0.9994742557828102*x\n", - "Iteration 3300\n", - "y = 0.0020912374417970912 + 0.9995991422066534*x\n", - "Iteration 3400\n", - "y = 0.0017218570181314194 + 0.9996661035040911*x\n", - "Iteration 3500\n", - "y = 0.0014294620821387657 + 0.9996922803546607*x\n", - "Iteration 3600\n", - "y = 0.0012405238514359416 + 0.9997455508343457*x\n", - "Iteration 3700\n", - "y = 0.0010073098877846675 + 0.999783563658199*x\n", - "Iteration 3800\n", - "y = 0.0008595170460348628 + 0.9998230970978605*x\n", - "Iteration 3900\n", - "y = 0.0007109894887433046 + 0.9998455123904334*x\n", - "Iteration 4000\n", - "y = 0.0006004078268029727 + 0.9998625899850877*x\n", - "Iteration 4100\n", - "y = 0.0005002665526051534 + 0.9999021008909665*x\n", - "Iteration 4200\n", - "y = 0.00042043879502743817 + 0.9999083394167763*x\n", - "Iteration 4300\n", - "y = 0.00034712046017588467 + 0.9999226341779709*x\n", - "Iteration 4400\n", - "y = 0.0002918159150337169 + 0.9999341373894769*x\n", - "Iteration 4500\n", - "y = 0.00023759176842522305 + 0.9999421976498841*x\n", - "Iteration 4600\n", - "y = 0.0002031163044737048 + 0.9999551395282233*x\n", - "Iteration 4700\n", - "y = 0.00017652745265826287 + 0.9999627827656518*x\n", - "Iteration 4800\n", - "y = 0.00014397038691800682 + 0.9999694413255239*x\n", - "Iteration 4900\n", - "y = 0.00011977885596124304 + 0.9999765829401767*x\n", - "Iteration 5000\n", - "y = 9.742011104497176e-05 + 0.9999780765656271*x\n", - "Iteration 5100\n", - "y = 8.405139675455769e-05 + 0.9999830106577814*x\n", - "Iteration 5200\n", - "y = 7.199360094961494e-05 + 0.9999853763532276*x\n", - "Iteration 5300\n", - "y = 5.9438595609082104e-05 + 0.9999879821035998*x\n", - "Iteration 5400\n", - "y = 4.799553234174064e-05 + 0.9999900691345857*x\n", - "Iteration 5500\n", - "y = 4.0372638139457874e-05 + 0.9999915160547906*x\n", - "Iteration 5600\n", - "y = 3.294290517155539e-05 + 0.9999926620335875*x\n", - "Iteration 5700\n", - "y = 2.8426587896418184e-05 + 0.9999943696968747*x\n", - "Iteration 5800\n", - "y = 2.3897665033254145e-05 + 0.9999952434923006*x\n", - "Iteration 5900\n", - "y = 1.9640055052909126e-05 + 0.9999959414903599*x\n", - "Iteration 6000\n", - "y = 1.684732609374185e-05 + 0.9999966681434076*x\n", - "Iteration 6100\n", - "y = 1.3996448973319881e-05 + 0.9999971896668423*x\n", - "Iteration 6200\n", - "y = 1.2071327881619936e-05 + 0.9999976352142692*x\n", - "Iteration 6300\n", - "y = 1.0143608586031199e-05 + 0.9999979501909246*x\n", - "Iteration 6400\n", - "y = 8.397847594666258e-06 + 0.9999982734836176*x\n", - "Iteration 6500\n", - "y = 7.081087850144272e-06 + 0.9999984771962163*x\n", - "Iteration 6600\n", - "y = 6.0865568463067215e-06 + 0.9999986630504831*x\n", - "Iteration 6700\n", - "y = 5.2112590599668e-06 + 0.9999989126192039*x\n", - "Iteration 6800\n", - "y = 4.425142363347888e-06 + 0.9999991143161019*x\n", - "Iteration 6900\n", - "y = 3.6347556610242305e-06 + 0.999999226262124*x\n", - "Iteration 7000\n", - "y = 3.047110472416464e-06 + 0.9999993667641297*x\n", - "Iteration 7100\n", - "y = 2.508535947738019e-06 + 0.9999994774776229*x\n", - "Iteration 7200\n", - "y = 2.026042884003383e-06 + 0.9999995076761915*x\n", - "Iteration 7300\n", - "y = 1.7169970615109301e-06 + 0.9999996214099223*x\n", - "Iteration 7400\n", - "y = 1.4659379241352074e-06 + 0.9999996855150445*x\n", - "Iteration 7500\n", - "y = 1.267811305769468e-06 + 0.9999997446919678*x\n", - "Iteration 7600\n", - "y = 1.0536289441560765e-06 + 0.9999997798683502*x\n", - "Iteration 7700\n", - "y = 8.727083155544904e-07 + 0.9999998075812689*x\n", - "Iteration 7800\n", - "y = 7.428466938114932e-07 + 0.9999998435690732*x\n", - "Iteration 7900\n", - "y = 6.033858150784952e-07 + 0.9999998667627915*x\n", - "Iteration 8000\n", - "y = 4.905624695607078e-07 + 0.9999998924754667*x\n", - "Iteration 8100\n", - "y = 4.1878487383983215e-07 + 0.9999999055028954*x\n", - "Iteration 8200\n", - "y = 3.4797906980686223e-07 + 0.9999999229429366*x\n", - "Iteration 8300\n", - "y = 2.9187629789054726e-07 + 0.9999999351414686*x\n", - "Iteration 8400\n", - "y = 2.3800753119535694e-07 + 0.9999999428751907*x\n", - "Iteration 8500\n", - "y = 2.1125252960406036e-07 + 0.9999999580874736*x\n", - "Iteration 8600\n", - "y = 1.7275538359612736e-07 + 0.9999999621112141*x\n", - "Iteration 8700\n", - "y = 1.4948029160490067e-07 + 0.9999999704478516*x\n", - "Iteration 8800\n", - "y = 1.2167562136618061e-07 + 0.9999999742622795*x\n", - "Iteration 8900\n", - "y = 1.0085585252142786e-07 + 0.9999999776730842*x\n", - "Iteration 9000\n", - "y = 8.579969756011814e-08 + 0.9999999831918347*x\n", - "Iteration 9100\n", - "y = 7.27897125945934e-08 + 0.9999999850177265*x\n", - "Iteration 9200\n", - "y = 6.060072539637118e-08 + 0.9999999869821679*x\n", - "Iteration 9300\n", - "y = 4.832855949920762e-08 + 0.9999999881311815*x\n", - "Iteration 9400\n", - "y = 4.205603897144143e-08 + 0.9999999913444346*x\n", - "Iteration 9500\n", - "y = 3.578918130017679e-08 + 0.9999999926595672*x\n", - "Iteration 9600\n", - "y = 2.9520297195090112e-08 + 0.9999999938305414*x\n", - "Iteration 9700\n", - "y = 2.4525485672141094e-08 + 0.9999999948040391*x\n", - "Iteration 9800\n", - "y = 2.0603269757320282e-08 + 0.9999999958682226*x\n", - "Iteration 9900\n", - "y = 1.6352124973857764e-08 + 0.9999999961601749*x\n", - "y = 1.3978513232660109e-08 + 0.9999999970810329*x\n", - "RMSprop\n", - "Iteration 0\n", - "y = 0.6827153827276677 + 0.7136147159861074*x\n", - "Iteration 100\n", - "y = 0.4613436576630757 + 0.8911490421103776*x\n", - "Iteration 200\n", - "y = 0.1641445844953397 + 0.9765644548900715*x\n", - "Iteration 300\n", - "y = 0.005416941925482668 + 0.9958119047757997*x\n", - "Iteration 400\n", - "y = -0.008575596423691137 + 1.0009337008824903*x\n", - "Iteration 500\n", - "y = 0.0011265931342927362 + 0.9994590506235699*x\n", - "Iteration 600\n", - "y = -0.019256660349787402 + 1.0029342829038248*x\n", - "Iteration 700\n", - "y = -7.901965098594169e-05 + 0.999896746799356*x\n", - "Iteration 800\n", - "y = 0.0010274411861273501 + 0.9996722279684327*x\n", - "Iteration 900\n", - "y = -0.0002664928805054597 + 1.000216490800304*x\n", - "Iteration 1000\n", - "y = 0.022408710850394694 + 1.0025402500655463*x\n", - "Iteration 1100\n", - "y = -0.0032849047666134844 + 0.9992583541515305*x\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Iteration 1200\n", - "y = -0.0007664471045705575 + 1.0002642184776036*x\n", - "Iteration 1300\n", - "y = -0.021958706880734306 + 1.003498640175805*x\n", - "Iteration 1400\n", - "y = -0.015584512173962473 + 0.9848432560044733*x\n", - "Iteration 1500\n", - "y = -0.00013040805631450714 + 0.9996115058478232*x\n", - "Iteration 1600\n", - "y = 0.002924356459606971 + 1.0017756413448418*x\n", - "Iteration 1700\n", - "y = 0.00028922302118505097 + 0.999895063767121*x\n", - "Iteration 1800\n", - "y = 0.004173454138749314 + 0.9977761099422883*x\n", - "Iteration 1900\n", - "y = 0.008285548513884912 + 0.9972693671576253*x\n", - "Iteration 2000\n", - "y = 0.0037519184753459795 + 0.9991495583156801*x\n", - "Iteration 2100\n", - "y = -0.0009917925918777797 + 1.0006309126324515*x\n", - "Iteration 2200\n", - "y = -0.005797292073705403 + 1.001185732126061*x\n", - "Iteration 2300\n", - "y = -0.0024461824844422613 + 1.0008023430416744*x\n", - "Iteration 2400\n", - "y = -0.0018481187011078282 + 0.996790630504706*x\n", - "Iteration 2500\n", - "y = 0.015076565167656184 + 0.9940245585355336*x\n", - "Iteration 2600\n", - "y = -0.00336552791882152 + 1.000816589542082*x\n", - "Iteration 2700\n", - "y = -0.007711886970819105 + 1.0005968435053925*x\n", - "Iteration 2800\n", - "y = 0.004036458271196503 + 0.9980993641270244*x\n", - "Iteration 2900\n", - "y = 0.00755381916295596 + 0.9913713030311821*x\n", - "Iteration 3000\n", - "y = -0.007462354406979531 + 0.9990929971166894*x\n", - "Iteration 3100\n", - "y = 0.0017013839220933447 + 1.0046121935156789*x\n", - "Iteration 3200\n", - "y = 0.002549610608176222 + 0.9811771184284725*x\n", - "Iteration 3300\n", - "y = -0.0008406823225092473 + 0.9964544787926721*x\n", - "Iteration 3400\n", - "y = 0.00022657235956706143 + 1.001661743718075*x\n", - "Iteration 3500\n", - "y = 0.00927907207267838 + 1.0111509515169403*x\n", - "Iteration 3600\n", - "y = 0.0023024236036593075 + 0.9992039348865193*x\n", - "Iteration 3700\n", - "y = -0.0006597887442904594 + 1.0003505161666206*x\n", - "Iteration 3800\n", - "y = 0.0002869058615782528 + 0.9998364562886809*x\n", - "Iteration 3900\n", - "y = 0.007744411310317621 + 0.9974624502449049*x\n", - "Iteration 4000\n", - "y = 0.002807386271596544 + 0.9966061645050988*x\n", - "Iteration 4100\n", - "y = -0.003392981462673309 + 1.0124873840670952*x\n", - "Iteration 4200\n", - "y = -0.006226717369419612 + 0.9836426768214036*x\n", - "Iteration 4300\n", - "y = -0.002499263973157743 + 1.0024621234963387*x\n", - "Iteration 4400\n", - "y = 0.000786459741684581 + 0.9996068907377066*x\n", - "Iteration 4500\n", - "y = -0.013885836472497332 + 0.98501782205896*x\n", - "Iteration 4600\n", - "y = 0.0027157938203233025 + 0.9991170192133456*x\n", - "Iteration 4700\n", - "y = 0.0007138591579498406 + 1.0012813472435278*x\n", - "Iteration 4800\n", - "y = -0.012771693127577421 + 0.9775943533933739*x\n", - "Iteration 4900\n", - "y = -0.01659186680784981 + 0.9960943678488293*x\n", - "Iteration 5000\n", - "y = 0.00564418704303236 + 0.9974864589134017*x\n", - "Iteration 5100\n", - "y = 0.0002736726164160462 + 1.000533766062686*x\n", - "Iteration 5200\n", - "y = -0.0019941647807032135 + 1.0015601235206242*x\n", - "Iteration 5300\n", - "y = -0.006476602405958722 + 1.0034518749707466*x\n", - "Iteration 5400\n", - "y = 0.0007427644465457552 + 1.002728093096825*x\n", - "Iteration 5500\n", - "y = 0.0046487675269970006 + 1.002810091014998*x\n", - "Iteration 5600\n", - "y = 0.025769758554076257 + 1.0127688270680373*x\n", - "Iteration 5700\n", - "y = -0.0042951255595749405 + 1.0010716326676183*x\n", - "Iteration 5800\n", - "y = -0.008068636860778397 + 0.9877757885071299*x\n", - "Iteration 5900\n", - "y = 0.003450598977099211 + 1.001726671142766*x\n", - "Iteration 6000\n", - "y = -0.004041807231042041 + 1.0022722345787372*x\n", - "Iteration 6100\n", - "y = 0.004267783918492054 + 1.000295233436237*x\n", - "Iteration 6200\n", - "y = -0.0056225140463435986 + 1.0070122459950046*x\n", - "Iteration 6300\n", - "y = -0.003235094708712298 + 1.0004924711288574*x\n", - "Iteration 6400\n", - "y = -0.006780850359893946 + 0.9956229363195707*x\n", - "Iteration 6500\n", - "y = 0.0004787190339910564 + 1.0001032053801957*x\n", - "Iteration 6600\n", - "y = -0.003352041305418763 + 0.9950335084288989*x\n", - "Iteration 6700\n", - "y = -0.004781303872024403 + 0.9980264243399204*x\n", - "Iteration 6800\n", - "y = 0.004826125511710525 + 0.9987438685190084*x\n", - "Iteration 6900\n", - "y = 0.01686124865478895 + 1.0097884533632295*x\n", - "Iteration 7000\n", - "y = 0.005459832417801409 + 1.0047429713936475*x\n", - "Iteration 7100\n", - "y = 0.013865786101604652 + 0.9960259195432627*x\n", - "Iteration 7200\n", - "y = -0.005933092998259135 + 1.0027771611928813*x\n", - "Iteration 7300\n", - "y = 0.00862304332326407 + 0.9954828005217338*x\n", - "Iteration 7400\n", - "y = -0.0016032139468913859 + 1.0008090120392943*x\n", - "Iteration 7500\n", - "y = 0.0030326295588126574 + 0.9966219225192973*x\n", - "Iteration 7600\n", - "y = -0.013317271255791796 + 1.0040204327246824*x\n", - "Iteration 7700\n", - "y = -0.0018852242994510703 + 1.000117321206579*x\n", - "Iteration 7800\n", - "y = -0.0005638059805626334 + 0.9970121486962463*x\n", - "Iteration 7900\n", - "y = -0.00756664152504323 + 1.0004804196155637*x\n", - "Iteration 8000\n", - "y = -0.003008432788919161 + 0.9995411212472487*x\n", - "Iteration 8100\n", - "y = 0.006288770186498472 + 0.9954637861523473*x\n", - "Iteration 8200\n", - "y = -0.010488420713660969 + 1.0008535663825082*x\n", - "Iteration 8300\n", - "y = 0.0004644311563401782 + 1.0006834331070662*x\n", - "Iteration 8400\n", - "y = 0.020443405696192827 + 0.9958956003205478*x\n", - "Iteration 8500\n", - "y = 0.0009955010118736039 + 0.9987797136410242*x\n", - "Iteration 8600\n", - "y = -0.011030965683844665 + 0.9888970571552462*x\n", - "Iteration 8700\n", - "y = 1.3738329377985088e-06 + 1.000003677876151*x\n", - "Iteration 8800\n", - "y = -5.3122243081662327e-05 + 0.9999689735171635*x\n", - "Iteration 8900\n", - "y = 0.014876148409598184 + 1.004505318045117*x\n", - "Iteration 9000\n", - "y = -0.007948523568615803 + 0.9852503297758233*x\n", - "Iteration 9100\n", - "y = 0.0003128111105465842 + 0.9997217700955713*x\n", - "Iteration 9200\n", - "y = 0.005322736855551152 + 0.9971767718064711*x\n", - "Iteration 9300\n", - "y = -0.013617256809249096 + 0.9959545298171486*x\n", - "Iteration 9400\n", - "y = 0.0029412398412501507 + 0.9999989947409157*x\n", - "Iteration 9500\n", - "y = -0.00340750760661034 + 0.9983407221245729*x\n", - "Iteration 9600\n", - "y = -0.0005875504713386392 + 1.0022516904038645*x\n", - "Iteration 9700\n", - "y = -0.003536715734204159 + 1.000769397732234*x\n", - "Iteration 9800\n", - "y = 0.006625135270312074 + 0.9936483257178053*x\n", - "Iteration 9900\n", - "y = 0.0035222747520052902 + 1.005384237268717*x\n", - "y = 0.006041686587868328 + 0.9980403561026004*x\n", - "Adadelta\n", - "Iteration 0\n", - "y = 0.3737801953255442 + 0.6954232880866397*x\n", - "Iteration 100\n", - "y = 0.45446798994451765 + 0.7642810918478647*x\n", - "Iteration 200\n", - "y = 0.503724013503607 + 0.8061108885966569*x\n", - "Iteration 300\n", - "y = 0.5372449606942332 + 0.8427484125351289*x\n", - "Iteration 400\n", - "y = 0.5494500294658365 + 0.8700735464493226*x\n", - "Iteration 500\n", - "y = 0.5247347573025634 + 0.8810887878647324*x\n", - "Iteration 600\n", - "y = 0.504017795661083 + 0.8937987270543221*x\n", - "Iteration 700\n", - "y = 0.4749651708372001 + 0.8971005764339413*x\n", - "Iteration 800\n", - "y = 0.4400397105308388 + 0.9014096500974156*x\n", - "Iteration 900\n", - "y = 0.40824997114309086 + 0.9134682609947273*x\n", - "Iteration 1000\n", - "y = 0.37381393291084897 + 0.9194850353352613*x\n", - "Iteration 1100\n", - "y = 0.3517276407239353 + 0.931088854072381*x\n", - "Iteration 1200\n", - "y = 0.3155689840081007 + 0.9297539531195199*x\n", - "Iteration 1300\n", - "y = 0.28516476451211253 + 0.9410833878329703*x\n", - "Iteration 1400\n", - "y = 0.25151839633081546 + 0.9468630317486699*x\n", - "Iteration 1500\n", - "y = 0.2187631891156385 + 0.9536963275180232*x\n", - "Iteration 1600\n", - "y = 0.18813494915312937 + 0.9599508678970319*x\n", - "Iteration 1700\n", - "y = 0.16346802017358564 + 0.9681810668133642*x\n", - "Iteration 1800\n", - "y = 0.1427335223054219 + 0.9726379550723799*x\n", - "Iteration 1900\n", - "y = 0.11095931576581414 + 0.9756626085219838*x\n", - "Iteration 2000\n", - "y = 0.08467626594832124 + 0.9809827312292054*x\n", - "Iteration 2100\n", - "y = 0.055776093501635086 + 0.986682127065862*x\n", - "Iteration 2200\n", - "y = 0.03128161032349964 + 0.994151344816365*x\n", - "Iteration 2300\n", - "y = 0.011134563694370887 + 0.9980101133207862*x\n", - "Iteration 2400\n", - "y = -0.0002819231467387675 + 0.9994292604031647*x\n", - "Iteration 2500\n", - "y = -0.0007530001080388465 + 0.9996876078971861*x\n", - "Iteration 2600\n", - "y = 1.4299320787029975e-05 + 1.0002415850327648*x\n", - "Iteration 2700\n", - "y = 0.00048437023526542024 + 1.000636329524222*x\n", - "Iteration 2800\n", - "y = 0.00017081810272224221 + 1.0002339219595149*x\n", - "Iteration 2900\n", - "y = -0.00047720186234186763 + 1.0002837940293001*x\n", - "Iteration 3000\n", - "y = -0.00048827011918332745 + 1.0000810463921026*x\n", - "Iteration 3100\n", - "y = -0.0001568847949445286 + 0.9999644370576438*x\n", - "Iteration 3200\n", - "y = -0.0008464061365265306 + 1.000160614161929*x\n", - "Iteration 3300\n", - "y = -0.0012632585693276611 + 1.0008587748768185*x\n", - "Iteration 3400\n", - "y = 8.936672728558989e-05 + 0.9998637431800027*x\n", - "Iteration 3500\n", - "y = 0.00028773253132511624 + 0.999979297608201*x\n", - "Iteration 3600\n", - "y = -0.0005940538060823068 + 0.9997986460772728*x\n", - "Iteration 3700\n", - "y = -0.00017555787997768101 + 0.9992967782058932*x\n", - "Iteration 3800\n", - "y = -0.0010902486276842423 + 0.9993665313074814*x\n", - "Iteration 3900\n", - "y = 0.00011090105568572678 + 0.9999390648187347*x\n", - "Iteration 4000\n", - "y = -0.0007925602555049502 + 1.0003670190881373*x\n", - "Iteration 4100\n", - "y = -9.85030472217507e-05 + 1.0000452185396647*x\n", - "Iteration 4200\n", - "y = -0.001322185011795518 + 1.000252860600806*x\n", - "Iteration 4300\n", - "y = -0.000622778427580351 + 0.9996172454917239*x\n", - "Iteration 4400\n", - "y = 0.0008793034307541049 + 0.9991323399931926*x\n", - "Iteration 4500\n", - "y = 2.2097921067741522e-05 + 0.9999916633445198*x\n", - "Iteration 4600\n", - "y = -0.00013584268124752416 + 1.000083372558365*x\n", - "Iteration 4700\n", - "y = -0.0003792950127671934 + 1.0002199515238068*x\n", - "Iteration 4800\n", - "y = 0.0007407910291384943 + 1.0002752312416903*x\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Iteration 4900\n", - "y = -0.00018293553284564853 + 1.000084353469475*x\n", - "Iteration 5000\n", - "y = -0.00021086202011657087 + 1.000599467500034*x\n", - "Iteration 5100\n", - "y = -0.0002698118770557987 + 1.0002670984772302*x\n", - "Iteration 5200\n", - "y = -0.000174041228809895 + 1.0000873631031082*x\n", - "Iteration 5300\n", - "y = -0.000368303888004153 + 0.9988172336751648*x\n", - "Iteration 5400\n", - "y = 3.23267012324346e-08 + 1.0000015527829704*x\n", - "Iteration 5500\n", - "y = -0.0007458676117796859 + 1.0002497117687457*x\n", - "Iteration 5600\n", - "y = -0.0002750920909796192 + 0.9999100367963439*x\n", - "Iteration 5700\n", - "y = -0.0002603746682872479 + 1.000235272525123*x\n", - "Iteration 5800\n", - "y = 0.0006660635547060155 + 0.9997308283380719*x\n", - "Iteration 5900\n", - "y = 0.0007499650195894795 + 0.9998309531696397*x\n", - "Iteration 6000\n", - "y = 0.00026793080973281725 + 0.9999254737646864*x\n", - "Iteration 6100\n", - "y = -7.108162650266238e-05 + 1.0000467252894638*x\n", - "Iteration 6200\n", - "y = 2.7908004826124246e-05 + 0.9999831010503192*x\n", - "Iteration 6300\n", - "y = 7.319170492363453e-05 + 0.9999735463240744*x\n", - "Iteration 6400\n", - "y = 0.00010198021235678255 + 0.9999497167118081*x\n", - "Iteration 6500\n", - "y = -2.431716475506412e-06 + 1.000008780986074*x\n", - "Iteration 6600\n", - "y = 0.0003694589641208747 + 1.0000175469336479*x\n", - "Iteration 6700\n", - "y = -1.4022966755159485e-06 + 0.9999992007604116*x\n", - "Iteration 6800\n", - "y = -8.633507199438527e-05 + 1.000240477279819*x\n", - "Iteration 6900\n", - "y = 0.00044989195448772416 + 0.9999453497703747*x\n", - "Iteration 7000\n", - "y = -5.879042758639982e-05 + 1.0000300657398191*x\n", - "Iteration 7100\n", - "y = -0.0009824183149325324 + 0.9991641172677947*x\n", - "Iteration 7200\n", - "y = 9.609852626984948e-05 + 0.9999036811155302*x\n", - "Iteration 7300\n", - "y = -0.00019017894587889544 + 0.9999989991196587*x\n", - "Iteration 7400\n", - "y = 5.42016651517267e-05 + 0.9999736784994304*x\n", - "Iteration 7500\n", - "y = -0.00023740571251060384 + 0.9999013240309537*x\n", - "Iteration 7600\n", - "y = -0.00017011932836109704 + 1.000030745217648*x\n", - "Iteration 7700\n", - "y = -0.00035492964049595643 + 0.9996757749060617*x\n", - "Iteration 7800\n", - "y = -0.0004319532086695726 + 0.9999177720542175*x\n", - "Iteration 7900\n", - "y = -4.734275611628327e-08 + 0.9999378842893045*x\n", - "Iteration 8000\n", - "y = -2.209857409748423e-05 + 1.0000121727293267*x\n", - "Iteration 8100\n", - "y = 0.00029399636729362117 + 0.9998609266875158*x\n", - "Iteration 8200\n", - "y = -0.0001310883704963831 + 1.0000817661441546*x\n", - "Iteration 8300\n", - "y = 0.0008041036490104212 + 1.0000502734574366*x\n", - "Iteration 8400\n", - "y = -0.0005327049113160294 + 0.99990390969119*x\n", - "Iteration 8500\n", - "y = -0.00030909670698362434 + 0.9997314940272098*x\n", - "Iteration 8600\n", - "y = -0.0004119597523058274 + 0.9992781835784582*x\n", - "Iteration 8700\n", - "y = -1.5946249925514515e-05 + 1.00002552381485*x\n", - "Iteration 8800\n", - "y = -2.2331167035739685e-05 + 1.0000347524312192*x\n", - "Iteration 8900\n", - "y = -0.00022048165850817618 + 0.9996719966521392*x\n", - "Iteration 9000\n", - "y = -0.0003105653255593682 + 0.9999788024689434*x\n", - "Iteration 9100\n", - "y = 9.668358292685267e-05 + 0.9999706995785885*x\n", - "Iteration 9200\n", - "y = 0.00010238247450770996 + 0.9998004937879871*x\n", - "Iteration 9300\n", - "y = 0.00037115928665625714 + 1.000287762566172*x\n", - "Iteration 9400\n", - "y = -6.118416352593448e-05 + 1.000107454775657*x\n", - "Iteration 9500\n", - "y = -0.0009550721070318215 + 0.9996910774648767*x\n", - "Iteration 9600\n", - "y = 7.165902952687223e-05 + 1.0002360153679417*x\n", - "Iteration 9700\n", - "y = -0.0001307949755324461 + 1.0000442759668344*x\n", - "Iteration 9800\n", - "y = 0.0007364117746714955 + 0.9998109609245667*x\n", - "Iteration 9900\n", - "y = -0.0003427528909499275 + 1.0000402856136628*x\n", - "y = 0.0003127360718812701 + 1.0004689899136678*x\n" + "y = -2.4103486747254733e-08 + 0.9999999984873583*x\n", + "y = 4.7124723585610135e-07 + 0.9999992952944199*x\n" ] } ], @@ -996,6 +382,7 @@ "xs = [1,2,3,4,5,6,7]\n", "ys = [1,2,3,4,5,6,7]\n", "\n", + "\n", "# Gradient Descent\n", "model = Line()\n", "print(\"Gradient Descent: \")\n", @@ -1030,702 +417,47 @@ "model = Line()\n", "print(\"Adadelta\")\n", "adadelta(model, xs, ys)\n", + "print(model)\n", + "\n", + "\n", + "# Adam\n", + "model = Line()\n", + "print(\"Adam\")\n", + "adam(model, xs, ys)\n", "print(model)" ] }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 28, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "Gradient Descent: \n", - "Iteration 0\n", - "y = 0.6965931374130392 + 0.7601122786401455*x\n", - "Iteration 100\n", - "y = 0.617623544918069 + 1.8757445461082207*x\n", - "Iteration 200\n", - "y = 0.4176232831814616 + 1.9159812299345367*x\n", - "Iteration 300\n", - "y = 0.28238756130710596 + 1.9431883792444125*x\n", - "Iteration 400\n", - "y = 0.1909441786231191 + 1.961585247554065*x\n", - "Iteration 500\n", - "y = 0.12911220020207081 + 1.9740248001050498*x\n", - "Iteration 600\n", - "y = 0.0873027937338816 + 1.9824361484423931*x\n", - "Iteration 700\n", - "y = 0.059032204406802784 + 1.988123714820858*x\n", - "Iteration 800\n", - "y = 0.039916261646207894 + 1.9919695205124175*x\n", - "Iteration 900\n", - "y = 0.02699048696926038 + 1.9945699686536877*x\n", - "y = 0.01832191952439833 + 1.9963139384089141*x\n", - "Stochastic Gradient Descent: \n", - "Iteration 0\n", - "y = 0.9619258265142657 + 1.087046466635935*x\n", - "Iteration 100\n", - "y = 0.7920127114741361 + 1.8730271390796145*x\n", - "Iteration 200\n", - "y = 0.546286989963847 + 1.8525856253438988*x\n", - "Iteration 300\n", - "y = 0.3750536398579547 + 1.9349436535828175*x\n", - "Iteration 400\n", - "y = 0.2635085249438779 + 1.9449615901242314*x\n", - "Iteration 500\n", - "y = 0.1866254287868797 + 1.9626152126627878*x\n", - "Iteration 600\n", - "y = 0.1223567694333187 + 1.9718568321569878*x\n", - "Iteration 700\n", - "y = 0.08274738570538319 + 1.9785419924467536*x\n", - "Iteration 800\n", - "y = 0.05431416501871665 + 1.9865732519804664*x\n", - "Iteration 900\n", - "y = 0.03537368025849358 + 1.9898993717967888*x\n", - "y = 0.024016556578563636 + 1.9957094970178637*x\n", - "SGD + Momentum: \n", - "Iteration 0\n", - "y = 0.4357015493218762 + 0.16313132537794608*x\n", - "Iteration 100\n", - "y = 0.03437429906401527 + 2.0233986169269054*x\n", - "Iteration 200\n", - "y = 0.007032959165979722 + 1.9977607487712188*x\n", - "Iteration 300\n", - "y = 0.00041581701105788846 + 1.9998442725098746*x\n", - "Iteration 400\n", - "y = -1.6005163091342098e-06 + 2.000016204187183*x\n", - "Iteration 500\n", - "y = -8.861934936492956e-08 + 1.9999994861022012*x\n", - "Iteration 600\n", - "y = 6.928889468269653e-06 + 1.9999949280688167*x\n", - "Iteration 700\n", - "y = 6.597788280758348e-07 + 2.0000005322576277*x\n", - "Iteration 800\n", - "y = 4.581482668701209e-07 + 1.9999999584079404*x\n", - "Iteration 900\n", - "y = -8.513195132279453e-07 + 2.000002384972658*x\n", - "y = 4.769865694709039e-07 + 1.9999995706104974*x\n", - "Adagrad\n", - "Iteration 0\n", - "y = 0.9909007129257088 + 0.6851578166537742*x\n", - "Iteration 100\n", - "y = 1.752082669089777 + 1.581579077593311*x\n", - "Iteration 200\n", - "y = 1.6531567114050418 + 1.6606835046567618*x\n", - "Iteration 300\n", - "y = 1.5450696681444174 + 1.6823053575450688*x\n", - "Iteration 400\n", - "y = 1.4663489934198473 + 1.7070489395736645*x\n", - "Iteration 500\n", - "y = 1.351363888376385 + 1.7130720212360526*x\n", - "Iteration 600\n", - "y = 1.2678086511145235 + 1.7278507573766195*x\n", - "Iteration 700\n", - "y = 1.1712706810563158 + 1.7451602089657614*x\n", - "Iteration 800\n", - "y = 1.1132576537190118 + 1.7719659013229505*x\n", - "Iteration 900\n", - "y = 1.035517123163937 + 1.7899675001681676*x\n", - "Iteration 1000\n", - "y = 0.9674062829579255 + 1.7948502076582695*x\n", - "Iteration 1100\n", - "y = 0.9082204674720814 + 1.8071128351181995*x\n", - "Iteration 1200\n", - "y = 0.8581451473712463 + 1.8154408505731354*x\n", - "Iteration 1300\n", - "y = 0.8089968516149708 + 1.8328093166423218*x\n", - "Iteration 1400\n", - "y = 0.7657955053734683 + 1.8392007149707281*x\n", - "Iteration 1500\n", - "y = 0.7253357493250404 + 1.8516605673401425*x\n", - "Iteration 1600\n", - "y = 0.682539387341273 + 1.859319366876271*x\n", - "Iteration 1700\n", - "y = 0.6369999006640653 + 1.8634514515704137*x\n", - "Iteration 1800\n", - "y = 0.598817873375494 + 1.8673468379777314*x\n", - "Iteration 1900\n", - "y = 0.5663728642652257 + 1.8745579902145202*x\n", - "Iteration 2000\n", - "y = 0.5397001954509365 + 1.8897515255822968*x\n", - "Iteration 2100\n", - "y = 0.5037008810421729 + 1.892403543540384*x\n", - "Iteration 2200\n", - "y = 0.47389861922693477 + 1.89944214324104*x\n", - "Iteration 2300\n", - "y = 0.44450868566473434 + 1.90549298291394*x\n", - "Iteration 2400\n", - "y = 0.4214024035128387 + 1.9111343460932955*x\n", - "Iteration 2500\n", - "y = 0.3952791860604048 + 1.9142752531345693*x\n", - "Iteration 2600\n", - "y = 0.3659430889961222 + 1.916988189656578*x\n", - "Iteration 2700\n", - "y = 0.3483948220969853 + 1.9249129004722139*x\n", - "Iteration 2800\n", - "y = 0.33235503260927507 + 1.9301450417446813*x\n", - "Iteration 2900\n", - "y = 0.3197901105644779 + 1.936407582605034*x\n", - "Iteration 3000\n", - "y = 0.29976305689632343 + 1.9373260640754866*x\n", - "Iteration 3100\n", - "y = 0.2843666968242483 + 1.941735934755993*x\n", - "Iteration 3200\n", - "y = 0.2692031077146555 + 1.9448569527675903*x\n", - "Iteration 3300\n", - "y = 0.2488251455784735 + 1.9452466519871805*x\n", - "Iteration 3400\n", - "y = 0.2342843571290378 + 1.9501046582154946*x\n", - "Iteration 3500\n", - "y = 0.21988661911352536 + 1.9533270428254617*x\n", - "Iteration 3600\n", - "y = 0.2105228727400937 + 1.955795462193796*x\n", - "Iteration 3700\n", - "y = 0.19493901199836053 + 1.9561507881440283*x\n", - "Iteration 3800\n", - "y = 0.18880117483148465 + 1.9610708152335732*x\n", - "Iteration 3900\n", - "y = 0.17803176767432233 + 1.9625444060857709*x\n", - "Iteration 4000\n", - "y = 0.16711441869227095 + 1.9646134303791487*x\n", - "Iteration 4100\n", - "y = 0.15810281894455053 + 1.9669602997188*x\n", - "Iteration 4200\n", - "y = 0.15287490128429693 + 1.9693218801239154*x\n", - "Iteration 4300\n", - "y = 0.14065981580502707 + 1.968745417771871*x\n", - "Iteration 4400\n", - "y = 0.13544320458564318 + 1.972610509170239*x\n", - "Iteration 4500\n", - "y = 0.12713739736917717 + 1.9732876288943733*x\n", - "Iteration 4600\n", - "y = 0.12073607702270527 + 1.9754767014209547*x\n", - "Iteration 4700\n", - "y = 0.11455068545005868 + 1.976473259600258*x\n", - "Iteration 4800\n", - "y = 0.10887697216212634 + 1.9785797402131986*x\n", - "Iteration 4900\n", - "y = 0.10250516615793719 + 1.9790490254209663*x\n", - "Iteration 5000\n", - "y = 0.09522008487108682 + 1.9796346706769523*x\n", - "Iteration 5100\n", - "y = 0.08858127597176003 + 1.9810893662180828*x\n", - "Iteration 5200\n", - "y = 0.08483749892988729 + 1.982768861515039*x\n", - "Iteration 5300\n", - "y = 0.07886767929269177 + 1.9836549838799284*x\n", - "Iteration 5400\n", - "y = 0.07311361272510263 + 1.9838669836579859*x\n", - "Iteration 5500\n", - "y = 0.06894011176345524 + 1.985044887714682*x\n", - "Iteration 5600\n", - "y = 0.06561188197000849 + 1.9859855261439086*x\n", - "Iteration 5700\n", - "y = 0.06198253754597117 + 1.9875182936539633*x\n", - "Iteration 5800\n", - "y = 0.05818535248057164 + 1.9877238348206285*x\n", - "Iteration 5900\n", - "y = 0.05486798438629148 + 1.9885673727902582*x\n", - "Iteration 6000\n", - "y = 0.05203838159661378 + 1.9891546755284164*x\n", - "Iteration 6100\n", - "y = 0.0484075162579466 + 1.9893463417367592*x\n", - "Iteration 6200\n", - "y = 0.04581501469093452 + 1.9902166936933152*x\n", - "Iteration 6300\n", - "y = 0.0436656810183715 + 1.9905747385133217*x\n", - "Iteration 6400\n", - "y = 0.041312642161708844 + 1.9915114489308239*x\n", - "Iteration 6500\n", - "y = 0.03940855403744079 + 1.9920867082705092*x\n", - "Iteration 6600\n", - "y = 0.03701409021754122 + 1.9923329499725895*x\n", - "Iteration 6700\n", - "y = 0.03492936297892485 + 1.9926572631404766*x\n", - "Iteration 6800\n", - "y = 0.03348341014569901 + 1.9932937318692237*x\n", - "Iteration 6900\n", - "y = 0.031576389769188445 + 1.9935755020721362*x\n", - "Iteration 7000\n", - "y = 0.029450949428277903 + 1.9937487240433411*x\n", - "Iteration 7100\n", - "y = 0.027849462455878844 + 1.994039140685735*x\n", - "Iteration 7200\n", - "y = 0.02645907129728384 + 1.9945658709046947*x\n", - "Iteration 7300\n", - "y = 0.024728927031110553 + 1.9948328724610187*x\n", - "Iteration 7400\n", - "y = 0.02364076205033738 + 1.995301333905134*x\n", - "Iteration 7500\n", - "y = 0.022164224351641244 + 1.9953898754328938*x\n", - "Iteration 7600\n", - "y = 0.020645285154022756 + 1.995577735066188*x\n", - "Iteration 7700\n", - "y = 0.01959386986885427 + 1.9958031488469503*x\n", - "Iteration 7800\n", - "y = 0.018395035579167336 + 1.9960658262335285*x\n", - "Iteration 7900\n", - "y = 0.017550365216273463 + 1.996400177358067*x\n", - "Iteration 8000\n", - "y = 0.01646944688515055 + 1.996364976041052*x\n", - "Iteration 8100\n", - "y = 0.01572605806439289 + 1.996716659515437*x\n", - "Iteration 8200\n", - "y = 0.015200581929641493 + 1.9970621567834477*x\n", - "Iteration 8300\n", - "y = 0.014138795270536781 + 1.9971356679677228*x\n", - "Iteration 8400\n", - "y = 0.013464774825949283 + 1.9972932631081701*x\n", - "Iteration 8500\n", - "y = 0.012706131057463436 + 1.9974110679122312*x\n", - "Iteration 8600\n", - "y = 0.012066268208895998 + 1.9976311955123802*x\n", - "Iteration 8700\n", - "y = 0.011173160510520747 + 1.9976373822300846*x\n", - "Iteration 8800\n", - "y = 0.010500077142079577 + 1.9977509290847406*x\n", - "Iteration 8900\n", - "y = 0.009955166444508366 + 1.9979002520238816*x\n", - "Iteration 9000\n", - "y = 0.009360975496985422 + 1.9979921570347314*x\n", - "Iteration 9100\n", - "y = 0.00900368092345526 + 1.9982302007810173*x\n", - "Iteration 9200\n", - "y = 0.008569581519229328 + 1.9982885032552313*x\n", - "Iteration 9300\n", - "y = 0.008085989294916947 + 1.9984215977579864*x\n", - "Iteration 9400\n", - "y = 0.00752066029356973 + 1.9984226193340426*x\n", - "Iteration 9500\n", - "y = 0.007164866359334228 + 1.9985243732263827*x\n", - "Iteration 9600\n", - "y = 0.006820207250137622 + 1.9985814746697566*x\n", - "Iteration 9700\n", - "y = 0.006457444952760931 + 1.9986186533358985*x\n", - "Iteration 9800\n", - "y = 0.006187705321134588 + 1.998739250433351*x\n", - "Iteration 9900\n", - "y = 0.005910240818639075 + 1.9988259362219711*x\n", - "y = 0.0055443237796650715 + 1.9988402031873795*x\n", - "RMSprop\n", - "Iteration 0\n", - "y = 0.3617918714937247 + 0.8935314611793963*x\n", - "Iteration 100\n", - "y = 1.112143205305486 + 1.5829729739940772*x\n", - "Iteration 200\n", - "y = 1.0585978795258977 + 1.7833251480156982*x\n", - "Iteration 300\n", - "y = 0.7407569067399042 + 1.86060673080023*x\n", - "Iteration 400\n", - "y = 0.42064360806770007 + 1.9173416530941951*x\n", - "Iteration 500\n", - "y = 0.1111922206643335 + 1.9781148405601097*x\n", - "Iteration 600\n", - "y = 0.001379989704715848 + 2.000825479832667*x\n", - "Iteration 700\n", - "y = -0.005263769903600025 + 1.9951254183042317*x\n", - "Iteration 800\n", - "y = 0.002066339142489657 + 2.000527389852606*x\n", - "Iteration 900\n", - "y = -0.0018974037452918922 + 2.0004246592997204*x\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Iteration 1000\n", - "y = -0.039749095115406675 + 1.987267555742025*x\n", - "Iteration 1100\n", - "y = 0.005145669157455596 + 1.9984807726790454*x\n", - "Iteration 1200\n", - "y = -0.0018140213445952532 + 2.0037880205456013*x\n", - "Iteration 1300\n", - "y = -0.01278392583858953 + 2.0039946207483874*x\n", - "Iteration 1400\n", - "y = 0.0006377257328787176 + 1.999931949377187*x\n", - "Iteration 1500\n", - "y = 0.0009988082670943522 + 1.9943520576447586*x\n", - "Iteration 1600\n", - "y = -0.01806246811933287 + 1.990932459663341*x\n", - "Iteration 1700\n", - "y = 0.003213682754240674 + 1.999003713429964*x\n", - "Iteration 1800\n", - "y = 0.011017420383813752 + 1.9970627341422795*x\n", - "Iteration 1900\n", - "y = -0.009959479920880443 + 2.0064445494038146*x\n", - "Iteration 2000\n", - "y = 0.005708949443180763 + 1.998207036748069*x\n", - "Iteration 2100\n", - "y = -0.0018150808151644426 + 2.0003408117602763*x\n", - "Iteration 2200\n", - "y = -0.00026099203332354817 + 1.998588111772451*x\n", - "Iteration 2300\n", - "y = -0.008553796181148434 + 1.9855130641651118*x\n", - "Iteration 2400\n", - "y = 0.00021324745550753877 + 2.0011468628037545*x\n", - "Iteration 2500\n", - "y = 0.0003281239054911537 + 1.9998022119570147*x\n", - "Iteration 2600\n", - "y = -0.007675847124862011 + 1.9886455178959226*x\n", - "Iteration 2700\n", - "y = 0.0008883708558365342 + 2.0000632313132063*x\n", - "Iteration 2800\n", - "y = -0.00687138003536456 + 2.001969910081975*x\n", - "Iteration 2900\n", - "y = -0.002180629593012344 + 1.999979421229616*x\n", - "Iteration 3000\n", - "y = 0.023770950558197647 + 1.9936958717485012*x\n", - "Iteration 3100\n", - "y = -0.0009907633618757314 + 2.000043934000511*x\n", - "Iteration 3200\n", - "y = 0.000420119600382865 + 1.9998617994403107*x\n", - "Iteration 3300\n", - "y = 0.01453001408007369 + 1.9934154454703121*x\n", - "Iteration 3400\n", - "y = -0.0026727121181083133 + 2.0009139063502044*x\n", - "Iteration 3500\n", - "y = 0.013028222857044768 + 2.007482456109788*x\n", - "Iteration 3600\n", - "y = -0.00121031965803373 + 2.0002209422805435*x\n", - "Iteration 3700\n", - "y = -0.007866015881214185 + 2.0037779537526617*x\n", - "Iteration 3800\n", - "y = 0.002897638917493862 + 1.998571751217742*x\n", - "Iteration 3900\n", - "y = -0.0014974180283573396 + 2.0003637721136536*x\n", - "Iteration 4000\n", - "y = -0.000671947083914073 + 1.999472014677302*x\n", - "Iteration 4100\n", - "y = 0.00498768427618511 + 1.9983437312880752*x\n", - "Iteration 4200\n", - "y = -0.00831301984785493 + 2.0041549228809497*x\n", - "Iteration 4300\n", - "y = -0.00031632027945036853 + 1.9998090048269228*x\n", - "Iteration 4400\n", - "y = 0.001069989178476908 + 1.9995701928587326*x\n", - "Iteration 4500\n", - "y = -0.0026546082920498853 + 2.000350406236675*x\n", - "Iteration 4600\n", - "y = -0.003535097434274846 + 1.9940003125763315*x\n", - "Iteration 4700\n", - "y = 0.0029590453393060993 + 2.002472261531745*x\n", - "Iteration 4800\n", - "y = 0.006296740927637731 + 1.9984689158019062*x\n", - "Iteration 4900\n", - "y = 0.005277866570282745 + 2.009966222300057*x\n", - "Iteration 5000\n", - "y = -0.0036468857223469213 + 1.9982953157903156*x\n", - "Iteration 5100\n", - "y = 0.007120604467174619 + 2.003384798643488*x\n", - "Iteration 5200\n", - "y = -0.0008533823564239276 + 2.0004744615977263*x\n", - "Iteration 5300\n", - "y = 0.002027762405976359 + 1.998254300772477*x\n", - "Iteration 5400\n", - "y = 0.0036023905324369518 + 1.9962283192455492*x\n", - "Iteration 5500\n", - "y = -0.006259424080465047 + 2.00230533560054*x\n", - "Iteration 5600\n", - "y = 0.008393599939131786 + 2.019473618148715*x\n", - "Iteration 5700\n", - "y = -0.004050781931848291 + 2.0028367476512647*x\n", - "Iteration 5800\n", - "y = -0.0008371108760339479 + 2.0139258788525116*x\n", - "Iteration 5900\n", - "y = -0.002885408986600588 + 2.0020799185868388*x\n", - "Iteration 6000\n", - "y = 0.0053486797844405146 + 2.0024814243971716*x\n", - "Iteration 6100\n", - "y = 0.0033502490503319856 + 1.9977984418251111*x\n", - "Iteration 6200\n", - "y = 0.006127720551620504 + 2.0016390400463493*x\n", - "Iteration 6300\n", - "y = -0.001822037695541198 + 2.0018140619687257*x\n", - "Iteration 6400\n", - "y = -0.01079328552863512 + 2.0037761462415054*x\n", - "Iteration 6500\n", - "y = 0.00717505822557291 + 2.0002046020407143*x\n", - "Iteration 6600\n", - "y = 0.008800774749567117 + 2.0083165181185345*x\n", - "Iteration 6700\n", - "y = -0.014007435444718863 + 1.9983551539585414*x\n", - "Iteration 6800\n", - "y = 0.0051346133060544774 + 2.000154001014758*x\n", - "Iteration 6900\n", - "y = -0.003621173344141699 + 2.000588337910482*x\n", - "Iteration 7000\n", - "y = 0.0016322509363011957 + 1.9993573151771353*x\n", - "Iteration 7100\n", - "y = -0.001304944778832026 + 2.000625310850428*x\n", - "Iteration 7200\n", - "y = 0.002067417129937262 + 2.000184889451102*x\n", - "Iteration 7300\n", - "y = -0.005988455813080613 + 2.0077379629708*x\n", - "Iteration 7400\n", - "y = -0.00802139938656621 + 1.9992232943780295*x\n", - "Iteration 7500\n", - "y = -0.00231443475705327 + 2.0005518272429454*x\n", - "Iteration 7600\n", - "y = 0.0033070611934870944 + 2.0008275114675347*x\n", - "Iteration 7700\n", - "y = -0.006321870496990727 + 1.9997121317668411*x\n", - "Iteration 7800\n", - "y = 0.00041857429593791275 + 1.9923646157248096*x\n", - "Iteration 7900\n", - "y = -0.02528608229047568 + 1.986590769618216*x\n", - "Iteration 8000\n", - "y = -0.01938780384403209 + 2.008172789141479*x\n", - "Iteration 8100\n", - "y = 0.006421405527212393 + 1.9842548123557364*x\n", - "Iteration 8200\n", - "y = 0.01589584925668931 + 2.0083314389292974*x\n", - "Iteration 8300\n", - "y = -0.016649136579011476 + 1.9853758028242927*x\n", - "Iteration 8400\n", - "y = -1.51266700507626e-05 + 2.0000118355513514*x\n", - "Iteration 8500\n", - "y = -5.361102978206546e-05 + 2.0000177909536254*x\n", - "Iteration 8600\n", - "y = -0.0075402522254625366 + 1.9898598252275863*x\n", - "Iteration 8700\n", - "y = 0.010084811825024608 + 1.9984178068716432*x\n", - "Iteration 8800\n", - "y = -0.006582726540762198 + 2.0046617518584764*x\n", - "Iteration 8900\n", - "y = -0.0006801046136838308 + 1.9995312304839528*x\n", - "Iteration 9000\n", - "y = -0.005396359895809784 + 1.9997415176351763*x\n", - "Iteration 9100\n", - "y = -0.01029870164535155 + 2.001326738259382*x\n", - "Iteration 9200\n", - "y = -0.008363402959870082 + 2.001004330294006*x\n", - "Iteration 9300\n", - "y = -0.004623361779183724 + 2.002624012829024*x\n", - "Iteration 9400\n", - "y = 0.0023337854043705823 + 1.9991196683520087*x\n", - "Iteration 9500\n", - "y = -0.003479605462548233 + 1.9991766085646165*x\n", - "Iteration 9600\n", - "y = 1.9113254449676645e-05 + 1.9999998433258432*x\n", - "Iteration 9700\n", - "y = 0.0004901021539388111 + 1.9996842127475674*x\n", - "Iteration 9800\n", - "y = -0.0037188565728199625 + 2.0012411521197166*x\n", - "Iteration 9900\n", - "y = 0.0044477763680744 + 1.9999679396358498*x\n", - "y = -0.0011675162828293233 + 1.9985121181517174*x\n", - "Adadelta\n", + "Adam\n", "Iteration 0\n", - "y = 0.41096539511307256 + 0.49772101678589104*x\n", + "y = 0.7770901717569257 + 0.9433753832651581*x\n", "Iteration 100\n", - "y = 0.5135798913712705 + 0.5696340910509599*x\n", + "y = 0.32725296586754515 + 1.900221970247115*x\n", "Iteration 200\n", - "y = 0.6159518945690069 + 0.6388291385750045*x\n", + "y = 0.004147083459202631 + 1.99892111204854*x\n", "Iteration 300\n", - "y = 0.7092237980405368 + 0.6970127295556214*x\n", + "y = -3.4688751132678575e-05 + 2.000013509462116*x\n", "Iteration 400\n", - "y = 0.8000787749721939 + 0.7543026226018996*x\n", + "y = -9.06655931188767e-08 + 1.9999999764669432*x\n", "Iteration 500\n", - "y = 0.888457081567781 + 0.8115820641162809*x\n", + "y = 1.8387422421644338e-10 + 2.0000000000594307*x\n", "Iteration 600\n", - "y = 0.9700712825110333 + 0.8664154632647859*x\n", + "y = 4.361065425337935e-12 + 1.9999999999993485*x\n", "Iteration 700\n", - "y = 1.052913095059698 + 0.9228620322870312*x\n", + "y = 1.1991224399817034e-13 + 1.9999999999999527*x\n", "Iteration 800\n", - "y = 1.1427113559550428 + 0.9872195554878949*x\n", + "y = -2.5150603804494032e-17 + 2.000000000000001*x\n", "Iteration 900\n", - "y = 1.2156348797474466 + 1.0393615432768293*x\n", - "Iteration 1000\n", - "y = 1.28370930318869 + 1.0928250473441095*x\n", - "Iteration 1100\n", - "y = 1.3658196689296205 + 1.1554026707086344*x\n", - "Iteration 1200\n", - "y = 1.4392769212672991 + 1.2133508338684948*x\n", - "Iteration 1300\n", - "y = 1.4980452955086607 + 1.2596297240563434*x\n", - "Iteration 1400\n", - "y = 1.5525476509395997 + 1.304787577243706*x\n", - "Iteration 1500\n", - "y = 1.595337893009435 + 1.3434453631056735*x\n", - "Iteration 1600\n", - "y = 1.640360027928069 + 1.3864646800798137*x\n", - "Iteration 1700\n", - "y = 1.6844946083322918 + 1.428290059679983*x\n", - "Iteration 1800\n", - "y = 1.7255113510793385 + 1.472600826345899*x\n", - "Iteration 1900\n", - "y = 1.743913071501007 + 1.5053963988127388*x\n", - "Iteration 2000\n", - "y = 1.759343796504421 + 1.5334651004381192*x\n", - "Iteration 2100\n", - "y = 1.776818555354216 + 1.5628279732789563*x\n", - "Iteration 2200\n", - "y = 1.78362773959923 + 1.5917055647784415*x\n", - "Iteration 2300\n", - "y = 1.767585502021221 + 1.6096269021826028*x\n", - "Iteration 2400\n", - "y = 1.7387011247315654 + 1.6183577319981461*x\n", - "Iteration 2500\n", - "y = 1.698660507418387 + 1.6284167901121807*x\n", - "Iteration 2600\n", - "y = 1.6669960589851716 + 1.63823204083993*x\n", - "Iteration 2700\n", - "y = 1.6343083984307476 + 1.6492788741150122*x\n", - "Iteration 2800\n", - "y = 1.613170732340005 + 1.664159395506083*x\n", - "Iteration 2900\n", - "y = 1.5612647587480215 + 1.657932217894772*x\n", - "Iteration 3000\n", - "y = 1.5286468803675302 + 1.6655548278266634*x\n", - "Iteration 3100\n", - "y = 1.5091490416926587 + 1.6810679423849475*x\n", - "Iteration 3200\n", - "y = 1.471892557581634 + 1.687024489777794*x\n", - "Iteration 3300\n", - "y = 1.4388786269393954 + 1.6967734369019145*x\n", - "Iteration 3400\n", - "y = 1.3959013004978815 + 1.7024133205324061*x\n", - "Iteration 3500\n", - "y = 1.3561811530406536 + 1.707095283009188*x\n", - "Iteration 3600\n", - "y = 1.3371628988825486 + 1.7206721934439055*x\n", - "Iteration 3700\n", - "y = 1.2966889049328 + 1.723337381347839*x\n", - "Iteration 3800\n", - "y = 1.2662759661567522 + 1.7344195725906013*x\n", - "Iteration 3900\n", - "y = 1.2258099189301868 + 1.737440713843289*x\n", - "Iteration 4000\n", - "y = 1.1923213294612487 + 1.7450313631919752*x\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Iteration 4100\n", - "y = 1.1703668469299928 + 1.7578849874677622*x\n", - "Iteration 4200\n", - "y = 1.1299443751200116 + 1.7596081596761088*x\n", - "Iteration 4300\n", - "y = 1.0857688501669025 + 1.7603755539483317*x\n", - "Iteration 4400\n", - "y = 1.067999887278753 + 1.7764118268018219*x\n", - "Iteration 4500\n", - "y = 1.027562399896915 + 1.781172047294927*x\n", - "Iteration 4600\n", - "y = 1.0135197357347023 + 1.8003402340929056*x\n", - "Iteration 4700\n", - "y = 0.9867339323786954 + 1.8049278823229686*x\n", - "Iteration 4800\n", - "y = 0.9570536816927211 + 1.8130726368629162*x\n", - "Iteration 4900\n", - "y = 0.916086185933129 + 1.8121264546965077*x\n", - "Iteration 5000\n", - "y = 0.8769579109672395 + 1.8178313845022325*x\n", - "Iteration 5100\n", - "y = 0.8375535213298363 + 1.8197866853883014*x\n", - "Iteration 5200\n", - "y = 0.8102089699606075 + 1.830294102584346*x\n", - "Iteration 5300\n", - "y = 0.7689489406316838 + 1.8354861163350307*x\n", - "Iteration 5400\n", - "y = 0.7308673994842835 + 1.8382178533040052*x\n", - "Iteration 5500\n", - "y = 0.6944176540689229 + 1.8482404095568499*x\n", - "Iteration 5600\n", - "y = 0.658045742524041 + 1.8554471817071359*x\n", - "Iteration 5700\n", - "y = 0.62844627375962 + 1.8704283878917267*x\n", - "Iteration 5800\n", - "y = 0.5913435534368703 + 1.8791222975164237*x\n", - "Iteration 5900\n", - "y = 0.557880285523112 + 1.8856786959768919*x\n", - "Iteration 6000\n", - "y = 0.5283773774681808 + 1.8904341686177837*x\n", - "Iteration 6100\n", - "y = 0.49319643189670126 + 1.8944019026229129*x\n", - "Iteration 6200\n", - "y = 0.4609224230313071 + 1.9028915395364874*x\n", - "Iteration 6300\n", - "y = 0.42138690265759426 + 1.9021790027550698*x\n", - "Iteration 6400\n", - "y = 0.3970450482092413 + 1.9153321103241103*x\n", - "Iteration 6500\n", - "y = 0.36572250604762574 + 1.9193010121546064*x\n", - "Iteration 6600\n", - "y = 0.3393945033197329 + 1.9302320567756317*x\n", - "Iteration 6700\n", - "y = 0.3195116556438301 + 1.9387953821760682*x\n", - "Iteration 6800\n", - "y = 0.28123080105581444 + 1.9391273898400867*x\n", - "Iteration 6900\n", - "y = 0.2564038851696523 + 1.949412865769459*x\n", - "Iteration 7000\n", - "y = 0.222593366420209 + 1.9542986067640842*x\n", - "Iteration 7100\n", - "y = 0.19240271968462694 + 1.9606054521401473*x\n", - "Iteration 7200\n", - "y = 0.16086644794298657 + 1.9645176738815031*x\n", - "Iteration 7300\n", - "y = 0.13394368951339167 + 1.972807413889778*x\n", - "Iteration 7400\n", - "y = 0.1033010927351425 + 1.976535976077865*x\n", - "Iteration 7500\n", - "y = 0.07330081308513484 + 1.9867981356223945*x\n", - "Iteration 7600\n", - "y = 0.04591682632321923 + 1.9900516435531488*x\n", - "Iteration 7700\n", - "y = 0.026016025769042016 + 1.9961993630414174*x\n", - "Iteration 7800\n", - "y = 0.005871260836460236 + 1.9987489150441338*x\n", - "Iteration 7900\n", - "y = 0.00018543267521520636 + 1.9999684544122942*x\n", - "Iteration 8000\n", - "y = 0.00016069301093268778 + 2.000403053425948*x\n", - "Iteration 8100\n", - "y = 0.0001495165478503408 + 1.9999055770362804*x\n", - "Iteration 8200\n", - "y = -0.00021203944400251267 + 2.0000418860959344*x\n", - "Iteration 8300\n", - "y = -0.000454197749859367 + 1.999221586600376*x\n", - "Iteration 8400\n", - "y = 0.0009087599563562013 + 2.0008070164067924*x\n", - "Iteration 8500\n", - "y = -0.0003932522065875782 + 1.9993590721304573*x\n", - "Iteration 8600\n", - "y = -0.0011023116431645233 + 1.9999449653143944*x\n", - "Iteration 8700\n", - "y = 7.194341189289567e-06 + 2.000000939780076*x\n", - "Iteration 8800\n", - "y = -0.0001273701353909281 + 2.0000586485237015*x\n", - "Iteration 8900\n", - "y = -2.195422541230059e-05 + 2.0000024542513333*x\n", - "Iteration 9000\n", - "y = -9.505541145328072e-05 + 2.0000455552247303*x\n", - "Iteration 9100\n", - "y = -2.895797595100533e-07 + 1.9999865527702694*x\n", - "Iteration 9200\n", - "y = 0.0009239866572852231 + 2.0009316180949837*x\n", - "Iteration 9300\n", - "y = -0.00015004980985254754 + 2.000061477284803*x\n", - "Iteration 9400\n", - "y = -0.0011018145778223334 + 2.000405348640581*x\n", - "Iteration 9500\n", - "y = 3.882083990660555e-05 + 1.99999128375624*x\n", - "Iteration 9600\n", - "y = 0.002341366721941023 + 2.000177268798233*x\n", - "Iteration 9700\n", - "y = 8.582218044672942e-05 + 1.9999793949381646*x\n", - "Iteration 9800\n", - "y = -7.287261379954102e-05 + 1.9999807707232216*x\n", - "Iteration 9900\n", - "y = 8.230621575420588e-05 + 1.9999560748798169*x\n", - "y = 1.1635285794088352e-05 + 1.9999702674692446*x\n" + "y = -8.298690846122729e-17 + 2.0*x\n", + "y = 9.891579815200104e-17 + 2.0*x\n" ] } ], @@ -1734,6 +466,7 @@ "xs = [1,2,3,4,5,6,7]\n", "ys = [2,4,6,8,10,12,14]\n", "\n", + "\n", "# Gradient Descent\n", "model = Line()\n", "print(\"Gradient Descent: \")\n", @@ -1768,702 +501,46 @@ "model = Line()\n", "print(\"Adadelta\")\n", "adadelta(model, xs, ys)\n", + "print(model)\n", + "\n", + "# Adam\n", + "model = Line()\n", + "print(\"Adam\")\n", + "adam(model, xs, ys)\n", "print(model)" ] }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 29, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "Gradient Descent: \n", + "Adam\n", "Iteration 0\n", - "y = 0.3758312748922319 + 0.9861076428125647*x\n", + "y = 0.7139491962653317 + 0.6462402260655279*x\n", "Iteration 100\n", - "y = 0.7298486719499636 + 2.0543498967332554*x\n", + "y = 1.5012704143511661 + 1.913754894630153*x\n", "Iteration 200\n", - "y = 0.8173296897367237 + 2.036750189498243*x\n", + "y = 1.147275006702479 + 1.9626892285633653*x\n", "Iteration 300\n", - "y = 0.8764824052780473 + 2.0248496595087433*x\n", + "y = 1.0187321764566213 + 1.9959358177402393*x\n", "Iteration 400\n", - "y = 0.9164801538689702 + 2.0168027862204627*x\n", + "y = 1.0018116802276642 + 1.9995771873157036*x\n", "Iteration 500\n", - "y = 0.9435257404950816 + 2.0113616697512993*x\n", + "y = 1.0001550519477989 + 1.9999713016943046*x\n", "Iteration 600\n", - "y = 0.9618133637168664 + 2.0076825079986067*x\n", + "y = 1.0000046235977667 + 1.9999989555915936*x\n", "Iteration 700\n", - "y = 0.9741790471729282 + 2.0051947407767163*x\n", + "y = 0.9999999689046957 + 2.0000000091643306*x\n", "Iteration 800\n", - "y = 0.9825404468737576 + 2.0035125679976087*x\n", + "y = 1.0000000007114835 + 1.999999999822266*x\n", "Iteration 900\n", - "y = 0.9881942390968437 + 2.0023751202356674*x\n", - "y = 0.9919859096488965 + 2.001612299988075*x\n", - "Stochastic Gradient Descent: \n", - "Iteration 0\n", - "y = 0.9153217017005606 + 0.34378094132412806*x\n", - "Iteration 100\n", - "y = 1.1818667130118197 + 1.9585589470138083*x\n", - "Iteration 200\n", - "y = 1.117518306355778 + 1.9755644836754853*x\n", - "Iteration 300\n", - "y = 1.0847443749817014 + 1.9853921780077008*x\n", - "Iteration 400\n", - "y = 1.0553999353012187 + 1.9821880215759884*x\n", - "Iteration 500\n", - "y = 1.0383634461008067 + 1.9944754938756803*x\n", - "Iteration 600\n", - "y = 1.025382430134353 + 1.9954382947377485*x\n", - "Iteration 700\n", - "y = 1.017071735717545 + 1.9960754343538933*x\n", - "Iteration 800\n", - "y = 1.0124707435584295 + 1.9982181968759725*x\n", - "Iteration 900\n", - "y = 1.0083118589831706 + 1.99851274149033*x\n", - "y = 1.0054053368750777 + 1.9992228722028862*x\n", - "SGD + Momentum: \n", - "Iteration 0\n", - "y = 0.20190901079079593 + 1.0518015116766273*x\n", - "Iteration 100\n", - "y = 1.0571865268999219 + 1.9352148905303475*x\n", - "Iteration 200\n", - "y = 1.1786162978971049 + 1.757988907429454*x\n", - "Iteration 300\n", - "y = 1.0788638670750186 + 1.9472585317836537*x\n", - "Iteration 400\n", - "y = 1.006748704347011 + 1.9734339051541958*x\n", - "Iteration 500\n", - "y = 0.9343812881913555 + 2.023858333175996*x\n", - "Iteration 600\n", - "y = 1.0012657312666593 + 2.012962154470421*x\n", - "Iteration 700\n", - "y = 1.0013102950137485 + 1.9999783949545147*x\n", - "Iteration 800\n", - "y = 0.998734351972466 + 2.000339221136913*x\n", - "Iteration 900\n", - "y = 1.0000768929259078 + 1.999978546462978*x\n", - "y = 0.9999897833887086 + 1.9999763351360174*x\n", - "Adagrad\n", - "Iteration 0\n", - "y = 0.46197321190711843 + 0.7846328197159732*x\n", - "Iteration 100\n", - "y = 1.5052377440267988 + 1.7671401283436048*x\n", - "Iteration 200\n", - "y = 1.5594483598257016 + 1.8544038545585*x\n", - "Iteration 300\n", - "y = 1.5416573022476152 + 1.8826674398565306*x\n", - "Iteration 400\n", - "y = 1.5105934430891446 + 1.8894737717553833*x\n", - "Iteration 500\n", - "y = 1.4851077194273175 + 1.9004893472809155*x\n", - "Iteration 600\n", - "y = 1.4529679236833892 + 1.904230324715688*x\n", - "Iteration 700\n", - "y = 1.423047644860893 + 1.9086247163631147*x\n", - "Iteration 800\n", - "y = 1.402257621745607 + 1.9186799788576734*x\n", - "Iteration 900\n", - "y = 1.3763020939271235 + 1.922098963338237*x\n", - "Iteration 1000\n", - "y = 1.350180410295563 + 1.9250267975694428*x\n", - "Iteration 1100\n", - "y = 1.3295156720779202 + 1.9317733869265763*x\n", - "Iteration 1200\n", - "y = 1.3118689737973368 + 1.9365404702053604*x\n", - "Iteration 1300\n", - "y = 1.2899837775106766 + 1.939753831557365*x\n", - "Iteration 1400\n", - "y = 1.2695413102750541 + 1.942386962434873*x\n", - "Iteration 1500\n", - "y = 1.2531919440311503 + 1.946722160682169*x\n", - "Iteration 1600\n", - "y = 1.2380934268103059 + 1.9507246478420963*x\n", - "Iteration 1700\n", - "y = 1.224888221768581 + 1.953009315802062*x\n", - "Iteration 1800\n", - "y = 1.213659518362757 + 1.9571066727315032*x\n", - "Iteration 1900\n", - "y = 1.2013752159564801 + 1.9603783167894842*x\n", - "Iteration 2000\n", - "y = 1.1873258795817543 + 1.9610920817269761*x\n", - "Iteration 2100\n", - "y = 1.1769599553743608 + 1.9636110266334854*x\n", - "Iteration 2200\n", - "y = 1.1658629388445951 + 1.9652361589969969*x\n", - "Iteration 2300\n", - "y = 1.1554438951894321 + 1.9676660909306207*x\n", - "Iteration 2400\n", - "y = 1.1445331859470291 + 1.9692525352686878*x\n", - "Iteration 2500\n", - "y = 1.1348165297115897 + 1.9717949820531606*x\n", - "Iteration 2600\n", - "y = 1.1261048508515679 + 1.9745872018696229*x\n", - "Iteration 2700\n", - "y = 1.1191752856709707 + 1.9758014723282842*x\n", - "Iteration 2800\n", - "y = 1.1125467385150944 + 1.9768074389209953*x\n", - "Iteration 2900\n", - "y = 1.1041050923458837 + 1.977919291075865*x\n", - "Iteration 3000\n", - "y = 1.0985709783531161 + 1.9797003591385502*x\n", - "Iteration 3100\n", - "y = 1.0912979877489628 + 1.9804107529481685*x\n", - "Iteration 3200\n", - "y = 1.0853550561726002 + 1.9823552363158488*x\n", - "Iteration 3300\n", - "y = 1.0814933360886434 + 1.9838710061177682*x\n", - "Iteration 3400\n", - "y = 1.0766163050856172 + 1.9844690602690538*x\n", - "Iteration 3500\n", - "y = 1.071815989216191 + 1.9852828101337072*x\n", - "Iteration 3600\n", - "y = 1.0673208500377687 + 1.986254780509397*x\n", - "Iteration 3700\n", - "y = 1.0637196169802738 + 1.9868289841930862*x\n", - "Iteration 3800\n", - "y = 1.0595250611587275 + 1.9873273220140755*x\n", - "Iteration 3900\n", - "y = 1.0561681662719131 + 1.988090310151372*x\n", - "Iteration 4000\n", - "y = 1.0538463556883015 + 1.9890449401729267*x\n", - "Iteration 4100\n", - "y = 1.0509363546534622 + 1.9898534645690964*x\n", - "Iteration 4200\n", - "y = 1.0478800337372782 + 1.9902579290975961*x\n", - "Iteration 4300\n", - "y = 1.0446674040332962 + 1.9908348181849524*x\n", - "Iteration 4400\n", - "y = 1.042142936612279 + 1.9914270079054939*x\n", - "Iteration 4500\n", - "y = 1.0391197596535569 + 1.9913132403175375*x\n", - "Iteration 4600\n", - "y = 1.0376170643415348 + 1.992257447184021*x\n", - "Iteration 4700\n", - "y = 1.0350019291975099 + 1.9927613163711613*x\n", - "Iteration 4800\n", - "y = 1.0324034579627575 + 1.9932344569026264*x\n", - "Iteration 4900\n", - "y = 1.0298988839592453 + 1.9935772770832347*x\n", - "Iteration 5000\n", - "y = 1.0277353985761852 + 1.9938274794384503*x\n", - "Iteration 5100\n", - "y = 1.0259721842561857 + 1.9943834837322028*x\n", - "Iteration 5200\n", - "y = 1.0241614336282632 + 1.9949389334014196*x\n", - "Iteration 5300\n", - "y = 1.0226287044325455 + 1.9951708325900213*x\n", - "Iteration 5400\n", - "y = 1.0207977721159034 + 1.995434466269164*x\n", - "Iteration 5500\n", - "y = 1.0198384470166668 + 1.9959134818907638*x\n", - "Iteration 5600\n", - "y = 1.0186259895364618 + 1.9959664856945551*x\n", - "Iteration 5700\n", - "y = 1.017459644197282 + 1.9962759466631248*x\n", - "Iteration 5800\n", - "y = 1.0165381607180675 + 1.9964814679055072*x\n", - "Iteration 5900\n", - "y = 1.0157824967831135 + 1.9968415272488336*x\n", - "Iteration 6000\n", - "y = 1.0146462651059427 + 1.9969600537779528*x\n", - "Iteration 6100\n", - "y = 1.013577687106957 + 1.99701209404658*x\n", - "Iteration 6200\n", - "y = 1.0128402167607555 + 1.9972862794166322*x\n", - "Iteration 6300\n", - "y = 1.0120468061442849 + 1.9974653590050073*x\n", - "Iteration 6400\n", - "y = 1.0113206958064636 + 1.9977453141225925*x\n", - "Iteration 6500\n", - "y = 1.010530159402851 + 1.9977613337659024*x\n", - "Iteration 6600\n", - "y = 1.009980138597552 + 1.9980332618281365*x\n", - "Iteration 6700\n", - "y = 1.0093487196824678 + 1.9981551053299673*x\n", - "Iteration 6800\n", - "y = 1.0087331396475556 + 1.998201166952451*x\n", - "Iteration 6900\n", - "y = 1.0080090839353189 + 1.998197389772428*x\n", - "Iteration 7000\n", - "y = 1.0074757348954773 + 1.9983154896568611*x\n", - "Iteration 7100\n", - "y = 1.007236458180851 + 1.9985444842510762*x\n", - "Iteration 7200\n", - "y = 1.0067529319418242 + 1.9986379407621238*x\n", - "Iteration 7300\n", - "y = 1.00634190190281 + 1.9987309387790442*x\n", - "Iteration 7400\n", - "y = 1.0059226313323375 + 1.9987311909429517*x\n", - "Iteration 7500\n", - "y = 1.005524467191352 + 1.9987729293651593*x\n", - "Iteration 7600\n", - "y = 1.005274208836092 + 1.9989066343627426*x\n", - "Iteration 7700\n", - "y = 1.0048318638028684 + 1.998906325926456*x\n", - "Iteration 7800\n", - "y = 1.0045302861661434 + 1.9989896151630528*x\n", - "Iteration 7900\n", - "y = 1.004369639050609 + 1.9991210307541332*x\n", - "Iteration 8000\n", - "y = 1.004072150462645 + 1.9991550130190237*x\n", - "Iteration 8100\n", - "y = 1.0038815482480965 + 1.9992146571494605*x\n", - "Iteration 8200\n", - "y = 1.0036053090270298 + 1.999266580744465*x\n", - "Iteration 8300\n", - "y = 1.0033362397114984 + 1.9992921528020966*x\n", - "Iteration 8400\n", - "y = 1.003138139750984 + 1.999355461201285*x\n", - "Iteration 8500\n", - "y = 1.0028797750786755 + 1.9993582896792537*x\n", - "Iteration 8600\n", - "y = 1.0027090165063688 + 1.9994153894934181*x\n", - "Iteration 8700\n", - "y = 1.0025466578018487 + 1.9994710623624246*x\n", - "Iteration 8800\n", - "y = 1.0024144151030312 + 1.9995004213878222*x\n", - "Iteration 8900\n", - "y = 1.0022599591672063 + 1.999535323343836*x\n", - "Iteration 9000\n", - "y = 1.0020805679749356 + 1.9995521623130912*x\n", - "Iteration 9100\n", - "y = 1.0019722538695515 + 1.9995814312463804*x\n", - "Iteration 9200\n", - "y = 1.0018692579468356 + 1.999610295930305*x\n", - "Iteration 9300\n", - "y = 1.001778330673022 + 1.9996462470664884*x\n", - "Iteration 9400\n", - "y = 1.00166834588169 + 1.9996589842747674*x\n", - "Iteration 9500\n", - "y = 1.0015628034035202 + 1.9996712678406139*x\n", - "Iteration 9600\n", - "y = 1.0014988307870971 + 1.9997063145149494*x\n", - "Iteration 9700\n", - "y = 1.0013759781959388 + 1.9997020889180313*x\n", - "Iteration 9800\n", - "y = 1.0013072772407925 + 1.9997292563041686*x\n", - "Iteration 9900\n", - "y = 1.001213260512901 + 1.9997442780684724*x\n", - "y = 1.001153467040044 + 1.9997696502601017*x\n", - "RMSprop\n", - "Iteration 0\n", - "y = 0.2667985444668855 + 0.14627453827716944*x\n", - "Iteration 100\n", - "y = 1.2115532208964666 + 0.9810284459215135*x\n", - "Iteration 200\n", - "y = 1.948568290850704 + 1.6564224481614276*x\n", - "Iteration 300\n", - "y = 1.8368235793727399 + 1.8535190890878868*x\n", - "Iteration 400\n", - "y = 1.5025089794518447 + 1.9076224035453175*x\n", - "Iteration 500\n", - "y = 1.2253246112433784 + 1.9626249064778567*x\n", - "Iteration 600\n", - "y = 1.011998317935152 + 1.9852302459313313*x\n", - "Iteration 700\n", - "y = 1.003770302710899 + 2.0071998964838667*x\n", - "Iteration 800\n", - "y = 0.997336318684003 + 1.9987923138667592*x\n", - "Iteration 900\n", - "y = 1.0258796734055562 + 2.003795708893619*x\n", - "Iteration 1000\n", - "y = 0.9971908625915417 + 1.9947659208257067*x\n", - "Iteration 1100\n", - "y = 0.9845550178577193 + 1.989552391180007*x\n", - "Iteration 1200\n", - "y = 1.001041297605822 + 1.9998373684011863*x\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Iteration 1300\n", - "y = 1.013456608574291 + 2.003760445243725*x\n", - "Iteration 1400\n", - "y = 1.000604429686155 + 1.999814973888711*x\n", - "Iteration 1500\n", - "y = 0.9983353003157828 + 2.002377400504234*x\n", - "Iteration 1600\n", - "y = 1.0079070299852295 + 1.99911172255804*x\n", - "Iteration 1700\n", - "y = 1.0057974653348982 + 2.0092064057307333*x\n", - "Iteration 1800\n", - "y = 1.0012182835397407 + 1.9912445656314368*x\n", - "Iteration 1900\n", - "y = 0.9935730533373829 + 2.004907314371268*x\n", - "Iteration 2000\n", - "y = 1.0068544594217217 + 1.9945796914142868*x\n", - "Iteration 2100\n", - "y = 1.0034692386766948 + 1.999877157270015*x\n", - "Iteration 2200\n", - "y = 1.0238524454768172 + 2.003215919656625*x\n", - "Iteration 2300\n", - "y = 1.0006305118741923 + 2.000139723628984*x\n", - "Iteration 2400\n", - "y = 1.0133749706891342 + 2.0028699564784285*x\n", - "Iteration 2500\n", - "y = 1.0065502964187343 + 1.998588577290943*x\n", - "Iteration 2600\n", - "y = 1.0001875380771728 + 2.0005311718749885*x\n", - "Iteration 2700\n", - "y = 0.9983088913058962 + 1.9998294101697964*x\n", - "Iteration 2800\n", - "y = 1.0024019194786755 + 1.9994309554909746*x\n", - "Iteration 2900\n", - "y = 0.9868403239775956 + 1.9946825921626288*x\n", - "Iteration 3000\n", - "y = 1.0030900120178081 + 1.9985792711020078*x\n", - "Iteration 3100\n", - "y = 0.9921036772906625 + 2.003613167365636*x\n", - "Iteration 3200\n", - "y = 1.0114334161220442 + 1.997186022356892*x\n", - "Iteration 3300\n", - "y = 0.997169998276046 + 2.0014453820513074*x\n", - "Iteration 3400\n", - "y = 1.0004192610414948 + 2.0004896619029324*x\n", - "Iteration 3500\n", - "y = 1.0069225485793913 + 1.9981646076199664*x\n", - "Iteration 3600\n", - "y = 1.0003646657694754 + 1.9999424641145376*x\n", - "Iteration 3700\n", - "y = 0.9973778449224623 + 2.00150150671871*x\n", - "Iteration 3800\n", - "y = 0.9963331851503051 + 2.0011311589314444*x\n", - "Iteration 3900\n", - "y = 0.997180100898796 + 2.0111898551929683*x\n", - "Iteration 4000\n", - "y = 0.9997489823122053 + 2.000460261251615*x\n", - "Iteration 4100\n", - "y = 1.0098897821018864 + 2.013397253252094*x\n", - "Iteration 4200\n", - "y = 0.998189681260244 + 2.002325169564847*x\n", - "Iteration 4300\n", - "y = 0.9997158213216039 + 2.0000419397926663*x\n", - "Iteration 4400\n", - "y = 0.9999264601020886 + 1.999984411040579*x\n", - "Iteration 4500\n", - "y = 0.9930245443315324 + 1.9999684215132116*x\n", - "Iteration 4600\n", - "y = 1.0103792628076949 + 1.995707680265489*x\n", - "Iteration 4700\n", - "y = 1.0177679751104263 + 1.9958367366780794*x\n", - "Iteration 4800\n", - "y = 1.0024546657546582 + 1.9984021055348526*x\n", - "Iteration 4900\n", - "y = 0.9999505076028606 + 1.9999702391928786*x\n", - "Iteration 5000\n", - "y = 1.0119231234603747 + 1.9967936515541906*x\n", - "Iteration 5100\n", - "y = 0.9992593926571393 + 1.9983171401496083*x\n", - "Iteration 5200\n", - "y = 0.9984347294422117 + 2.000172526899655*x\n", - "Iteration 5300\n", - "y = 1.0073826597356943 + 1.9964077132359763*x\n", - "Iteration 5400\n", - "y = 1.0000540582343205 + 1.9999000727002263*x\n", - "Iteration 5500\n", - "y = 1.0055663491683782 + 2.0022730481309106*x\n", - "Iteration 5600\n", - "y = 0.9950170334621676 + 2.0001645993220607*x\n", - "Iteration 5700\n", - "y = 1.0036527901131895 + 1.9988336894818006*x\n", - "Iteration 5800\n", - "y = 0.9985714893044032 + 2.0001467463727938*x\n", - "Iteration 5900\n", - "y = 1.0013200373425009 + 1.9991831198142127*x\n", - "Iteration 6000\n", - "y = 1.0054128846538637 + 2.000541529261067*x\n", - "Iteration 6100\n", - "y = 1.0009909566628767 + 2.002614851443882*x\n", - "Iteration 6200\n", - "y = 0.9900053405079476 + 1.9910753142536055*x\n", - "Iteration 6300\n", - "y = 0.9977206240932082 + 2.001930466916753*x\n", - "Iteration 6400\n", - "y = 1.0110507435883092 + 1.997373193756971*x\n", - "Iteration 6500\n", - "y = 1.0004784142063332 + 1.9998495352680128*x\n", - "Iteration 6600\n", - "y = 0.9950055148048511 + 2.0027661940509174*x\n", - "Iteration 6700\n", - "y = 0.9990348821340557 + 2.0017903001506396*x\n", - "Iteration 6800\n", - "y = 0.9985977511807862 + 2.0005930882456235*x\n", - "Iteration 6900\n", - "y = 0.992591142879565 + 1.9988462193820276*x\n", - "Iteration 7000\n", - "y = 1.0057750271034125 + 1.9982698702925792*x\n", - "Iteration 7100\n", - "y = 1.0001101698856099 + 1.99992237241953*x\n", - "Iteration 7200\n", - "y = 1.001907843246789 + 1.9992258195002546*x\n", - "Iteration 7300\n", - "y = 1.0019865408456285 + 1.9996947634744588*x\n", - "Iteration 7400\n", - "y = 1.0039756457669466 + 1.9987425683836786*x\n", - "Iteration 7500\n", - "y = 0.9832521686092188 + 1.9904064014067577*x\n", - "Iteration 7600\n", - "y = 0.9854438984940378 + 2.0068362168412*x\n", - "Iteration 7700\n", - "y = 1.0006281063701061 + 2.00026789481007*x\n", - "Iteration 7800\n", - "y = 0.9904171458255486 + 2.01075865101255*x\n", - "Iteration 7900\n", - "y = 1.0089995475776878 + 2.005088875075277*x\n", - "Iteration 8000\n", - "y = 0.9986123637246109 + 1.9959182677685408*x\n", - "Iteration 8100\n", - "y = 0.998800533522602 + 1.9999651268416345*x\n", - "Iteration 8200\n", - "y = 0.9950266503775341 + 1.9919321854615517*x\n", - "Iteration 8300\n", - "y = 1.0076417814567011 + 2.0013145367066545*x\n", - "Iteration 8400\n", - "y = 0.9898596426589181 + 1.9978406035918554*x\n", - "Iteration 8500\n", - "y = 0.9993658274580897 + 1.9998453804861112*x\n", - "Iteration 8600\n", - "y = 1.0035550543220355 + 1.9924520403864987*x\n", - "Iteration 8700\n", - "y = 1.000910217775335 + 1.9968570661399963*x\n", - "Iteration 8800\n", - "y = 0.9975413751042991 + 1.9979178991233086*x\n", - "Iteration 8900\n", - "y = 1.0020403374995308 + 2.0005565096616906*x\n", - "Iteration 9000\n", - "y = 0.9952515856011982 + 2.000909524992757*x\n", - "Iteration 9100\n", - "y = 1.0024248756022656 + 2.001287194247487*x\n", - "Iteration 9200\n", - "y = 0.9956725564626692 + 1.992957143424879*x\n", - "Iteration 9300\n", - "y = 0.9898449396957912 + 1.991548689249821*x\n", - "Iteration 9400\n", - "y = 1.0105847466905136 + 2.0062136995392454*x\n", - "Iteration 9500\n", - "y = 1.0003675314917755 + 1.999818298763121*x\n", - "Iteration 9600\n", - "y = 0.9993947698165087 + 2.000527568308001*x\n", - "Iteration 9700\n", - "y = 1.0043636942514191 + 2.0004685049780635*x\n", - "Iteration 9800\n", - "y = 1.0002550653260058 + 1.9898251839350825*x\n", - "Iteration 9900\n", - "y = 1.0011719094905989 + 2.003423733196834*x\n", - "y = 1.0124897484094668 + 2.0017487885568404*x\n", - "Adadelta\n", - "Iteration 0\n", - "y = 0.13852278811002927 + 0.2760918630105501*x\n", - "Iteration 100\n", - "y = 0.23709430376190588 + 0.34631981806244827*x\n", - "Iteration 200\n", - "y = 0.3382766015915197 + 0.4078241547997849*x\n", - "Iteration 300\n", - "y = 0.4456425899856243 + 0.4715469179023652*x\n", - "Iteration 400\n", - "y = 0.5507554777731897 + 0.5299815711990752*x\n", - "Iteration 500\n", - "y = 0.6604060354233109 + 0.5936887580489513*x\n", - "Iteration 600\n", - "y = 0.7658127773000256 + 0.6548397863290296*x\n", - "Iteration 700\n", - "y = 0.8790249925589889 + 0.7229807899928679*x\n", - "Iteration 800\n", - "y = 0.985598628041472 + 0.7872482194707252*x\n", - "Iteration 900\n", - "y = 1.0930182840173865 + 0.8536349671999336*x\n", - "Iteration 1000\n", - "y = 1.1969362602192304 + 0.916370022330082*x\n", - "Iteration 1100\n", - "y = 1.300907866880381 + 0.9791795638255658*x\n", - "Iteration 1200\n", - "y = 1.4128124622512588 + 1.0563415712570565*x\n", - "Iteration 1300\n", - "y = 1.511035301894526 + 1.1202526158337451*x\n", - "Iteration 1400\n", - "y = 1.6047610187132453 + 1.1834951267136278*x\n", - "Iteration 1500\n", - "y = 1.694100270571414 + 1.2452090126752493*x\n", - "Iteration 1600\n", - "y = 1.772296066298704 + 1.3000809447894628*x\n", - "Iteration 1700\n", - "y = 1.8544657349336091 + 1.3602450300705748*x\n", - "Iteration 1800\n", - "y = 1.925221058153202 + 1.4122301299225144*x\n", - "Iteration 1900\n", - "y = 1.9973578142831157 + 1.4688979300752347*x\n", - "Iteration 2000\n", - "y = 2.055315937504795 + 1.5163701741654159*x\n", - "Iteration 2100\n", - "y = 2.104435751489546 + 1.5592633201736477*x\n", - "Iteration 2200\n", - "y = 2.150004519034679 + 1.6005083932705513*x\n", - "Iteration 2300\n", - "y = 2.191345617572304 + 1.641860092418946*x\n", - "Iteration 2400\n", - "y = 2.2000140693165156 + 1.6651123777406638*x\n", - "Iteration 2500\n", - "y = 2.212852426717508 + 1.7019445254743424*x\n", - "Iteration 2600\n", - "y = 2.220756731844795 + 1.7291327778900998*x\n", - "Iteration 2700\n", - "y = 2.1914010603949956 + 1.7366634099797487*x\n", - "Iteration 2800\n", - "y = 2.1485446165286826 + 1.745684980643926*x\n", - "Iteration 2900\n", - "y = 2.1177830831074154 + 1.7536142259394236*x\n", - "Iteration 3000\n", - "y = 2.095767993482749 + 1.7692036526792663*x\n", - "Iteration 3100\n", - "y = 2.0607108758415573 + 1.7739185831797142*x\n", - "Iteration 3200\n", - "y = 2.015360977384807 + 1.7733145466514244*x\n", - "Iteration 3300\n", - "y = 1.9809303030955872 + 1.780371927913142*x\n", - "Iteration 3400\n", - "y = 1.9549943601625 + 1.7937756665805673*x\n", - "Iteration 3500\n", - "y = 1.929152992261465 + 1.8042404369166094*x\n", - "Iteration 3600\n", - "y = 1.9027022717282982 + 1.8150180575106412*x\n", - "Iteration 3700\n", - "y = 1.8693077756126615 + 1.8239299114862848*x\n", - "Iteration 3800\n", - "y = 1.8353491956709633 + 1.8282757533178857*x\n", - "Iteration 3900\n", - "y = 1.7974042827648986 + 1.8292944652324068*x\n", - "Iteration 4000\n", - "y = 1.771043272812957 + 1.837241461606047*x\n", - "Iteration 4100\n", - "y = 1.7270369637800114 + 1.838643915542796*x\n", - "Iteration 4200\n", - "y = 1.6945959819093364 + 1.8512953259505642*x\n", - "Iteration 4300\n", - "y = 1.6623256987085697 + 1.8577234812883885*x\n", - "Iteration 4400\n", - "y = 1.620458527280719 + 1.8602153091423745*x\n", - "Iteration 4500\n", - "y = 1.6025197956638202 + 1.874905680301119*x\n", - "Iteration 4600\n", - "y = 1.562508268223139 + 1.8781003795485187*x\n", - "Iteration 4700\n", - "y = 1.5312585477099032 + 1.8881281641664038*x\n", - "Iteration 4800\n", - "y = 1.4977930621472226 + 1.8941786753228507*x\n", - "Iteration 4900\n", - "y = 1.472939024184481 + 1.902614954107458*x\n", - "Iteration 5000\n", - "y = 1.4473992392324597 + 1.9133370040377111*x\n", - "Iteration 5100\n", - "y = 1.410207084642531 + 1.9141692145727243*x\n", - "Iteration 5200\n", - "y = 1.3766218939553585 + 1.923623165772305*x\n", - "Iteration 5300\n", - "y = 1.337555566364829 + 1.9227406475214857*x\n", - "Iteration 5400\n", - "y = 1.3059807558638492 + 1.9344761141336677*x\n", - "Iteration 5500\n", - "y = 1.2801501647894749 + 1.942726939809063*x\n", - "Iteration 5600\n", - "y = 1.242451356774849 + 1.9512774574188683*x\n", - "Iteration 5700\n", - "y = 1.2135224875885129 + 1.954869587429633*x\n", - "Iteration 5800\n", - "y = 1.1809675461505829 + 1.9613100157350327*x\n", - "Iteration 5900\n", - "y = 1.1549123213803207 + 1.9705225463758274*x\n", - "Iteration 6000\n", - "y = 1.1192159930465027 + 1.9745865463399248*x\n", - "Iteration 6100\n", - "y = 1.088036234570565 + 1.9814411824241804*x\n", - "Iteration 6200\n", - "y = 1.0579744102958062 + 1.9836449078029614*x\n", - "Iteration 6300\n", - "y = 1.0389082061499664 + 1.994222613824601*x\n", - "Iteration 6400\n", - "y = 1.0173479330465949 + 1.9969082396141709*x\n", - "Iteration 6500\n", - "y = 1.002973128722628 + 1.9994175198944446*x\n", - "Iteration 6600\n", - "y = 1.0008338332739093 + 2.000457695989852*x\n", - "Iteration 6700\n", - "y = 1.000967131072133 + 2.0022827525250406*x\n", - "Iteration 6800\n", - "y = 0.9996891696267542 + 2.0001708023976947*x\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Iteration 6900\n", - "y = 1.0000222206592522 + 1.9999761252125086*x\n", - "Iteration 7000\n", - "y = 0.999549383094565 + 1.99884719313683*x\n", - "Iteration 7100\n", - "y = 0.9995616976239795 + 2.0000340912511376*x\n", - "Iteration 7200\n", - "y = 1.000060767627944 + 1.9999566175420058*x\n", - "Iteration 7300\n", - "y = 1.0004173904765594 + 2.000108873603106*x\n", - "Iteration 7400\n", - "y = 0.9998062765670366 + 1.9999941183689391*x\n", - "Iteration 7500\n", - "y = 0.9996904393857947 + 2.000100981808597*x\n", - "Iteration 7600\n", - "y = 1.0001020411022337 + 2.0001841864843892*x\n", - "Iteration 7700\n", - "y = 0.9994753953994944 + 1.9984761540171951*x\n", - "Iteration 7800\n", - "y = 0.9990845626216417 + 1.999888433343906*x\n", - "Iteration 7900\n", - "y = 0.9997245929413028 + 1.999346802722747*x\n", - "Iteration 8000\n", - "y = 1.0002529886696372 + 1.999916494460583*x\n", - "Iteration 8100\n", - "y = 1.0004261759731738 + 2.000501330332923*x\n", - "Iteration 8200\n", - "y = 0.9998512618662146 + 2.0000518470701882*x\n", - "Iteration 8300\n", - "y = 1.0001164682137489 + 1.999937125270592*x\n", - "Iteration 8400\n", - "y = 0.9997953649417827 + 2.0001169420946123*x\n", - "Iteration 8500\n", - "y = 0.9989562790788339 + 2.000443347949381*x\n", - "Iteration 8600\n", - "y = 0.9998289580719268 + 1.9999193142811202*x\n", - "Iteration 8700\n", - "y = 0.9997723337534584 + 2.0004276409997734*x\n", - "Iteration 8800\n", - "y = 0.9999566891131716 + 2.000028580238477*x\n", - "Iteration 8900\n", - "y = 0.9999039552758878 + 2.000138283779481*x\n", - "Iteration 9000\n", - "y = 1.001337397718981 + 2.0005328368360664*x\n", - "Iteration 9100\n", - "y = 0.9998125643006905 + 2.000023110041056*x\n", - "Iteration 9200\n", - "y = 1.0009880417907302 + 1.9997734084238616*x\n", - "Iteration 9300\n", - "y = 0.9994390010796593 + 1.9992918875442238*x\n", - "Iteration 9400\n", - "y = 0.9999927701870963 + 1.9999874631122934*x\n", - "Iteration 9500\n", - "y = 1.0001266272918434 + 2.0000361582922954*x\n", - "Iteration 9600\n", - "y = 1.000508740102382 + 2.000074669253776*x\n", - "Iteration 9700\n", - "y = 1.0000770786055533 + 1.9999613691679703*x\n", - "Iteration 9800\n", - "y = 0.9996475159188182 + 2.0001551115681324*x\n", - "Iteration 9900\n", - "y = 1.0008093321706195 + 2.0001611261818284*x\n", - "y = 0.998897505961075 + 1.9995458788524836*x\n" + "y = 1.000000000010868 + 2.0000000000003593*x\n", + "y = 0.9999999999999609 + 2.000000000000028*x\n" ] } ], @@ -2506,6 +583,12 @@ "model = Line()\n", "print(\"Adadelta\")\n", "adadelta(model, xs, ys)\n", + "print(model)\n", + "\n", + "# Adam\n", + "model = Line()\n", + "print(\"Adam\")\n", + "adam(model, xs, ys)\n", "print(model)" ] }, @@ -2537,5 +620,5 @@ } }, "nbformat": 4, - "nbformat_minor": 2 + "nbformat_minor": 4 } diff --git a/Deep Learning from Scratch in Python/.ipynb_checkpoints/README-checkpoint.md b/Deep Learning from Scratch in Python/.ipynb_checkpoints/README-checkpoint.md new file mode 100644 index 0000000..0c966e3 --- /dev/null +++ b/Deep Learning from Scratch in Python/.ipynb_checkpoints/README-checkpoint.md @@ -0,0 +1,8 @@ +# Deep Learning from Scratch in Python +In this project we take a deep dive into deep learning by coding from scratch all the building blocks of deep neural nets! +I am sporadically coding more components in, so far here are what we managed to build: +- Batch Gradient Descent +- Stochastic Gradient Descent +- Mini-Batch Gradient Descent +- Perceptron +- Multi-layer-perceptron (with forward and backward pass) diff --git a/Deep Learning from Scratch in Python/Gradient Descent Optimization Algorithms.ipynb b/Deep Learning from Scratch in Python/Gradient Descent Optimization Algorithms.ipynb index 0e7600b..d8dc501 100644 --- a/Deep Learning from Scratch in Python/Gradient Descent Optimization Algorithms.ipynb +++ b/Deep Learning from Scratch in Python/Gradient Descent Optimization Algorithms.ipynb @@ -10,7 +10,8 @@ "- [Stochastic Gradient Descent + Momentum](https://youtu.be/7EuiXb6hFAM)\n", "- [Adagrad](https://youtu.be/EGt-UOIIdDk)\n", "- [RMSprop](https://youtu.be/nLCuzsQaAKE)\n", - "- AdaDelta\n", + "- [AdaDelta](https://youtu.be/6gvh0IySNMs)\n", + "- Adam\n", "\n", "## Tests\n", "In order to demonstrate the algorithms capabilities to optimize a function we used these simple test setup:\n", @@ -19,7 +20,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 26, "metadata": {}, "outputs": [], "source": [ @@ -292,702 +293,87 @@ " \n", " if i % 100 == 0:\n", " print(f\"Iteration {i}\")\n", - " print(model)" + " print(model)\n", + " \n", + "\n", + "def adam(model, xs, ys, learning_rate = 0.1, b1 = 0.9, b2 = 0.999, epsilon = 0.00000001, max_iteration = 1000):\n", + " \"\"\"\n", + " Adam: This is the adam optimizer that build upong adadelta and RMSProp\n", + " model: The model we want to optimize the parameter on\n", + " xs: the feature of my dataset\n", + " ys: the continous value (target)\n", + " learning_rate: the amount of learning we want to happen at each time step (default is 0.1 and will be updated by the optimizer)\n", + " b1: this is the first decaying average with proposed default value of 0.9 (deep learning purposes)\n", + " b2: this is the second decaying average with proposed default value of 0.999 (deep learning purposes)\n", + " epsilon: a variable for numerical stability during the division\n", + " max_iteration: the number of sgd round we want to do before stopping the optimization\n", + " \"\"\"\n", + " \n", + " \n", + " # Variable Initialization\n", + " m = [0, 0] # two m for each parameter\n", + " v = [0, 0] # two v for each parameter\n", + " g = [0, 0] # two gradient\n", + " t = 1 # time steps\n", + " \n", + " for i in range(max_iteration):\n", + " # Calculate the gradients \n", + " x, y = stochastic_sample(xs, ys)\n", + " g[0] = model.dx_w0(x, y)\n", + " g[1] = model.dx_w1(x, y)\n", + "\n", + " # Update the m and v parameter\n", + " m = [b1*m_i + (1 - b1)*g_i for m_i, g_i in zip(m, g)]\n", + " v = [b2*v_i + (1 - b2)*(g_i**2) for v_i, g_i in zip(v, g)]\n", + "\n", + " # Bias correction for m and v\n", + " m_cor = [m_i / (1 - (b1**t)) for m_i in m]\n", + " v_cor = [v_i / (1 - (b2**t)) for v_i in v]\n", + "\n", + " # Update the parameter\n", + " model.w0 = model.w0 - (learning_rate / (np.sqrt(v_cor[0]) + epsilon))*m_cor[0]\n", + " model.w1 = model.w1 - (learning_rate / (np.sqrt(v_cor[1]) + epsilon))*m_cor[1]\n", + "\n", + " t = t + 1\n", + " \n", + " if i % 100 == 0:\n", + " print(f\"Iteration {i}\")\n", + " print(model)\n", + " \n", + " " ] }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 27, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "Gradient Descent: \n", + "Adam\n", "Iteration 0\n", - "y = 0.15080809254765393 + 0.6790908370317389*x\n", + "y = 0.8575880576520295 + 0.8405747654640998*x\n", "Iteration 100\n", - "y = 0.14070772861001737 + 0.9716919751062303*x\n", + "y = -0.001590081073315357 + 0.9984342651063493*x\n", "Iteration 200\n", - "y = 0.09514343174678791 + 0.980858744143106*x\n", + "y = 2.70122392056152e-05 + 1.0001344335031852*x\n", "Iteration 300\n", - "y = 0.06433386917675844 + 0.9870571091711977*x\n", + "y = -1.4876611515003004e-06 + 1.0000033575642253*x\n", "Iteration 400\n", - "y = 0.04350112926625649 + 0.9912483055313238*x\n", + "y = 1.030624925582267e-05 + 0.9999970769162957*x\n", "Iteration 500\n", - "y = 0.029414494599109095 + 0.9940822991489187*x\n", + "y = 2.567876097403007e-06 + 0.9999995341571982*x\n", "Iteration 600\n", - "y = 0.01988942602444484 + 0.9959985825044247*x\n", + "y = 3.001062476619204e-08 + 0.9999999996624379*x\n", "Iteration 700\n", - "y = 0.013448786830212731 + 0.9972943306231896*x\n", + "y = 3.5509149827671966e-09 + 0.9999999957864142*x\n", "Iteration 800\n", - "y = 0.009093770075728168 + 0.9981704866376216*x\n", + "y = 2.013987394989773e-08 + 0.9999999952565595*x\n", "Iteration 900\n", - "y = 0.006149004756654367 + 0.9987629238177406*x\n", - "y = 0.00417412143896754 + 0.9991602370760215*x\n", - "Stochastic Gradient Descent: \n", - "Iteration 0\n", - "y = 0.15993686766783655 + 0.1833518650965968*x\n", - "Iteration 100\n", - "y = 0.21828157996824554 + 0.9594516741177225*x\n", - "Iteration 200\n", - "y = 0.1451844200607905 + 0.9741557959554868*x\n", - "Iteration 300\n", - "y = 0.1023440101887112 + 0.9800172156010936*x\n", - "Iteration 400\n", - "y = 0.07450363961602721 + 0.9893311113895694*x\n", - "Iteration 500\n", - "y = 0.046350587935699336 + 0.9881459700038466*x\n", - "Iteration 600\n", - "y = 0.030476212073656663 + 0.9947271999445939*x\n", - "Iteration 700\n", - "y = 0.021391053052246013 + 0.9959265131392894*x\n", - "Iteration 800\n", - "y = 0.014571200279042585 + 0.9967328328159811*x\n", - "Iteration 900\n", - "y = 0.009519640905117088 + 0.997649803991943*x\n", - "y = 0.006666932714079679 + 0.998500306222465*x\n", - "SGD + Momentum: \n", - "Iteration 0\n", - "y = 0.45244291469084846 + 0.9353652979013074*x\n", - "Iteration 100\n", - "y = -0.2248285078215096 + 2.6361555442512574*x\n", - "Iteration 200\n", - "y = -12.384110603326564 + -0.9941162210048384*x\n", - "Iteration 300\n", - "y = -0.09392179857751858 + 1.0244628660411053*x\n", - "Iteration 400\n", - "y = -0.13879271342536312 + 0.9636770795356304*x\n", - "Iteration 500\n", - "y = -0.44087896381090375 + 0.8087858583453921*x\n", - "Iteration 600\n", - "y = 0.011663448270001958 + 1.0025566762581413*x\n", - "Iteration 700\n", - "y = 0.0007482973122936055 + 0.9998250344477012*x\n", - "Iteration 800\n", - "y = -7.183493853088314e-05 + 1.0000396032465118*x\n", - "Iteration 900\n", - "y = -2.1474874379620237e-05 + 1.0001801674854078*x\n", - "y = 4.0014297998085525e-05 + 0.9996092661868773*x\n", - "Adagrad\n", - "Iteration 0\n", - "y = 0.14957133873587877 + 0.349770642623898*x\n", - "Iteration 100\n", - "y = 0.5789747422257289 + 0.883780412031906*x\n", - "Iteration 200\n", - "y = 0.476060907959348 + 0.9021321576916074*x\n", - "Iteration 300\n", - "y = 0.3883109597254469 + 0.9167686675736807*x\n", - "Iteration 400\n", - "y = 0.32213311654349985 + 0.931919861161982*x\n", - "Iteration 500\n", - "y = 0.26703558629401253 + 0.9468055802251013*x\n", - "Iteration 600\n", - "y = 0.22881588002466632 + 0.9522323308080116*x\n", - "Iteration 700\n", - "y = 0.18778860232158315 + 0.9582100069795217*x\n", - "Iteration 800\n", - "y = 0.15554079368406587 + 0.9661025574708333*x\n", - "Iteration 900\n", - "y = 0.13264118785089948 + 0.9731507500316178*x\n", - "Iteration 1000\n", - "y = 0.1070410549333178 + 0.9749049605055626*x\n", - "Iteration 1100\n", - "y = 0.09185365717182185 + 0.9807915717798473*x\n", - "Iteration 1200\n", - "y = 0.08111295608842777 + 0.9850047591169898*x\n", - "Iteration 1300\n", - "y = 0.06211168920968319 + 0.9838182610782438*x\n", - "Iteration 1400\n", - "y = 0.05380151228795558 + 0.9883694454256858*x\n", - "Iteration 1500\n", - "y = 0.04424068877458216 + 0.9903814386731185*x\n", - "Iteration 1600\n", - "y = 0.03729834508521443 + 0.9916615367006102*x\n", - "Iteration 1700\n", - "y = 0.031939592115124336 + 0.9933359574629548*x\n", - "Iteration 1800\n", - "y = 0.02765181756924617 + 0.9947914270963896*x\n", - "Iteration 1900\n", - "y = 0.022584337990350704 + 0.9952052508861745*x\n", - "Iteration 2000\n", - "y = 0.018842544645614277 + 0.9960323037855057*x\n", - "Iteration 2100\n", - "y = 0.015706324717892087 + 0.9964034044711639*x\n", - "Iteration 2200\n", - "y = 0.013376815996403906 + 0.9970421549139848*x\n", - "Iteration 2300\n", - "y = 0.011140802753997708 + 0.9975800313527812*x\n", - "Iteration 2400\n", - "y = 0.009180596246359064 + 0.998101061323726*x\n", - "Iteration 2500\n", - "y = 0.007884923400885832 + 0.9985257724925584*x\n", - "Iteration 2600\n", - "y = 0.0064791120032062 + 0.9985724869754915*x\n", - "Iteration 2700\n", - "y = 0.005665873552865111 + 0.9988468038704624*x\n", - "Iteration 2800\n", - "y = 0.0048065607643679445 + 0.9990486498541105*x\n", - "Iteration 2900\n", - "y = 0.003968266097986841 + 0.9991154584760444*x\n", - "Iteration 3000\n", - "y = 0.0033298137947775286 + 0.999309958328142*x\n", - "Iteration 3100\n", - "y = 0.0029504826077003834 + 0.9994302759246211*x\n", - "Iteration 3200\n", - "y = 0.002380617673238137 + 0.9994742557828102*x\n", - "Iteration 3300\n", - "y = 0.0020912374417970912 + 0.9995991422066534*x\n", - "Iteration 3400\n", - "y = 0.0017218570181314194 + 0.9996661035040911*x\n", - "Iteration 3500\n", - "y = 0.0014294620821387657 + 0.9996922803546607*x\n", - "Iteration 3600\n", - "y = 0.0012405238514359416 + 0.9997455508343457*x\n", - "Iteration 3700\n", - "y = 0.0010073098877846675 + 0.999783563658199*x\n", - "Iteration 3800\n", - "y = 0.0008595170460348628 + 0.9998230970978605*x\n", - "Iteration 3900\n", - "y = 0.0007109894887433046 + 0.9998455123904334*x\n", - "Iteration 4000\n", - "y = 0.0006004078268029727 + 0.9998625899850877*x\n", - "Iteration 4100\n", - "y = 0.0005002665526051534 + 0.9999021008909665*x\n", - "Iteration 4200\n", - "y = 0.00042043879502743817 + 0.9999083394167763*x\n", - "Iteration 4300\n", - "y = 0.00034712046017588467 + 0.9999226341779709*x\n", - "Iteration 4400\n", - "y = 0.0002918159150337169 + 0.9999341373894769*x\n", - "Iteration 4500\n", - "y = 0.00023759176842522305 + 0.9999421976498841*x\n", - "Iteration 4600\n", - "y = 0.0002031163044737048 + 0.9999551395282233*x\n", - "Iteration 4700\n", - "y = 0.00017652745265826287 + 0.9999627827656518*x\n", - "Iteration 4800\n", - "y = 0.00014397038691800682 + 0.9999694413255239*x\n", - "Iteration 4900\n", - "y = 0.00011977885596124304 + 0.9999765829401767*x\n", - "Iteration 5000\n", - "y = 9.742011104497176e-05 + 0.9999780765656271*x\n", - "Iteration 5100\n", - "y = 8.405139675455769e-05 + 0.9999830106577814*x\n", - "Iteration 5200\n", - "y = 7.199360094961494e-05 + 0.9999853763532276*x\n", - "Iteration 5300\n", - "y = 5.9438595609082104e-05 + 0.9999879821035998*x\n", - "Iteration 5400\n", - "y = 4.799553234174064e-05 + 0.9999900691345857*x\n", - "Iteration 5500\n", - "y = 4.0372638139457874e-05 + 0.9999915160547906*x\n", - "Iteration 5600\n", - "y = 3.294290517155539e-05 + 0.9999926620335875*x\n", - "Iteration 5700\n", - "y = 2.8426587896418184e-05 + 0.9999943696968747*x\n", - "Iteration 5800\n", - "y = 2.3897665033254145e-05 + 0.9999952434923006*x\n", - "Iteration 5900\n", - "y = 1.9640055052909126e-05 + 0.9999959414903599*x\n", - "Iteration 6000\n", - "y = 1.684732609374185e-05 + 0.9999966681434076*x\n", - "Iteration 6100\n", - "y = 1.3996448973319881e-05 + 0.9999971896668423*x\n", - "Iteration 6200\n", - "y = 1.2071327881619936e-05 + 0.9999976352142692*x\n", - "Iteration 6300\n", - "y = 1.0143608586031199e-05 + 0.9999979501909246*x\n", - "Iteration 6400\n", - "y = 8.397847594666258e-06 + 0.9999982734836176*x\n", - "Iteration 6500\n", - "y = 7.081087850144272e-06 + 0.9999984771962163*x\n", - "Iteration 6600\n", - "y = 6.0865568463067215e-06 + 0.9999986630504831*x\n", - "Iteration 6700\n", - "y = 5.2112590599668e-06 + 0.9999989126192039*x\n", - "Iteration 6800\n", - "y = 4.425142363347888e-06 + 0.9999991143161019*x\n", - "Iteration 6900\n", - "y = 3.6347556610242305e-06 + 0.999999226262124*x\n", - "Iteration 7000\n", - "y = 3.047110472416464e-06 + 0.9999993667641297*x\n", - "Iteration 7100\n", - "y = 2.508535947738019e-06 + 0.9999994774776229*x\n", - "Iteration 7200\n", - "y = 2.026042884003383e-06 + 0.9999995076761915*x\n", - "Iteration 7300\n", - "y = 1.7169970615109301e-06 + 0.9999996214099223*x\n", - "Iteration 7400\n", - "y = 1.4659379241352074e-06 + 0.9999996855150445*x\n", - "Iteration 7500\n", - "y = 1.267811305769468e-06 + 0.9999997446919678*x\n", - "Iteration 7600\n", - "y = 1.0536289441560765e-06 + 0.9999997798683502*x\n", - "Iteration 7700\n", - "y = 8.727083155544904e-07 + 0.9999998075812689*x\n", - "Iteration 7800\n", - "y = 7.428466938114932e-07 + 0.9999998435690732*x\n", - "Iteration 7900\n", - "y = 6.033858150784952e-07 + 0.9999998667627915*x\n", - "Iteration 8000\n", - "y = 4.905624695607078e-07 + 0.9999998924754667*x\n", - "Iteration 8100\n", - "y = 4.1878487383983215e-07 + 0.9999999055028954*x\n", - "Iteration 8200\n", - "y = 3.4797906980686223e-07 + 0.9999999229429366*x\n", - "Iteration 8300\n", - "y = 2.9187629789054726e-07 + 0.9999999351414686*x\n", - "Iteration 8400\n", - "y = 2.3800753119535694e-07 + 0.9999999428751907*x\n", - "Iteration 8500\n", - "y = 2.1125252960406036e-07 + 0.9999999580874736*x\n", - "Iteration 8600\n", - "y = 1.7275538359612736e-07 + 0.9999999621112141*x\n", - "Iteration 8700\n", - "y = 1.4948029160490067e-07 + 0.9999999704478516*x\n", - "Iteration 8800\n", - "y = 1.2167562136618061e-07 + 0.9999999742622795*x\n", - "Iteration 8900\n", - "y = 1.0085585252142786e-07 + 0.9999999776730842*x\n", - "Iteration 9000\n", - "y = 8.579969756011814e-08 + 0.9999999831918347*x\n", - "Iteration 9100\n", - "y = 7.27897125945934e-08 + 0.9999999850177265*x\n", - "Iteration 9200\n", - "y = 6.060072539637118e-08 + 0.9999999869821679*x\n", - "Iteration 9300\n", - "y = 4.832855949920762e-08 + 0.9999999881311815*x\n", - "Iteration 9400\n", - "y = 4.205603897144143e-08 + 0.9999999913444346*x\n", - "Iteration 9500\n", - "y = 3.578918130017679e-08 + 0.9999999926595672*x\n", - "Iteration 9600\n", - "y = 2.9520297195090112e-08 + 0.9999999938305414*x\n", - "Iteration 9700\n", - "y = 2.4525485672141094e-08 + 0.9999999948040391*x\n", - "Iteration 9800\n", - "y = 2.0603269757320282e-08 + 0.9999999958682226*x\n", - "Iteration 9900\n", - "y = 1.6352124973857764e-08 + 0.9999999961601749*x\n", - "y = 1.3978513232660109e-08 + 0.9999999970810329*x\n", - "RMSprop\n", - "Iteration 0\n", - "y = 0.6827153827276677 + 0.7136147159861074*x\n", - "Iteration 100\n", - "y = 0.4613436576630757 + 0.8911490421103776*x\n", - "Iteration 200\n", - "y = 0.1641445844953397 + 0.9765644548900715*x\n", - "Iteration 300\n", - "y = 0.005416941925482668 + 0.9958119047757997*x\n", - "Iteration 400\n", - "y = -0.008575596423691137 + 1.0009337008824903*x\n", - "Iteration 500\n", - "y = 0.0011265931342927362 + 0.9994590506235699*x\n", - "Iteration 600\n", - "y = -0.019256660349787402 + 1.0029342829038248*x\n", - "Iteration 700\n", - "y = -7.901965098594169e-05 + 0.999896746799356*x\n", - "Iteration 800\n", - "y = 0.0010274411861273501 + 0.9996722279684327*x\n", - "Iteration 900\n", - "y = -0.0002664928805054597 + 1.000216490800304*x\n", - "Iteration 1000\n", - "y = 0.022408710850394694 + 1.0025402500655463*x\n", - "Iteration 1100\n", - "y = -0.0032849047666134844 + 0.9992583541515305*x\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Iteration 1200\n", - "y = -0.0007664471045705575 + 1.0002642184776036*x\n", - "Iteration 1300\n", - "y = -0.021958706880734306 + 1.003498640175805*x\n", - "Iteration 1400\n", - "y = -0.015584512173962473 + 0.9848432560044733*x\n", - "Iteration 1500\n", - "y = -0.00013040805631450714 + 0.9996115058478232*x\n", - "Iteration 1600\n", - "y = 0.002924356459606971 + 1.0017756413448418*x\n", - "Iteration 1700\n", - "y = 0.00028922302118505097 + 0.999895063767121*x\n", - "Iteration 1800\n", - "y = 0.004173454138749314 + 0.9977761099422883*x\n", - "Iteration 1900\n", - "y = 0.008285548513884912 + 0.9972693671576253*x\n", - "Iteration 2000\n", - "y = 0.0037519184753459795 + 0.9991495583156801*x\n", - "Iteration 2100\n", - "y = -0.0009917925918777797 + 1.0006309126324515*x\n", - "Iteration 2200\n", - "y = -0.005797292073705403 + 1.001185732126061*x\n", - "Iteration 2300\n", - "y = -0.0024461824844422613 + 1.0008023430416744*x\n", - "Iteration 2400\n", - "y = -0.0018481187011078282 + 0.996790630504706*x\n", - "Iteration 2500\n", - "y = 0.015076565167656184 + 0.9940245585355336*x\n", - "Iteration 2600\n", - "y = -0.00336552791882152 + 1.000816589542082*x\n", - "Iteration 2700\n", - "y = -0.007711886970819105 + 1.0005968435053925*x\n", - "Iteration 2800\n", - "y = 0.004036458271196503 + 0.9980993641270244*x\n", - "Iteration 2900\n", - "y = 0.00755381916295596 + 0.9913713030311821*x\n", - "Iteration 3000\n", - "y = -0.007462354406979531 + 0.9990929971166894*x\n", - "Iteration 3100\n", - "y = 0.0017013839220933447 + 1.0046121935156789*x\n", - "Iteration 3200\n", - "y = 0.002549610608176222 + 0.9811771184284725*x\n", - "Iteration 3300\n", - "y = -0.0008406823225092473 + 0.9964544787926721*x\n", - "Iteration 3400\n", - "y = 0.00022657235956706143 + 1.001661743718075*x\n", - "Iteration 3500\n", - "y = 0.00927907207267838 + 1.0111509515169403*x\n", - "Iteration 3600\n", - "y = 0.0023024236036593075 + 0.9992039348865193*x\n", - "Iteration 3700\n", - "y = -0.0006597887442904594 + 1.0003505161666206*x\n", - "Iteration 3800\n", - "y = 0.0002869058615782528 + 0.9998364562886809*x\n", - "Iteration 3900\n", - "y = 0.007744411310317621 + 0.9974624502449049*x\n", - "Iteration 4000\n", - "y = 0.002807386271596544 + 0.9966061645050988*x\n", - "Iteration 4100\n", - "y = -0.003392981462673309 + 1.0124873840670952*x\n", - "Iteration 4200\n", - "y = -0.006226717369419612 + 0.9836426768214036*x\n", - "Iteration 4300\n", - "y = -0.002499263973157743 + 1.0024621234963387*x\n", - "Iteration 4400\n", - "y = 0.000786459741684581 + 0.9996068907377066*x\n", - "Iteration 4500\n", - "y = -0.013885836472497332 + 0.98501782205896*x\n", - "Iteration 4600\n", - "y = 0.0027157938203233025 + 0.9991170192133456*x\n", - "Iteration 4700\n", - "y = 0.0007138591579498406 + 1.0012813472435278*x\n", - "Iteration 4800\n", - "y = -0.012771693127577421 + 0.9775943533933739*x\n", - "Iteration 4900\n", - "y = -0.01659186680784981 + 0.9960943678488293*x\n", - "Iteration 5000\n", - "y = 0.00564418704303236 + 0.9974864589134017*x\n", - "Iteration 5100\n", - "y = 0.0002736726164160462 + 1.000533766062686*x\n", - "Iteration 5200\n", - "y = -0.0019941647807032135 + 1.0015601235206242*x\n", - "Iteration 5300\n", - "y = -0.006476602405958722 + 1.0034518749707466*x\n", - "Iteration 5400\n", - "y = 0.0007427644465457552 + 1.002728093096825*x\n", - "Iteration 5500\n", - "y = 0.0046487675269970006 + 1.002810091014998*x\n", - "Iteration 5600\n", - "y = 0.025769758554076257 + 1.0127688270680373*x\n", - "Iteration 5700\n", - "y = -0.0042951255595749405 + 1.0010716326676183*x\n", - "Iteration 5800\n", - "y = -0.008068636860778397 + 0.9877757885071299*x\n", - "Iteration 5900\n", - "y = 0.003450598977099211 + 1.001726671142766*x\n", - "Iteration 6000\n", - "y = -0.004041807231042041 + 1.0022722345787372*x\n", - "Iteration 6100\n", - "y = 0.004267783918492054 + 1.000295233436237*x\n", - "Iteration 6200\n", - "y = -0.0056225140463435986 + 1.0070122459950046*x\n", - "Iteration 6300\n", - "y = -0.003235094708712298 + 1.0004924711288574*x\n", - "Iteration 6400\n", - "y = -0.006780850359893946 + 0.9956229363195707*x\n", - "Iteration 6500\n", - "y = 0.0004787190339910564 + 1.0001032053801957*x\n", - "Iteration 6600\n", - "y = -0.003352041305418763 + 0.9950335084288989*x\n", - "Iteration 6700\n", - "y = -0.004781303872024403 + 0.9980264243399204*x\n", - "Iteration 6800\n", - "y = 0.004826125511710525 + 0.9987438685190084*x\n", - "Iteration 6900\n", - "y = 0.01686124865478895 + 1.0097884533632295*x\n", - "Iteration 7000\n", - "y = 0.005459832417801409 + 1.0047429713936475*x\n", - "Iteration 7100\n", - "y = 0.013865786101604652 + 0.9960259195432627*x\n", - "Iteration 7200\n", - "y = -0.005933092998259135 + 1.0027771611928813*x\n", - "Iteration 7300\n", - "y = 0.00862304332326407 + 0.9954828005217338*x\n", - "Iteration 7400\n", - "y = -0.0016032139468913859 + 1.0008090120392943*x\n", - "Iteration 7500\n", - "y = 0.0030326295588126574 + 0.9966219225192973*x\n", - "Iteration 7600\n", - "y = -0.013317271255791796 + 1.0040204327246824*x\n", - "Iteration 7700\n", - "y = -0.0018852242994510703 + 1.000117321206579*x\n", - "Iteration 7800\n", - "y = -0.0005638059805626334 + 0.9970121486962463*x\n", - "Iteration 7900\n", - "y = -0.00756664152504323 + 1.0004804196155637*x\n", - "Iteration 8000\n", - "y = -0.003008432788919161 + 0.9995411212472487*x\n", - "Iteration 8100\n", - "y = 0.006288770186498472 + 0.9954637861523473*x\n", - "Iteration 8200\n", - "y = -0.010488420713660969 + 1.0008535663825082*x\n", - "Iteration 8300\n", - "y = 0.0004644311563401782 + 1.0006834331070662*x\n", - "Iteration 8400\n", - "y = 0.020443405696192827 + 0.9958956003205478*x\n", - "Iteration 8500\n", - "y = 0.0009955010118736039 + 0.9987797136410242*x\n", - "Iteration 8600\n", - "y = -0.011030965683844665 + 0.9888970571552462*x\n", - "Iteration 8700\n", - "y = 1.3738329377985088e-06 + 1.000003677876151*x\n", - "Iteration 8800\n", - "y = -5.3122243081662327e-05 + 0.9999689735171635*x\n", - "Iteration 8900\n", - "y = 0.014876148409598184 + 1.004505318045117*x\n", - "Iteration 9000\n", - "y = -0.007948523568615803 + 0.9852503297758233*x\n", - "Iteration 9100\n", - "y = 0.0003128111105465842 + 0.9997217700955713*x\n", - "Iteration 9200\n", - "y = 0.005322736855551152 + 0.9971767718064711*x\n", - "Iteration 9300\n", - "y = -0.013617256809249096 + 0.9959545298171486*x\n", - "Iteration 9400\n", - "y = 0.0029412398412501507 + 0.9999989947409157*x\n", - "Iteration 9500\n", - "y = -0.00340750760661034 + 0.9983407221245729*x\n", - "Iteration 9600\n", - "y = -0.0005875504713386392 + 1.0022516904038645*x\n", - "Iteration 9700\n", - "y = -0.003536715734204159 + 1.000769397732234*x\n", - "Iteration 9800\n", - "y = 0.006625135270312074 + 0.9936483257178053*x\n", - "Iteration 9900\n", - "y = 0.0035222747520052902 + 1.005384237268717*x\n", - "y = 0.006041686587868328 + 0.9980403561026004*x\n", - "Adadelta\n", - "Iteration 0\n", - "y = 0.3737801953255442 + 0.6954232880866397*x\n", - "Iteration 100\n", - "y = 0.45446798994451765 + 0.7642810918478647*x\n", - "Iteration 200\n", - "y = 0.503724013503607 + 0.8061108885966569*x\n", - "Iteration 300\n", - "y = 0.5372449606942332 + 0.8427484125351289*x\n", - "Iteration 400\n", - "y = 0.5494500294658365 + 0.8700735464493226*x\n", - "Iteration 500\n", - "y = 0.5247347573025634 + 0.8810887878647324*x\n", - "Iteration 600\n", - "y = 0.504017795661083 + 0.8937987270543221*x\n", - "Iteration 700\n", - "y = 0.4749651708372001 + 0.8971005764339413*x\n", - "Iteration 800\n", - "y = 0.4400397105308388 + 0.9014096500974156*x\n", - "Iteration 900\n", - "y = 0.40824997114309086 + 0.9134682609947273*x\n", - "Iteration 1000\n", - "y = 0.37381393291084897 + 0.9194850353352613*x\n", - "Iteration 1100\n", - "y = 0.3517276407239353 + 0.931088854072381*x\n", - "Iteration 1200\n", - "y = 0.3155689840081007 + 0.9297539531195199*x\n", - "Iteration 1300\n", - "y = 0.28516476451211253 + 0.9410833878329703*x\n", - "Iteration 1400\n", - "y = 0.25151839633081546 + 0.9468630317486699*x\n", - "Iteration 1500\n", - "y = 0.2187631891156385 + 0.9536963275180232*x\n", - "Iteration 1600\n", - "y = 0.18813494915312937 + 0.9599508678970319*x\n", - "Iteration 1700\n", - "y = 0.16346802017358564 + 0.9681810668133642*x\n", - "Iteration 1800\n", - "y = 0.1427335223054219 + 0.9726379550723799*x\n", - "Iteration 1900\n", - "y = 0.11095931576581414 + 0.9756626085219838*x\n", - "Iteration 2000\n", - "y = 0.08467626594832124 + 0.9809827312292054*x\n", - "Iteration 2100\n", - "y = 0.055776093501635086 + 0.986682127065862*x\n", - "Iteration 2200\n", - "y = 0.03128161032349964 + 0.994151344816365*x\n", - "Iteration 2300\n", - "y = 0.011134563694370887 + 0.9980101133207862*x\n", - "Iteration 2400\n", - "y = -0.0002819231467387675 + 0.9994292604031647*x\n", - "Iteration 2500\n", - "y = -0.0007530001080388465 + 0.9996876078971861*x\n", - "Iteration 2600\n", - "y = 1.4299320787029975e-05 + 1.0002415850327648*x\n", - "Iteration 2700\n", - "y = 0.00048437023526542024 + 1.000636329524222*x\n", - "Iteration 2800\n", - "y = 0.00017081810272224221 + 1.0002339219595149*x\n", - "Iteration 2900\n", - "y = -0.00047720186234186763 + 1.0002837940293001*x\n", - "Iteration 3000\n", - "y = -0.00048827011918332745 + 1.0000810463921026*x\n", - "Iteration 3100\n", - "y = -0.0001568847949445286 + 0.9999644370576438*x\n", - "Iteration 3200\n", - "y = -0.0008464061365265306 + 1.000160614161929*x\n", - "Iteration 3300\n", - "y = -0.0012632585693276611 + 1.0008587748768185*x\n", - "Iteration 3400\n", - "y = 8.936672728558989e-05 + 0.9998637431800027*x\n", - "Iteration 3500\n", - "y = 0.00028773253132511624 + 0.999979297608201*x\n", - "Iteration 3600\n", - "y = -0.0005940538060823068 + 0.9997986460772728*x\n", - "Iteration 3700\n", - "y = -0.00017555787997768101 + 0.9992967782058932*x\n", - "Iteration 3800\n", - "y = -0.0010902486276842423 + 0.9993665313074814*x\n", - "Iteration 3900\n", - "y = 0.00011090105568572678 + 0.9999390648187347*x\n", - "Iteration 4000\n", - "y = -0.0007925602555049502 + 1.0003670190881373*x\n", - "Iteration 4100\n", - "y = -9.85030472217507e-05 + 1.0000452185396647*x\n", - "Iteration 4200\n", - "y = -0.001322185011795518 + 1.000252860600806*x\n", - "Iteration 4300\n", - "y = -0.000622778427580351 + 0.9996172454917239*x\n", - "Iteration 4400\n", - "y = 0.0008793034307541049 + 0.9991323399931926*x\n", - "Iteration 4500\n", - "y = 2.2097921067741522e-05 + 0.9999916633445198*x\n", - "Iteration 4600\n", - "y = -0.00013584268124752416 + 1.000083372558365*x\n", - "Iteration 4700\n", - "y = -0.0003792950127671934 + 1.0002199515238068*x\n", - "Iteration 4800\n", - "y = 0.0007407910291384943 + 1.0002752312416903*x\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Iteration 4900\n", - "y = -0.00018293553284564853 + 1.000084353469475*x\n", - "Iteration 5000\n", - "y = -0.00021086202011657087 + 1.000599467500034*x\n", - "Iteration 5100\n", - "y = -0.0002698118770557987 + 1.0002670984772302*x\n", - "Iteration 5200\n", - "y = -0.000174041228809895 + 1.0000873631031082*x\n", - "Iteration 5300\n", - "y = -0.000368303888004153 + 0.9988172336751648*x\n", - "Iteration 5400\n", - "y = 3.23267012324346e-08 + 1.0000015527829704*x\n", - "Iteration 5500\n", - "y = -0.0007458676117796859 + 1.0002497117687457*x\n", - "Iteration 5600\n", - "y = -0.0002750920909796192 + 0.9999100367963439*x\n", - "Iteration 5700\n", - "y = -0.0002603746682872479 + 1.000235272525123*x\n", - "Iteration 5800\n", - "y = 0.0006660635547060155 + 0.9997308283380719*x\n", - "Iteration 5900\n", - "y = 0.0007499650195894795 + 0.9998309531696397*x\n", - "Iteration 6000\n", - "y = 0.00026793080973281725 + 0.9999254737646864*x\n", - "Iteration 6100\n", - "y = -7.108162650266238e-05 + 1.0000467252894638*x\n", - "Iteration 6200\n", - "y = 2.7908004826124246e-05 + 0.9999831010503192*x\n", - "Iteration 6300\n", - "y = 7.319170492363453e-05 + 0.9999735463240744*x\n", - "Iteration 6400\n", - "y = 0.00010198021235678255 + 0.9999497167118081*x\n", - "Iteration 6500\n", - "y = -2.431716475506412e-06 + 1.000008780986074*x\n", - "Iteration 6600\n", - "y = 0.0003694589641208747 + 1.0000175469336479*x\n", - "Iteration 6700\n", - "y = -1.4022966755159485e-06 + 0.9999992007604116*x\n", - "Iteration 6800\n", - "y = -8.633507199438527e-05 + 1.000240477279819*x\n", - "Iteration 6900\n", - "y = 0.00044989195448772416 + 0.9999453497703747*x\n", - "Iteration 7000\n", - "y = -5.879042758639982e-05 + 1.0000300657398191*x\n", - "Iteration 7100\n", - "y = -0.0009824183149325324 + 0.9991641172677947*x\n", - "Iteration 7200\n", - "y = 9.609852626984948e-05 + 0.9999036811155302*x\n", - "Iteration 7300\n", - "y = -0.00019017894587889544 + 0.9999989991196587*x\n", - "Iteration 7400\n", - "y = 5.42016651517267e-05 + 0.9999736784994304*x\n", - "Iteration 7500\n", - "y = -0.00023740571251060384 + 0.9999013240309537*x\n", - "Iteration 7600\n", - "y = -0.00017011932836109704 + 1.000030745217648*x\n", - "Iteration 7700\n", - "y = -0.00035492964049595643 + 0.9996757749060617*x\n", - "Iteration 7800\n", - "y = -0.0004319532086695726 + 0.9999177720542175*x\n", - "Iteration 7900\n", - "y = -4.734275611628327e-08 + 0.9999378842893045*x\n", - "Iteration 8000\n", - "y = -2.209857409748423e-05 + 1.0000121727293267*x\n", - "Iteration 8100\n", - "y = 0.00029399636729362117 + 0.9998609266875158*x\n", - "Iteration 8200\n", - "y = -0.0001310883704963831 + 1.0000817661441546*x\n", - "Iteration 8300\n", - "y = 0.0008041036490104212 + 1.0000502734574366*x\n", - "Iteration 8400\n", - "y = -0.0005327049113160294 + 0.99990390969119*x\n", - "Iteration 8500\n", - "y = -0.00030909670698362434 + 0.9997314940272098*x\n", - "Iteration 8600\n", - "y = -0.0004119597523058274 + 0.9992781835784582*x\n", - "Iteration 8700\n", - "y = -1.5946249925514515e-05 + 1.00002552381485*x\n", - "Iteration 8800\n", - "y = -2.2331167035739685e-05 + 1.0000347524312192*x\n", - "Iteration 8900\n", - "y = -0.00022048165850817618 + 0.9996719966521392*x\n", - "Iteration 9000\n", - "y = -0.0003105653255593682 + 0.9999788024689434*x\n", - "Iteration 9100\n", - "y = 9.668358292685267e-05 + 0.9999706995785885*x\n", - "Iteration 9200\n", - "y = 0.00010238247450770996 + 0.9998004937879871*x\n", - "Iteration 9300\n", - "y = 0.00037115928665625714 + 1.000287762566172*x\n", - "Iteration 9400\n", - "y = -6.118416352593448e-05 + 1.000107454775657*x\n", - "Iteration 9500\n", - "y = -0.0009550721070318215 + 0.9996910774648767*x\n", - "Iteration 9600\n", - "y = 7.165902952687223e-05 + 1.0002360153679417*x\n", - "Iteration 9700\n", - "y = -0.0001307949755324461 + 1.0000442759668344*x\n", - "Iteration 9800\n", - "y = 0.0007364117746714955 + 0.9998109609245667*x\n", - "Iteration 9900\n", - "y = -0.0003427528909499275 + 1.0000402856136628*x\n", - "y = 0.0003127360718812701 + 1.0004689899136678*x\n" + "y = -2.4103486747254733e-08 + 0.9999999984873583*x\n", + "y = 4.7124723585610135e-07 + 0.9999992952944199*x\n" ] } ], @@ -996,6 +382,7 @@ "xs = [1,2,3,4,5,6,7]\n", "ys = [1,2,3,4,5,6,7]\n", "\n", + "\n", "# Gradient Descent\n", "model = Line()\n", "print(\"Gradient Descent: \")\n", @@ -1030,702 +417,47 @@ "model = Line()\n", "print(\"Adadelta\")\n", "adadelta(model, xs, ys)\n", + "print(model)\n", + "\n", + "\n", + "# Adam\n", + "model = Line()\n", + "print(\"Adam\")\n", + "adam(model, xs, ys)\n", "print(model)" ] }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 28, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "Gradient Descent: \n", - "Iteration 0\n", - "y = 0.6965931374130392 + 0.7601122786401455*x\n", - "Iteration 100\n", - "y = 0.617623544918069 + 1.8757445461082207*x\n", - "Iteration 200\n", - "y = 0.4176232831814616 + 1.9159812299345367*x\n", - "Iteration 300\n", - "y = 0.28238756130710596 + 1.9431883792444125*x\n", - "Iteration 400\n", - "y = 0.1909441786231191 + 1.961585247554065*x\n", - "Iteration 500\n", - "y = 0.12911220020207081 + 1.9740248001050498*x\n", - "Iteration 600\n", - "y = 0.0873027937338816 + 1.9824361484423931*x\n", - "Iteration 700\n", - "y = 0.059032204406802784 + 1.988123714820858*x\n", - "Iteration 800\n", - "y = 0.039916261646207894 + 1.9919695205124175*x\n", - "Iteration 900\n", - "y = 0.02699048696926038 + 1.9945699686536877*x\n", - "y = 0.01832191952439833 + 1.9963139384089141*x\n", - "Stochastic Gradient Descent: \n", - "Iteration 0\n", - "y = 0.9619258265142657 + 1.087046466635935*x\n", - "Iteration 100\n", - "y = 0.7920127114741361 + 1.8730271390796145*x\n", - "Iteration 200\n", - "y = 0.546286989963847 + 1.8525856253438988*x\n", - "Iteration 300\n", - "y = 0.3750536398579547 + 1.9349436535828175*x\n", - "Iteration 400\n", - "y = 0.2635085249438779 + 1.9449615901242314*x\n", - "Iteration 500\n", - "y = 0.1866254287868797 + 1.9626152126627878*x\n", - "Iteration 600\n", - "y = 0.1223567694333187 + 1.9718568321569878*x\n", - "Iteration 700\n", - "y = 0.08274738570538319 + 1.9785419924467536*x\n", - "Iteration 800\n", - "y = 0.05431416501871665 + 1.9865732519804664*x\n", - "Iteration 900\n", - "y = 0.03537368025849358 + 1.9898993717967888*x\n", - "y = 0.024016556578563636 + 1.9957094970178637*x\n", - "SGD + Momentum: \n", - "Iteration 0\n", - "y = 0.4357015493218762 + 0.16313132537794608*x\n", - "Iteration 100\n", - "y = 0.03437429906401527 + 2.0233986169269054*x\n", - "Iteration 200\n", - "y = 0.007032959165979722 + 1.9977607487712188*x\n", - "Iteration 300\n", - "y = 0.00041581701105788846 + 1.9998442725098746*x\n", - "Iteration 400\n", - "y = -1.6005163091342098e-06 + 2.000016204187183*x\n", - "Iteration 500\n", - "y = -8.861934936492956e-08 + 1.9999994861022012*x\n", - "Iteration 600\n", - "y = 6.928889468269653e-06 + 1.9999949280688167*x\n", - "Iteration 700\n", - "y = 6.597788280758348e-07 + 2.0000005322576277*x\n", - "Iteration 800\n", - "y = 4.581482668701209e-07 + 1.9999999584079404*x\n", - "Iteration 900\n", - "y = -8.513195132279453e-07 + 2.000002384972658*x\n", - "y = 4.769865694709039e-07 + 1.9999995706104974*x\n", - "Adagrad\n", - "Iteration 0\n", - "y = 0.9909007129257088 + 0.6851578166537742*x\n", - "Iteration 100\n", - "y = 1.752082669089777 + 1.581579077593311*x\n", - "Iteration 200\n", - "y = 1.6531567114050418 + 1.6606835046567618*x\n", - "Iteration 300\n", - "y = 1.5450696681444174 + 1.6823053575450688*x\n", - "Iteration 400\n", - "y = 1.4663489934198473 + 1.7070489395736645*x\n", - "Iteration 500\n", - "y = 1.351363888376385 + 1.7130720212360526*x\n", - "Iteration 600\n", - "y = 1.2678086511145235 + 1.7278507573766195*x\n", - "Iteration 700\n", - "y = 1.1712706810563158 + 1.7451602089657614*x\n", - "Iteration 800\n", - "y = 1.1132576537190118 + 1.7719659013229505*x\n", - "Iteration 900\n", - "y = 1.035517123163937 + 1.7899675001681676*x\n", - "Iteration 1000\n", - "y = 0.9674062829579255 + 1.7948502076582695*x\n", - "Iteration 1100\n", - "y = 0.9082204674720814 + 1.8071128351181995*x\n", - "Iteration 1200\n", - "y = 0.8581451473712463 + 1.8154408505731354*x\n", - "Iteration 1300\n", - "y = 0.8089968516149708 + 1.8328093166423218*x\n", - "Iteration 1400\n", - "y = 0.7657955053734683 + 1.8392007149707281*x\n", - "Iteration 1500\n", - "y = 0.7253357493250404 + 1.8516605673401425*x\n", - "Iteration 1600\n", - "y = 0.682539387341273 + 1.859319366876271*x\n", - "Iteration 1700\n", - "y = 0.6369999006640653 + 1.8634514515704137*x\n", - "Iteration 1800\n", - "y = 0.598817873375494 + 1.8673468379777314*x\n", - "Iteration 1900\n", - "y = 0.5663728642652257 + 1.8745579902145202*x\n", - "Iteration 2000\n", - "y = 0.5397001954509365 + 1.8897515255822968*x\n", - "Iteration 2100\n", - "y = 0.5037008810421729 + 1.892403543540384*x\n", - "Iteration 2200\n", - "y = 0.47389861922693477 + 1.89944214324104*x\n", - "Iteration 2300\n", - "y = 0.44450868566473434 + 1.90549298291394*x\n", - "Iteration 2400\n", - "y = 0.4214024035128387 + 1.9111343460932955*x\n", - "Iteration 2500\n", - "y = 0.3952791860604048 + 1.9142752531345693*x\n", - "Iteration 2600\n", - "y = 0.3659430889961222 + 1.916988189656578*x\n", - "Iteration 2700\n", - "y = 0.3483948220969853 + 1.9249129004722139*x\n", - "Iteration 2800\n", - "y = 0.33235503260927507 + 1.9301450417446813*x\n", - "Iteration 2900\n", - "y = 0.3197901105644779 + 1.936407582605034*x\n", - "Iteration 3000\n", - "y = 0.29976305689632343 + 1.9373260640754866*x\n", - "Iteration 3100\n", - "y = 0.2843666968242483 + 1.941735934755993*x\n", - "Iteration 3200\n", - "y = 0.2692031077146555 + 1.9448569527675903*x\n", - "Iteration 3300\n", - "y = 0.2488251455784735 + 1.9452466519871805*x\n", - "Iteration 3400\n", - "y = 0.2342843571290378 + 1.9501046582154946*x\n", - "Iteration 3500\n", - "y = 0.21988661911352536 + 1.9533270428254617*x\n", - "Iteration 3600\n", - "y = 0.2105228727400937 + 1.955795462193796*x\n", - "Iteration 3700\n", - "y = 0.19493901199836053 + 1.9561507881440283*x\n", - "Iteration 3800\n", - "y = 0.18880117483148465 + 1.9610708152335732*x\n", - "Iteration 3900\n", - "y = 0.17803176767432233 + 1.9625444060857709*x\n", - "Iteration 4000\n", - "y = 0.16711441869227095 + 1.9646134303791487*x\n", - "Iteration 4100\n", - "y = 0.15810281894455053 + 1.9669602997188*x\n", - "Iteration 4200\n", - "y = 0.15287490128429693 + 1.9693218801239154*x\n", - "Iteration 4300\n", - "y = 0.14065981580502707 + 1.968745417771871*x\n", - "Iteration 4400\n", - "y = 0.13544320458564318 + 1.972610509170239*x\n", - "Iteration 4500\n", - "y = 0.12713739736917717 + 1.9732876288943733*x\n", - "Iteration 4600\n", - "y = 0.12073607702270527 + 1.9754767014209547*x\n", - "Iteration 4700\n", - "y = 0.11455068545005868 + 1.976473259600258*x\n", - "Iteration 4800\n", - "y = 0.10887697216212634 + 1.9785797402131986*x\n", - "Iteration 4900\n", - "y = 0.10250516615793719 + 1.9790490254209663*x\n", - "Iteration 5000\n", - "y = 0.09522008487108682 + 1.9796346706769523*x\n", - "Iteration 5100\n", - "y = 0.08858127597176003 + 1.9810893662180828*x\n", - "Iteration 5200\n", - "y = 0.08483749892988729 + 1.982768861515039*x\n", - "Iteration 5300\n", - "y = 0.07886767929269177 + 1.9836549838799284*x\n", - "Iteration 5400\n", - "y = 0.07311361272510263 + 1.9838669836579859*x\n", - "Iteration 5500\n", - "y = 0.06894011176345524 + 1.985044887714682*x\n", - "Iteration 5600\n", - "y = 0.06561188197000849 + 1.9859855261439086*x\n", - "Iteration 5700\n", - "y = 0.06198253754597117 + 1.9875182936539633*x\n", - "Iteration 5800\n", - "y = 0.05818535248057164 + 1.9877238348206285*x\n", - "Iteration 5900\n", - "y = 0.05486798438629148 + 1.9885673727902582*x\n", - "Iteration 6000\n", - "y = 0.05203838159661378 + 1.9891546755284164*x\n", - "Iteration 6100\n", - "y = 0.0484075162579466 + 1.9893463417367592*x\n", - "Iteration 6200\n", - "y = 0.04581501469093452 + 1.9902166936933152*x\n", - "Iteration 6300\n", - "y = 0.0436656810183715 + 1.9905747385133217*x\n", - "Iteration 6400\n", - "y = 0.041312642161708844 + 1.9915114489308239*x\n", - "Iteration 6500\n", - "y = 0.03940855403744079 + 1.9920867082705092*x\n", - "Iteration 6600\n", - "y = 0.03701409021754122 + 1.9923329499725895*x\n", - "Iteration 6700\n", - "y = 0.03492936297892485 + 1.9926572631404766*x\n", - "Iteration 6800\n", - "y = 0.03348341014569901 + 1.9932937318692237*x\n", - "Iteration 6900\n", - "y = 0.031576389769188445 + 1.9935755020721362*x\n", - "Iteration 7000\n", - "y = 0.029450949428277903 + 1.9937487240433411*x\n", - "Iteration 7100\n", - "y = 0.027849462455878844 + 1.994039140685735*x\n", - "Iteration 7200\n", - "y = 0.02645907129728384 + 1.9945658709046947*x\n", - "Iteration 7300\n", - "y = 0.024728927031110553 + 1.9948328724610187*x\n", - "Iteration 7400\n", - "y = 0.02364076205033738 + 1.995301333905134*x\n", - "Iteration 7500\n", - "y = 0.022164224351641244 + 1.9953898754328938*x\n", - "Iteration 7600\n", - "y = 0.020645285154022756 + 1.995577735066188*x\n", - "Iteration 7700\n", - "y = 0.01959386986885427 + 1.9958031488469503*x\n", - "Iteration 7800\n", - "y = 0.018395035579167336 + 1.9960658262335285*x\n", - "Iteration 7900\n", - "y = 0.017550365216273463 + 1.996400177358067*x\n", - "Iteration 8000\n", - "y = 0.01646944688515055 + 1.996364976041052*x\n", - "Iteration 8100\n", - "y = 0.01572605806439289 + 1.996716659515437*x\n", - "Iteration 8200\n", - "y = 0.015200581929641493 + 1.9970621567834477*x\n", - "Iteration 8300\n", - "y = 0.014138795270536781 + 1.9971356679677228*x\n", - "Iteration 8400\n", - "y = 0.013464774825949283 + 1.9972932631081701*x\n", - "Iteration 8500\n", - "y = 0.012706131057463436 + 1.9974110679122312*x\n", - "Iteration 8600\n", - "y = 0.012066268208895998 + 1.9976311955123802*x\n", - "Iteration 8700\n", - "y = 0.011173160510520747 + 1.9976373822300846*x\n", - "Iteration 8800\n", - "y = 0.010500077142079577 + 1.9977509290847406*x\n", - "Iteration 8900\n", - "y = 0.009955166444508366 + 1.9979002520238816*x\n", - "Iteration 9000\n", - "y = 0.009360975496985422 + 1.9979921570347314*x\n", - "Iteration 9100\n", - "y = 0.00900368092345526 + 1.9982302007810173*x\n", - "Iteration 9200\n", - "y = 0.008569581519229328 + 1.9982885032552313*x\n", - "Iteration 9300\n", - "y = 0.008085989294916947 + 1.9984215977579864*x\n", - "Iteration 9400\n", - "y = 0.00752066029356973 + 1.9984226193340426*x\n", - "Iteration 9500\n", - "y = 0.007164866359334228 + 1.9985243732263827*x\n", - "Iteration 9600\n", - "y = 0.006820207250137622 + 1.9985814746697566*x\n", - "Iteration 9700\n", - "y = 0.006457444952760931 + 1.9986186533358985*x\n", - "Iteration 9800\n", - "y = 0.006187705321134588 + 1.998739250433351*x\n", - "Iteration 9900\n", - "y = 0.005910240818639075 + 1.9988259362219711*x\n", - "y = 0.0055443237796650715 + 1.9988402031873795*x\n", - "RMSprop\n", - "Iteration 0\n", - "y = 0.3617918714937247 + 0.8935314611793963*x\n", - "Iteration 100\n", - "y = 1.112143205305486 + 1.5829729739940772*x\n", - "Iteration 200\n", - "y = 1.0585978795258977 + 1.7833251480156982*x\n", - "Iteration 300\n", - "y = 0.7407569067399042 + 1.86060673080023*x\n", - "Iteration 400\n", - "y = 0.42064360806770007 + 1.9173416530941951*x\n", - "Iteration 500\n", - "y = 0.1111922206643335 + 1.9781148405601097*x\n", - "Iteration 600\n", - "y = 0.001379989704715848 + 2.000825479832667*x\n", - "Iteration 700\n", - "y = -0.005263769903600025 + 1.9951254183042317*x\n", - "Iteration 800\n", - "y = 0.002066339142489657 + 2.000527389852606*x\n", - "Iteration 900\n", - "y = -0.0018974037452918922 + 2.0004246592997204*x\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Iteration 1000\n", - "y = -0.039749095115406675 + 1.987267555742025*x\n", - "Iteration 1100\n", - "y = 0.005145669157455596 + 1.9984807726790454*x\n", - "Iteration 1200\n", - "y = -0.0018140213445952532 + 2.0037880205456013*x\n", - "Iteration 1300\n", - "y = -0.01278392583858953 + 2.0039946207483874*x\n", - "Iteration 1400\n", - "y = 0.0006377257328787176 + 1.999931949377187*x\n", - "Iteration 1500\n", - "y = 0.0009988082670943522 + 1.9943520576447586*x\n", - "Iteration 1600\n", - "y = -0.01806246811933287 + 1.990932459663341*x\n", - "Iteration 1700\n", - "y = 0.003213682754240674 + 1.999003713429964*x\n", - "Iteration 1800\n", - "y = 0.011017420383813752 + 1.9970627341422795*x\n", - "Iteration 1900\n", - "y = -0.009959479920880443 + 2.0064445494038146*x\n", - "Iteration 2000\n", - "y = 0.005708949443180763 + 1.998207036748069*x\n", - "Iteration 2100\n", - "y = -0.0018150808151644426 + 2.0003408117602763*x\n", - "Iteration 2200\n", - "y = -0.00026099203332354817 + 1.998588111772451*x\n", - "Iteration 2300\n", - "y = -0.008553796181148434 + 1.9855130641651118*x\n", - "Iteration 2400\n", - "y = 0.00021324745550753877 + 2.0011468628037545*x\n", - "Iteration 2500\n", - "y = 0.0003281239054911537 + 1.9998022119570147*x\n", - "Iteration 2600\n", - "y = -0.007675847124862011 + 1.9886455178959226*x\n", - "Iteration 2700\n", - "y = 0.0008883708558365342 + 2.0000632313132063*x\n", - "Iteration 2800\n", - "y = -0.00687138003536456 + 2.001969910081975*x\n", - "Iteration 2900\n", - "y = -0.002180629593012344 + 1.999979421229616*x\n", - "Iteration 3000\n", - "y = 0.023770950558197647 + 1.9936958717485012*x\n", - "Iteration 3100\n", - "y = -0.0009907633618757314 + 2.000043934000511*x\n", - "Iteration 3200\n", - "y = 0.000420119600382865 + 1.9998617994403107*x\n", - "Iteration 3300\n", - "y = 0.01453001408007369 + 1.9934154454703121*x\n", - "Iteration 3400\n", - "y = -0.0026727121181083133 + 2.0009139063502044*x\n", - "Iteration 3500\n", - "y = 0.013028222857044768 + 2.007482456109788*x\n", - "Iteration 3600\n", - "y = -0.00121031965803373 + 2.0002209422805435*x\n", - "Iteration 3700\n", - "y = -0.007866015881214185 + 2.0037779537526617*x\n", - "Iteration 3800\n", - "y = 0.002897638917493862 + 1.998571751217742*x\n", - "Iteration 3900\n", - "y = -0.0014974180283573396 + 2.0003637721136536*x\n", - "Iteration 4000\n", - "y = -0.000671947083914073 + 1.999472014677302*x\n", - "Iteration 4100\n", - "y = 0.00498768427618511 + 1.9983437312880752*x\n", - "Iteration 4200\n", - "y = -0.00831301984785493 + 2.0041549228809497*x\n", - "Iteration 4300\n", - "y = -0.00031632027945036853 + 1.9998090048269228*x\n", - "Iteration 4400\n", - "y = 0.001069989178476908 + 1.9995701928587326*x\n", - "Iteration 4500\n", - "y = -0.0026546082920498853 + 2.000350406236675*x\n", - "Iteration 4600\n", - "y = -0.003535097434274846 + 1.9940003125763315*x\n", - "Iteration 4700\n", - "y = 0.0029590453393060993 + 2.002472261531745*x\n", - "Iteration 4800\n", - "y = 0.006296740927637731 + 1.9984689158019062*x\n", - "Iteration 4900\n", - "y = 0.005277866570282745 + 2.009966222300057*x\n", - "Iteration 5000\n", - "y = -0.0036468857223469213 + 1.9982953157903156*x\n", - "Iteration 5100\n", - "y = 0.007120604467174619 + 2.003384798643488*x\n", - "Iteration 5200\n", - "y = -0.0008533823564239276 + 2.0004744615977263*x\n", - "Iteration 5300\n", - "y = 0.002027762405976359 + 1.998254300772477*x\n", - "Iteration 5400\n", - "y = 0.0036023905324369518 + 1.9962283192455492*x\n", - "Iteration 5500\n", - "y = -0.006259424080465047 + 2.00230533560054*x\n", - "Iteration 5600\n", - "y = 0.008393599939131786 + 2.019473618148715*x\n", - "Iteration 5700\n", - "y = -0.004050781931848291 + 2.0028367476512647*x\n", - "Iteration 5800\n", - "y = -0.0008371108760339479 + 2.0139258788525116*x\n", - "Iteration 5900\n", - "y = -0.002885408986600588 + 2.0020799185868388*x\n", - "Iteration 6000\n", - "y = 0.0053486797844405146 + 2.0024814243971716*x\n", - "Iteration 6100\n", - "y = 0.0033502490503319856 + 1.9977984418251111*x\n", - "Iteration 6200\n", - "y = 0.006127720551620504 + 2.0016390400463493*x\n", - "Iteration 6300\n", - "y = -0.001822037695541198 + 2.0018140619687257*x\n", - "Iteration 6400\n", - "y = -0.01079328552863512 + 2.0037761462415054*x\n", - "Iteration 6500\n", - "y = 0.00717505822557291 + 2.0002046020407143*x\n", - "Iteration 6600\n", - "y = 0.008800774749567117 + 2.0083165181185345*x\n", - "Iteration 6700\n", - "y = -0.014007435444718863 + 1.9983551539585414*x\n", - "Iteration 6800\n", - "y = 0.0051346133060544774 + 2.000154001014758*x\n", - "Iteration 6900\n", - "y = -0.003621173344141699 + 2.000588337910482*x\n", - "Iteration 7000\n", - "y = 0.0016322509363011957 + 1.9993573151771353*x\n", - "Iteration 7100\n", - "y = -0.001304944778832026 + 2.000625310850428*x\n", - "Iteration 7200\n", - "y = 0.002067417129937262 + 2.000184889451102*x\n", - "Iteration 7300\n", - "y = -0.005988455813080613 + 2.0077379629708*x\n", - "Iteration 7400\n", - "y = -0.00802139938656621 + 1.9992232943780295*x\n", - "Iteration 7500\n", - "y = -0.00231443475705327 + 2.0005518272429454*x\n", - "Iteration 7600\n", - "y = 0.0033070611934870944 + 2.0008275114675347*x\n", - "Iteration 7700\n", - "y = -0.006321870496990727 + 1.9997121317668411*x\n", - "Iteration 7800\n", - "y = 0.00041857429593791275 + 1.9923646157248096*x\n", - "Iteration 7900\n", - "y = -0.02528608229047568 + 1.986590769618216*x\n", - "Iteration 8000\n", - "y = -0.01938780384403209 + 2.008172789141479*x\n", - "Iteration 8100\n", - "y = 0.006421405527212393 + 1.9842548123557364*x\n", - "Iteration 8200\n", - "y = 0.01589584925668931 + 2.0083314389292974*x\n", - "Iteration 8300\n", - "y = -0.016649136579011476 + 1.9853758028242927*x\n", - "Iteration 8400\n", - "y = -1.51266700507626e-05 + 2.0000118355513514*x\n", - "Iteration 8500\n", - "y = -5.361102978206546e-05 + 2.0000177909536254*x\n", - "Iteration 8600\n", - "y = -0.0075402522254625366 + 1.9898598252275863*x\n", - "Iteration 8700\n", - "y = 0.010084811825024608 + 1.9984178068716432*x\n", - "Iteration 8800\n", - "y = -0.006582726540762198 + 2.0046617518584764*x\n", - "Iteration 8900\n", - "y = -0.0006801046136838308 + 1.9995312304839528*x\n", - "Iteration 9000\n", - "y = -0.005396359895809784 + 1.9997415176351763*x\n", - "Iteration 9100\n", - "y = -0.01029870164535155 + 2.001326738259382*x\n", - "Iteration 9200\n", - "y = -0.008363402959870082 + 2.001004330294006*x\n", - "Iteration 9300\n", - "y = -0.004623361779183724 + 2.002624012829024*x\n", - "Iteration 9400\n", - "y = 0.0023337854043705823 + 1.9991196683520087*x\n", - "Iteration 9500\n", - "y = -0.003479605462548233 + 1.9991766085646165*x\n", - "Iteration 9600\n", - "y = 1.9113254449676645e-05 + 1.9999998433258432*x\n", - "Iteration 9700\n", - "y = 0.0004901021539388111 + 1.9996842127475674*x\n", - "Iteration 9800\n", - "y = -0.0037188565728199625 + 2.0012411521197166*x\n", - "Iteration 9900\n", - "y = 0.0044477763680744 + 1.9999679396358498*x\n", - "y = -0.0011675162828293233 + 1.9985121181517174*x\n", - "Adadelta\n", + "Adam\n", "Iteration 0\n", - "y = 0.41096539511307256 + 0.49772101678589104*x\n", + "y = 0.7770901717569257 + 0.9433753832651581*x\n", "Iteration 100\n", - "y = 0.5135798913712705 + 0.5696340910509599*x\n", + "y = 0.32725296586754515 + 1.900221970247115*x\n", "Iteration 200\n", - "y = 0.6159518945690069 + 0.6388291385750045*x\n", + "y = 0.004147083459202631 + 1.99892111204854*x\n", "Iteration 300\n", - "y = 0.7092237980405368 + 0.6970127295556214*x\n", + "y = -3.4688751132678575e-05 + 2.000013509462116*x\n", "Iteration 400\n", - "y = 0.8000787749721939 + 0.7543026226018996*x\n", + "y = -9.06655931188767e-08 + 1.9999999764669432*x\n", "Iteration 500\n", - "y = 0.888457081567781 + 0.8115820641162809*x\n", + "y = 1.8387422421644338e-10 + 2.0000000000594307*x\n", "Iteration 600\n", - "y = 0.9700712825110333 + 0.8664154632647859*x\n", + "y = 4.361065425337935e-12 + 1.9999999999993485*x\n", "Iteration 700\n", - "y = 1.052913095059698 + 0.9228620322870312*x\n", + "y = 1.1991224399817034e-13 + 1.9999999999999527*x\n", "Iteration 800\n", - "y = 1.1427113559550428 + 0.9872195554878949*x\n", + "y = -2.5150603804494032e-17 + 2.000000000000001*x\n", "Iteration 900\n", - "y = 1.2156348797474466 + 1.0393615432768293*x\n", - "Iteration 1000\n", - "y = 1.28370930318869 + 1.0928250473441095*x\n", - "Iteration 1100\n", - "y = 1.3658196689296205 + 1.1554026707086344*x\n", - "Iteration 1200\n", - "y = 1.4392769212672991 + 1.2133508338684948*x\n", - "Iteration 1300\n", - "y = 1.4980452955086607 + 1.2596297240563434*x\n", - "Iteration 1400\n", - "y = 1.5525476509395997 + 1.304787577243706*x\n", - "Iteration 1500\n", - "y = 1.595337893009435 + 1.3434453631056735*x\n", - "Iteration 1600\n", - "y = 1.640360027928069 + 1.3864646800798137*x\n", - "Iteration 1700\n", - "y = 1.6844946083322918 + 1.428290059679983*x\n", - "Iteration 1800\n", - "y = 1.7255113510793385 + 1.472600826345899*x\n", - "Iteration 1900\n", - "y = 1.743913071501007 + 1.5053963988127388*x\n", - "Iteration 2000\n", - "y = 1.759343796504421 + 1.5334651004381192*x\n", - "Iteration 2100\n", - "y = 1.776818555354216 + 1.5628279732789563*x\n", - "Iteration 2200\n", - "y = 1.78362773959923 + 1.5917055647784415*x\n", - "Iteration 2300\n", - "y = 1.767585502021221 + 1.6096269021826028*x\n", - "Iteration 2400\n", - "y = 1.7387011247315654 + 1.6183577319981461*x\n", - "Iteration 2500\n", - "y = 1.698660507418387 + 1.6284167901121807*x\n", - "Iteration 2600\n", - "y = 1.6669960589851716 + 1.63823204083993*x\n", - "Iteration 2700\n", - "y = 1.6343083984307476 + 1.6492788741150122*x\n", - "Iteration 2800\n", - "y = 1.613170732340005 + 1.664159395506083*x\n", - "Iteration 2900\n", - "y = 1.5612647587480215 + 1.657932217894772*x\n", - "Iteration 3000\n", - "y = 1.5286468803675302 + 1.6655548278266634*x\n", - "Iteration 3100\n", - "y = 1.5091490416926587 + 1.6810679423849475*x\n", - "Iteration 3200\n", - "y = 1.471892557581634 + 1.687024489777794*x\n", - "Iteration 3300\n", - "y = 1.4388786269393954 + 1.6967734369019145*x\n", - "Iteration 3400\n", - "y = 1.3959013004978815 + 1.7024133205324061*x\n", - "Iteration 3500\n", - "y = 1.3561811530406536 + 1.707095283009188*x\n", - "Iteration 3600\n", - "y = 1.3371628988825486 + 1.7206721934439055*x\n", - "Iteration 3700\n", - "y = 1.2966889049328 + 1.723337381347839*x\n", - "Iteration 3800\n", - "y = 1.2662759661567522 + 1.7344195725906013*x\n", - "Iteration 3900\n", - "y = 1.2258099189301868 + 1.737440713843289*x\n", - "Iteration 4000\n", - "y = 1.1923213294612487 + 1.7450313631919752*x\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Iteration 4100\n", - "y = 1.1703668469299928 + 1.7578849874677622*x\n", - "Iteration 4200\n", - "y = 1.1299443751200116 + 1.7596081596761088*x\n", - "Iteration 4300\n", - "y = 1.0857688501669025 + 1.7603755539483317*x\n", - "Iteration 4400\n", - "y = 1.067999887278753 + 1.7764118268018219*x\n", - "Iteration 4500\n", - "y = 1.027562399896915 + 1.781172047294927*x\n", - "Iteration 4600\n", - "y = 1.0135197357347023 + 1.8003402340929056*x\n", - "Iteration 4700\n", - "y = 0.9867339323786954 + 1.8049278823229686*x\n", - "Iteration 4800\n", - "y = 0.9570536816927211 + 1.8130726368629162*x\n", - "Iteration 4900\n", - "y = 0.916086185933129 + 1.8121264546965077*x\n", - "Iteration 5000\n", - "y = 0.8769579109672395 + 1.8178313845022325*x\n", - "Iteration 5100\n", - "y = 0.8375535213298363 + 1.8197866853883014*x\n", - "Iteration 5200\n", - "y = 0.8102089699606075 + 1.830294102584346*x\n", - "Iteration 5300\n", - "y = 0.7689489406316838 + 1.8354861163350307*x\n", - "Iteration 5400\n", - "y = 0.7308673994842835 + 1.8382178533040052*x\n", - "Iteration 5500\n", - "y = 0.6944176540689229 + 1.8482404095568499*x\n", - "Iteration 5600\n", - "y = 0.658045742524041 + 1.8554471817071359*x\n", - "Iteration 5700\n", - "y = 0.62844627375962 + 1.8704283878917267*x\n", - "Iteration 5800\n", - "y = 0.5913435534368703 + 1.8791222975164237*x\n", - "Iteration 5900\n", - "y = 0.557880285523112 + 1.8856786959768919*x\n", - "Iteration 6000\n", - "y = 0.5283773774681808 + 1.8904341686177837*x\n", - "Iteration 6100\n", - "y = 0.49319643189670126 + 1.8944019026229129*x\n", - "Iteration 6200\n", - "y = 0.4609224230313071 + 1.9028915395364874*x\n", - "Iteration 6300\n", - "y = 0.42138690265759426 + 1.9021790027550698*x\n", - "Iteration 6400\n", - "y = 0.3970450482092413 + 1.9153321103241103*x\n", - "Iteration 6500\n", - "y = 0.36572250604762574 + 1.9193010121546064*x\n", - "Iteration 6600\n", - "y = 0.3393945033197329 + 1.9302320567756317*x\n", - "Iteration 6700\n", - "y = 0.3195116556438301 + 1.9387953821760682*x\n", - "Iteration 6800\n", - "y = 0.28123080105581444 + 1.9391273898400867*x\n", - "Iteration 6900\n", - "y = 0.2564038851696523 + 1.949412865769459*x\n", - "Iteration 7000\n", - "y = 0.222593366420209 + 1.9542986067640842*x\n", - "Iteration 7100\n", - "y = 0.19240271968462694 + 1.9606054521401473*x\n", - "Iteration 7200\n", - "y = 0.16086644794298657 + 1.9645176738815031*x\n", - "Iteration 7300\n", - "y = 0.13394368951339167 + 1.972807413889778*x\n", - "Iteration 7400\n", - "y = 0.1033010927351425 + 1.976535976077865*x\n", - "Iteration 7500\n", - "y = 0.07330081308513484 + 1.9867981356223945*x\n", - "Iteration 7600\n", - "y = 0.04591682632321923 + 1.9900516435531488*x\n", - "Iteration 7700\n", - "y = 0.026016025769042016 + 1.9961993630414174*x\n", - "Iteration 7800\n", - "y = 0.005871260836460236 + 1.9987489150441338*x\n", - "Iteration 7900\n", - "y = 0.00018543267521520636 + 1.9999684544122942*x\n", - "Iteration 8000\n", - "y = 0.00016069301093268778 + 2.000403053425948*x\n", - "Iteration 8100\n", - "y = 0.0001495165478503408 + 1.9999055770362804*x\n", - "Iteration 8200\n", - "y = -0.00021203944400251267 + 2.0000418860959344*x\n", - "Iteration 8300\n", - "y = -0.000454197749859367 + 1.999221586600376*x\n", - "Iteration 8400\n", - "y = 0.0009087599563562013 + 2.0008070164067924*x\n", - "Iteration 8500\n", - "y = -0.0003932522065875782 + 1.9993590721304573*x\n", - "Iteration 8600\n", - "y = -0.0011023116431645233 + 1.9999449653143944*x\n", - "Iteration 8700\n", - "y = 7.194341189289567e-06 + 2.000000939780076*x\n", - "Iteration 8800\n", - "y = -0.0001273701353909281 + 2.0000586485237015*x\n", - "Iteration 8900\n", - "y = -2.195422541230059e-05 + 2.0000024542513333*x\n", - "Iteration 9000\n", - "y = -9.505541145328072e-05 + 2.0000455552247303*x\n", - "Iteration 9100\n", - "y = -2.895797595100533e-07 + 1.9999865527702694*x\n", - "Iteration 9200\n", - "y = 0.0009239866572852231 + 2.0009316180949837*x\n", - "Iteration 9300\n", - "y = -0.00015004980985254754 + 2.000061477284803*x\n", - "Iteration 9400\n", - "y = -0.0011018145778223334 + 2.000405348640581*x\n", - "Iteration 9500\n", - "y = 3.882083990660555e-05 + 1.99999128375624*x\n", - "Iteration 9600\n", - "y = 0.002341366721941023 + 2.000177268798233*x\n", - "Iteration 9700\n", - "y = 8.582218044672942e-05 + 1.9999793949381646*x\n", - "Iteration 9800\n", - "y = -7.287261379954102e-05 + 1.9999807707232216*x\n", - "Iteration 9900\n", - "y = 8.230621575420588e-05 + 1.9999560748798169*x\n", - "y = 1.1635285794088352e-05 + 1.9999702674692446*x\n" + "y = -8.298690846122729e-17 + 2.0*x\n", + "y = 9.891579815200104e-17 + 2.0*x\n" ] } ], @@ -1734,6 +466,7 @@ "xs = [1,2,3,4,5,6,7]\n", "ys = [2,4,6,8,10,12,14]\n", "\n", + "\n", "# Gradient Descent\n", "model = Line()\n", "print(\"Gradient Descent: \")\n", @@ -1768,702 +501,46 @@ "model = Line()\n", "print(\"Adadelta\")\n", "adadelta(model, xs, ys)\n", + "print(model)\n", + "\n", + "# Adam\n", + "model = Line()\n", + "print(\"Adam\")\n", + "adam(model, xs, ys)\n", "print(model)" ] }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 29, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "Gradient Descent: \n", + "Adam\n", "Iteration 0\n", - "y = 0.3758312748922319 + 0.9861076428125647*x\n", + "y = 0.7139491962653317 + 0.6462402260655279*x\n", "Iteration 100\n", - "y = 0.7298486719499636 + 2.0543498967332554*x\n", + "y = 1.5012704143511661 + 1.913754894630153*x\n", "Iteration 200\n", - "y = 0.8173296897367237 + 2.036750189498243*x\n", + "y = 1.147275006702479 + 1.9626892285633653*x\n", "Iteration 300\n", - "y = 0.8764824052780473 + 2.0248496595087433*x\n", + "y = 1.0187321764566213 + 1.9959358177402393*x\n", "Iteration 400\n", - "y = 0.9164801538689702 + 2.0168027862204627*x\n", + "y = 1.0018116802276642 + 1.9995771873157036*x\n", "Iteration 500\n", - "y = 0.9435257404950816 + 2.0113616697512993*x\n", + "y = 1.0001550519477989 + 1.9999713016943046*x\n", "Iteration 600\n", - "y = 0.9618133637168664 + 2.0076825079986067*x\n", + "y = 1.0000046235977667 + 1.9999989555915936*x\n", "Iteration 700\n", - "y = 0.9741790471729282 + 2.0051947407767163*x\n", + "y = 0.9999999689046957 + 2.0000000091643306*x\n", "Iteration 800\n", - "y = 0.9825404468737576 + 2.0035125679976087*x\n", + "y = 1.0000000007114835 + 1.999999999822266*x\n", "Iteration 900\n", - "y = 0.9881942390968437 + 2.0023751202356674*x\n", - "y = 0.9919859096488965 + 2.001612299988075*x\n", - "Stochastic Gradient Descent: \n", - "Iteration 0\n", - "y = 0.9153217017005606 + 0.34378094132412806*x\n", - "Iteration 100\n", - "y = 1.1818667130118197 + 1.9585589470138083*x\n", - "Iteration 200\n", - "y = 1.117518306355778 + 1.9755644836754853*x\n", - "Iteration 300\n", - "y = 1.0847443749817014 + 1.9853921780077008*x\n", - "Iteration 400\n", - "y = 1.0553999353012187 + 1.9821880215759884*x\n", - "Iteration 500\n", - "y = 1.0383634461008067 + 1.9944754938756803*x\n", - "Iteration 600\n", - "y = 1.025382430134353 + 1.9954382947377485*x\n", - "Iteration 700\n", - "y = 1.017071735717545 + 1.9960754343538933*x\n", - "Iteration 800\n", - "y = 1.0124707435584295 + 1.9982181968759725*x\n", - "Iteration 900\n", - "y = 1.0083118589831706 + 1.99851274149033*x\n", - "y = 1.0054053368750777 + 1.9992228722028862*x\n", - "SGD + Momentum: \n", - "Iteration 0\n", - "y = 0.20190901079079593 + 1.0518015116766273*x\n", - "Iteration 100\n", - "y = 1.0571865268999219 + 1.9352148905303475*x\n", - "Iteration 200\n", - "y = 1.1786162978971049 + 1.757988907429454*x\n", - "Iteration 300\n", - "y = 1.0788638670750186 + 1.9472585317836537*x\n", - "Iteration 400\n", - "y = 1.006748704347011 + 1.9734339051541958*x\n", - "Iteration 500\n", - "y = 0.9343812881913555 + 2.023858333175996*x\n", - "Iteration 600\n", - "y = 1.0012657312666593 + 2.012962154470421*x\n", - "Iteration 700\n", - "y = 1.0013102950137485 + 1.9999783949545147*x\n", - "Iteration 800\n", - "y = 0.998734351972466 + 2.000339221136913*x\n", - "Iteration 900\n", - "y = 1.0000768929259078 + 1.999978546462978*x\n", - "y = 0.9999897833887086 + 1.9999763351360174*x\n", - "Adagrad\n", - "Iteration 0\n", - "y = 0.46197321190711843 + 0.7846328197159732*x\n", - "Iteration 100\n", - "y = 1.5052377440267988 + 1.7671401283436048*x\n", - "Iteration 200\n", - "y = 1.5594483598257016 + 1.8544038545585*x\n", - "Iteration 300\n", - "y = 1.5416573022476152 + 1.8826674398565306*x\n", - "Iteration 400\n", - "y = 1.5105934430891446 + 1.8894737717553833*x\n", - "Iteration 500\n", - "y = 1.4851077194273175 + 1.9004893472809155*x\n", - "Iteration 600\n", - "y = 1.4529679236833892 + 1.904230324715688*x\n", - "Iteration 700\n", - "y = 1.423047644860893 + 1.9086247163631147*x\n", - "Iteration 800\n", - "y = 1.402257621745607 + 1.9186799788576734*x\n", - "Iteration 900\n", - "y = 1.3763020939271235 + 1.922098963338237*x\n", - "Iteration 1000\n", - "y = 1.350180410295563 + 1.9250267975694428*x\n", - "Iteration 1100\n", - "y = 1.3295156720779202 + 1.9317733869265763*x\n", - "Iteration 1200\n", - "y = 1.3118689737973368 + 1.9365404702053604*x\n", - "Iteration 1300\n", - "y = 1.2899837775106766 + 1.939753831557365*x\n", - "Iteration 1400\n", - "y = 1.2695413102750541 + 1.942386962434873*x\n", - "Iteration 1500\n", - "y = 1.2531919440311503 + 1.946722160682169*x\n", - "Iteration 1600\n", - "y = 1.2380934268103059 + 1.9507246478420963*x\n", - "Iteration 1700\n", - "y = 1.224888221768581 + 1.953009315802062*x\n", - "Iteration 1800\n", - "y = 1.213659518362757 + 1.9571066727315032*x\n", - "Iteration 1900\n", - "y = 1.2013752159564801 + 1.9603783167894842*x\n", - "Iteration 2000\n", - "y = 1.1873258795817543 + 1.9610920817269761*x\n", - "Iteration 2100\n", - "y = 1.1769599553743608 + 1.9636110266334854*x\n", - "Iteration 2200\n", - "y = 1.1658629388445951 + 1.9652361589969969*x\n", - "Iteration 2300\n", - "y = 1.1554438951894321 + 1.9676660909306207*x\n", - "Iteration 2400\n", - "y = 1.1445331859470291 + 1.9692525352686878*x\n", - "Iteration 2500\n", - "y = 1.1348165297115897 + 1.9717949820531606*x\n", - "Iteration 2600\n", - "y = 1.1261048508515679 + 1.9745872018696229*x\n", - "Iteration 2700\n", - "y = 1.1191752856709707 + 1.9758014723282842*x\n", - "Iteration 2800\n", - "y = 1.1125467385150944 + 1.9768074389209953*x\n", - "Iteration 2900\n", - "y = 1.1041050923458837 + 1.977919291075865*x\n", - "Iteration 3000\n", - "y = 1.0985709783531161 + 1.9797003591385502*x\n", - "Iteration 3100\n", - "y = 1.0912979877489628 + 1.9804107529481685*x\n", - "Iteration 3200\n", - "y = 1.0853550561726002 + 1.9823552363158488*x\n", - "Iteration 3300\n", - "y = 1.0814933360886434 + 1.9838710061177682*x\n", - "Iteration 3400\n", - "y = 1.0766163050856172 + 1.9844690602690538*x\n", - "Iteration 3500\n", - "y = 1.071815989216191 + 1.9852828101337072*x\n", - "Iteration 3600\n", - "y = 1.0673208500377687 + 1.986254780509397*x\n", - "Iteration 3700\n", - "y = 1.0637196169802738 + 1.9868289841930862*x\n", - "Iteration 3800\n", - "y = 1.0595250611587275 + 1.9873273220140755*x\n", - "Iteration 3900\n", - "y = 1.0561681662719131 + 1.988090310151372*x\n", - "Iteration 4000\n", - "y = 1.0538463556883015 + 1.9890449401729267*x\n", - "Iteration 4100\n", - "y = 1.0509363546534622 + 1.9898534645690964*x\n", - "Iteration 4200\n", - "y = 1.0478800337372782 + 1.9902579290975961*x\n", - "Iteration 4300\n", - "y = 1.0446674040332962 + 1.9908348181849524*x\n", - "Iteration 4400\n", - "y = 1.042142936612279 + 1.9914270079054939*x\n", - "Iteration 4500\n", - "y = 1.0391197596535569 + 1.9913132403175375*x\n", - "Iteration 4600\n", - "y = 1.0376170643415348 + 1.992257447184021*x\n", - "Iteration 4700\n", - "y = 1.0350019291975099 + 1.9927613163711613*x\n", - "Iteration 4800\n", - "y = 1.0324034579627575 + 1.9932344569026264*x\n", - "Iteration 4900\n", - "y = 1.0298988839592453 + 1.9935772770832347*x\n", - "Iteration 5000\n", - "y = 1.0277353985761852 + 1.9938274794384503*x\n", - "Iteration 5100\n", - "y = 1.0259721842561857 + 1.9943834837322028*x\n", - "Iteration 5200\n", - "y = 1.0241614336282632 + 1.9949389334014196*x\n", - "Iteration 5300\n", - "y = 1.0226287044325455 + 1.9951708325900213*x\n", - "Iteration 5400\n", - "y = 1.0207977721159034 + 1.995434466269164*x\n", - "Iteration 5500\n", - "y = 1.0198384470166668 + 1.9959134818907638*x\n", - "Iteration 5600\n", - "y = 1.0186259895364618 + 1.9959664856945551*x\n", - "Iteration 5700\n", - "y = 1.017459644197282 + 1.9962759466631248*x\n", - "Iteration 5800\n", - "y = 1.0165381607180675 + 1.9964814679055072*x\n", - "Iteration 5900\n", - "y = 1.0157824967831135 + 1.9968415272488336*x\n", - "Iteration 6000\n", - "y = 1.0146462651059427 + 1.9969600537779528*x\n", - "Iteration 6100\n", - "y = 1.013577687106957 + 1.99701209404658*x\n", - "Iteration 6200\n", - "y = 1.0128402167607555 + 1.9972862794166322*x\n", - "Iteration 6300\n", - "y = 1.0120468061442849 + 1.9974653590050073*x\n", - "Iteration 6400\n", - "y = 1.0113206958064636 + 1.9977453141225925*x\n", - "Iteration 6500\n", - "y = 1.010530159402851 + 1.9977613337659024*x\n", - "Iteration 6600\n", - "y = 1.009980138597552 + 1.9980332618281365*x\n", - "Iteration 6700\n", - "y = 1.0093487196824678 + 1.9981551053299673*x\n", - "Iteration 6800\n", - "y = 1.0087331396475556 + 1.998201166952451*x\n", - "Iteration 6900\n", - "y = 1.0080090839353189 + 1.998197389772428*x\n", - "Iteration 7000\n", - "y = 1.0074757348954773 + 1.9983154896568611*x\n", - "Iteration 7100\n", - "y = 1.007236458180851 + 1.9985444842510762*x\n", - "Iteration 7200\n", - "y = 1.0067529319418242 + 1.9986379407621238*x\n", - "Iteration 7300\n", - "y = 1.00634190190281 + 1.9987309387790442*x\n", - "Iteration 7400\n", - "y = 1.0059226313323375 + 1.9987311909429517*x\n", - "Iteration 7500\n", - "y = 1.005524467191352 + 1.9987729293651593*x\n", - "Iteration 7600\n", - "y = 1.005274208836092 + 1.9989066343627426*x\n", - "Iteration 7700\n", - "y = 1.0048318638028684 + 1.998906325926456*x\n", - "Iteration 7800\n", - "y = 1.0045302861661434 + 1.9989896151630528*x\n", - "Iteration 7900\n", - "y = 1.004369639050609 + 1.9991210307541332*x\n", - "Iteration 8000\n", - "y = 1.004072150462645 + 1.9991550130190237*x\n", - "Iteration 8100\n", - "y = 1.0038815482480965 + 1.9992146571494605*x\n", - "Iteration 8200\n", - "y = 1.0036053090270298 + 1.999266580744465*x\n", - "Iteration 8300\n", - "y = 1.0033362397114984 + 1.9992921528020966*x\n", - "Iteration 8400\n", - "y = 1.003138139750984 + 1.999355461201285*x\n", - "Iteration 8500\n", - "y = 1.0028797750786755 + 1.9993582896792537*x\n", - "Iteration 8600\n", - "y = 1.0027090165063688 + 1.9994153894934181*x\n", - "Iteration 8700\n", - "y = 1.0025466578018487 + 1.9994710623624246*x\n", - "Iteration 8800\n", - "y = 1.0024144151030312 + 1.9995004213878222*x\n", - "Iteration 8900\n", - "y = 1.0022599591672063 + 1.999535323343836*x\n", - "Iteration 9000\n", - "y = 1.0020805679749356 + 1.9995521623130912*x\n", - "Iteration 9100\n", - "y = 1.0019722538695515 + 1.9995814312463804*x\n", - "Iteration 9200\n", - "y = 1.0018692579468356 + 1.999610295930305*x\n", - "Iteration 9300\n", - "y = 1.001778330673022 + 1.9996462470664884*x\n", - "Iteration 9400\n", - "y = 1.00166834588169 + 1.9996589842747674*x\n", - "Iteration 9500\n", - "y = 1.0015628034035202 + 1.9996712678406139*x\n", - "Iteration 9600\n", - "y = 1.0014988307870971 + 1.9997063145149494*x\n", - "Iteration 9700\n", - "y = 1.0013759781959388 + 1.9997020889180313*x\n", - "Iteration 9800\n", - "y = 1.0013072772407925 + 1.9997292563041686*x\n", - "Iteration 9900\n", - "y = 1.001213260512901 + 1.9997442780684724*x\n", - "y = 1.001153467040044 + 1.9997696502601017*x\n", - "RMSprop\n", - "Iteration 0\n", - "y = 0.2667985444668855 + 0.14627453827716944*x\n", - "Iteration 100\n", - "y = 1.2115532208964666 + 0.9810284459215135*x\n", - "Iteration 200\n", - "y = 1.948568290850704 + 1.6564224481614276*x\n", - "Iteration 300\n", - "y = 1.8368235793727399 + 1.8535190890878868*x\n", - "Iteration 400\n", - "y = 1.5025089794518447 + 1.9076224035453175*x\n", - "Iteration 500\n", - "y = 1.2253246112433784 + 1.9626249064778567*x\n", - "Iteration 600\n", - "y = 1.011998317935152 + 1.9852302459313313*x\n", - "Iteration 700\n", - "y = 1.003770302710899 + 2.0071998964838667*x\n", - "Iteration 800\n", - "y = 0.997336318684003 + 1.9987923138667592*x\n", - "Iteration 900\n", - "y = 1.0258796734055562 + 2.003795708893619*x\n", - "Iteration 1000\n", - "y = 0.9971908625915417 + 1.9947659208257067*x\n", - "Iteration 1100\n", - "y = 0.9845550178577193 + 1.989552391180007*x\n", - "Iteration 1200\n", - "y = 1.001041297605822 + 1.9998373684011863*x\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Iteration 1300\n", - "y = 1.013456608574291 + 2.003760445243725*x\n", - "Iteration 1400\n", - "y = 1.000604429686155 + 1.999814973888711*x\n", - "Iteration 1500\n", - "y = 0.9983353003157828 + 2.002377400504234*x\n", - "Iteration 1600\n", - "y = 1.0079070299852295 + 1.99911172255804*x\n", - "Iteration 1700\n", - "y = 1.0057974653348982 + 2.0092064057307333*x\n", - "Iteration 1800\n", - "y = 1.0012182835397407 + 1.9912445656314368*x\n", - "Iteration 1900\n", - "y = 0.9935730533373829 + 2.004907314371268*x\n", - "Iteration 2000\n", - "y = 1.0068544594217217 + 1.9945796914142868*x\n", - "Iteration 2100\n", - "y = 1.0034692386766948 + 1.999877157270015*x\n", - "Iteration 2200\n", - "y = 1.0238524454768172 + 2.003215919656625*x\n", - "Iteration 2300\n", - "y = 1.0006305118741923 + 2.000139723628984*x\n", - "Iteration 2400\n", - "y = 1.0133749706891342 + 2.0028699564784285*x\n", - "Iteration 2500\n", - "y = 1.0065502964187343 + 1.998588577290943*x\n", - "Iteration 2600\n", - "y = 1.0001875380771728 + 2.0005311718749885*x\n", - "Iteration 2700\n", - "y = 0.9983088913058962 + 1.9998294101697964*x\n", - "Iteration 2800\n", - "y = 1.0024019194786755 + 1.9994309554909746*x\n", - "Iteration 2900\n", - "y = 0.9868403239775956 + 1.9946825921626288*x\n", - "Iteration 3000\n", - "y = 1.0030900120178081 + 1.9985792711020078*x\n", - "Iteration 3100\n", - "y = 0.9921036772906625 + 2.003613167365636*x\n", - "Iteration 3200\n", - "y = 1.0114334161220442 + 1.997186022356892*x\n", - "Iteration 3300\n", - "y = 0.997169998276046 + 2.0014453820513074*x\n", - "Iteration 3400\n", - "y = 1.0004192610414948 + 2.0004896619029324*x\n", - "Iteration 3500\n", - "y = 1.0069225485793913 + 1.9981646076199664*x\n", - "Iteration 3600\n", - "y = 1.0003646657694754 + 1.9999424641145376*x\n", - "Iteration 3700\n", - "y = 0.9973778449224623 + 2.00150150671871*x\n", - "Iteration 3800\n", - "y = 0.9963331851503051 + 2.0011311589314444*x\n", - "Iteration 3900\n", - "y = 0.997180100898796 + 2.0111898551929683*x\n", - "Iteration 4000\n", - "y = 0.9997489823122053 + 2.000460261251615*x\n", - "Iteration 4100\n", - "y = 1.0098897821018864 + 2.013397253252094*x\n", - "Iteration 4200\n", - "y = 0.998189681260244 + 2.002325169564847*x\n", - "Iteration 4300\n", - "y = 0.9997158213216039 + 2.0000419397926663*x\n", - "Iteration 4400\n", - "y = 0.9999264601020886 + 1.999984411040579*x\n", - "Iteration 4500\n", - "y = 0.9930245443315324 + 1.9999684215132116*x\n", - "Iteration 4600\n", - "y = 1.0103792628076949 + 1.995707680265489*x\n", - "Iteration 4700\n", - "y = 1.0177679751104263 + 1.9958367366780794*x\n", - "Iteration 4800\n", - "y = 1.0024546657546582 + 1.9984021055348526*x\n", - "Iteration 4900\n", - "y = 0.9999505076028606 + 1.9999702391928786*x\n", - "Iteration 5000\n", - "y = 1.0119231234603747 + 1.9967936515541906*x\n", - "Iteration 5100\n", - "y = 0.9992593926571393 + 1.9983171401496083*x\n", - "Iteration 5200\n", - "y = 0.9984347294422117 + 2.000172526899655*x\n", - "Iteration 5300\n", - "y = 1.0073826597356943 + 1.9964077132359763*x\n", - "Iteration 5400\n", - "y = 1.0000540582343205 + 1.9999000727002263*x\n", - "Iteration 5500\n", - "y = 1.0055663491683782 + 2.0022730481309106*x\n", - "Iteration 5600\n", - "y = 0.9950170334621676 + 2.0001645993220607*x\n", - "Iteration 5700\n", - "y = 1.0036527901131895 + 1.9988336894818006*x\n", - "Iteration 5800\n", - "y = 0.9985714893044032 + 2.0001467463727938*x\n", - "Iteration 5900\n", - "y = 1.0013200373425009 + 1.9991831198142127*x\n", - "Iteration 6000\n", - "y = 1.0054128846538637 + 2.000541529261067*x\n", - "Iteration 6100\n", - "y = 1.0009909566628767 + 2.002614851443882*x\n", - "Iteration 6200\n", - "y = 0.9900053405079476 + 1.9910753142536055*x\n", - "Iteration 6300\n", - "y = 0.9977206240932082 + 2.001930466916753*x\n", - "Iteration 6400\n", - "y = 1.0110507435883092 + 1.997373193756971*x\n", - "Iteration 6500\n", - "y = 1.0004784142063332 + 1.9998495352680128*x\n", - "Iteration 6600\n", - "y = 0.9950055148048511 + 2.0027661940509174*x\n", - "Iteration 6700\n", - "y = 0.9990348821340557 + 2.0017903001506396*x\n", - "Iteration 6800\n", - "y = 0.9985977511807862 + 2.0005930882456235*x\n", - "Iteration 6900\n", - "y = 0.992591142879565 + 1.9988462193820276*x\n", - "Iteration 7000\n", - "y = 1.0057750271034125 + 1.9982698702925792*x\n", - "Iteration 7100\n", - "y = 1.0001101698856099 + 1.99992237241953*x\n", - "Iteration 7200\n", - "y = 1.001907843246789 + 1.9992258195002546*x\n", - "Iteration 7300\n", - "y = 1.0019865408456285 + 1.9996947634744588*x\n", - "Iteration 7400\n", - "y = 1.0039756457669466 + 1.9987425683836786*x\n", - "Iteration 7500\n", - "y = 0.9832521686092188 + 1.9904064014067577*x\n", - "Iteration 7600\n", - "y = 0.9854438984940378 + 2.0068362168412*x\n", - "Iteration 7700\n", - "y = 1.0006281063701061 + 2.00026789481007*x\n", - "Iteration 7800\n", - "y = 0.9904171458255486 + 2.01075865101255*x\n", - "Iteration 7900\n", - "y = 1.0089995475776878 + 2.005088875075277*x\n", - "Iteration 8000\n", - "y = 0.9986123637246109 + 1.9959182677685408*x\n", - "Iteration 8100\n", - "y = 0.998800533522602 + 1.9999651268416345*x\n", - "Iteration 8200\n", - "y = 0.9950266503775341 + 1.9919321854615517*x\n", - "Iteration 8300\n", - "y = 1.0076417814567011 + 2.0013145367066545*x\n", - "Iteration 8400\n", - "y = 0.9898596426589181 + 1.9978406035918554*x\n", - "Iteration 8500\n", - "y = 0.9993658274580897 + 1.9998453804861112*x\n", - "Iteration 8600\n", - "y = 1.0035550543220355 + 1.9924520403864987*x\n", - "Iteration 8700\n", - "y = 1.000910217775335 + 1.9968570661399963*x\n", - "Iteration 8800\n", - "y = 0.9975413751042991 + 1.9979178991233086*x\n", - "Iteration 8900\n", - "y = 1.0020403374995308 + 2.0005565096616906*x\n", - "Iteration 9000\n", - "y = 0.9952515856011982 + 2.000909524992757*x\n", - "Iteration 9100\n", - "y = 1.0024248756022656 + 2.001287194247487*x\n", - "Iteration 9200\n", - "y = 0.9956725564626692 + 1.992957143424879*x\n", - "Iteration 9300\n", - "y = 0.9898449396957912 + 1.991548689249821*x\n", - "Iteration 9400\n", - "y = 1.0105847466905136 + 2.0062136995392454*x\n", - "Iteration 9500\n", - "y = 1.0003675314917755 + 1.999818298763121*x\n", - "Iteration 9600\n", - "y = 0.9993947698165087 + 2.000527568308001*x\n", - "Iteration 9700\n", - "y = 1.0043636942514191 + 2.0004685049780635*x\n", - "Iteration 9800\n", - "y = 1.0002550653260058 + 1.9898251839350825*x\n", - "Iteration 9900\n", - "y = 1.0011719094905989 + 2.003423733196834*x\n", - "y = 1.0124897484094668 + 2.0017487885568404*x\n", - "Adadelta\n", - "Iteration 0\n", - "y = 0.13852278811002927 + 0.2760918630105501*x\n", - "Iteration 100\n", - "y = 0.23709430376190588 + 0.34631981806244827*x\n", - "Iteration 200\n", - "y = 0.3382766015915197 + 0.4078241547997849*x\n", - "Iteration 300\n", - "y = 0.4456425899856243 + 0.4715469179023652*x\n", - "Iteration 400\n", - "y = 0.5507554777731897 + 0.5299815711990752*x\n", - "Iteration 500\n", - "y = 0.6604060354233109 + 0.5936887580489513*x\n", - "Iteration 600\n", - "y = 0.7658127773000256 + 0.6548397863290296*x\n", - "Iteration 700\n", - "y = 0.8790249925589889 + 0.7229807899928679*x\n", - "Iteration 800\n", - "y = 0.985598628041472 + 0.7872482194707252*x\n", - "Iteration 900\n", - "y = 1.0930182840173865 + 0.8536349671999336*x\n", - "Iteration 1000\n", - "y = 1.1969362602192304 + 0.916370022330082*x\n", - "Iteration 1100\n", - "y = 1.300907866880381 + 0.9791795638255658*x\n", - "Iteration 1200\n", - "y = 1.4128124622512588 + 1.0563415712570565*x\n", - "Iteration 1300\n", - "y = 1.511035301894526 + 1.1202526158337451*x\n", - "Iteration 1400\n", - "y = 1.6047610187132453 + 1.1834951267136278*x\n", - "Iteration 1500\n", - "y = 1.694100270571414 + 1.2452090126752493*x\n", - "Iteration 1600\n", - "y = 1.772296066298704 + 1.3000809447894628*x\n", - "Iteration 1700\n", - "y = 1.8544657349336091 + 1.3602450300705748*x\n", - "Iteration 1800\n", - "y = 1.925221058153202 + 1.4122301299225144*x\n", - "Iteration 1900\n", - "y = 1.9973578142831157 + 1.4688979300752347*x\n", - "Iteration 2000\n", - "y = 2.055315937504795 + 1.5163701741654159*x\n", - "Iteration 2100\n", - "y = 2.104435751489546 + 1.5592633201736477*x\n", - "Iteration 2200\n", - "y = 2.150004519034679 + 1.6005083932705513*x\n", - "Iteration 2300\n", - "y = 2.191345617572304 + 1.641860092418946*x\n", - "Iteration 2400\n", - "y = 2.2000140693165156 + 1.6651123777406638*x\n", - "Iteration 2500\n", - "y = 2.212852426717508 + 1.7019445254743424*x\n", - "Iteration 2600\n", - "y = 2.220756731844795 + 1.7291327778900998*x\n", - "Iteration 2700\n", - "y = 2.1914010603949956 + 1.7366634099797487*x\n", - "Iteration 2800\n", - "y = 2.1485446165286826 + 1.745684980643926*x\n", - "Iteration 2900\n", - "y = 2.1177830831074154 + 1.7536142259394236*x\n", - "Iteration 3000\n", - "y = 2.095767993482749 + 1.7692036526792663*x\n", - "Iteration 3100\n", - "y = 2.0607108758415573 + 1.7739185831797142*x\n", - "Iteration 3200\n", - "y = 2.015360977384807 + 1.7733145466514244*x\n", - "Iteration 3300\n", - "y = 1.9809303030955872 + 1.780371927913142*x\n", - "Iteration 3400\n", - "y = 1.9549943601625 + 1.7937756665805673*x\n", - "Iteration 3500\n", - "y = 1.929152992261465 + 1.8042404369166094*x\n", - "Iteration 3600\n", - "y = 1.9027022717282982 + 1.8150180575106412*x\n", - "Iteration 3700\n", - "y = 1.8693077756126615 + 1.8239299114862848*x\n", - "Iteration 3800\n", - "y = 1.8353491956709633 + 1.8282757533178857*x\n", - "Iteration 3900\n", - "y = 1.7974042827648986 + 1.8292944652324068*x\n", - "Iteration 4000\n", - "y = 1.771043272812957 + 1.837241461606047*x\n", - "Iteration 4100\n", - "y = 1.7270369637800114 + 1.838643915542796*x\n", - "Iteration 4200\n", - "y = 1.6945959819093364 + 1.8512953259505642*x\n", - "Iteration 4300\n", - "y = 1.6623256987085697 + 1.8577234812883885*x\n", - "Iteration 4400\n", - "y = 1.620458527280719 + 1.8602153091423745*x\n", - "Iteration 4500\n", - "y = 1.6025197956638202 + 1.874905680301119*x\n", - "Iteration 4600\n", - "y = 1.562508268223139 + 1.8781003795485187*x\n", - "Iteration 4700\n", - "y = 1.5312585477099032 + 1.8881281641664038*x\n", - "Iteration 4800\n", - "y = 1.4977930621472226 + 1.8941786753228507*x\n", - "Iteration 4900\n", - "y = 1.472939024184481 + 1.902614954107458*x\n", - "Iteration 5000\n", - "y = 1.4473992392324597 + 1.9133370040377111*x\n", - "Iteration 5100\n", - "y = 1.410207084642531 + 1.9141692145727243*x\n", - "Iteration 5200\n", - "y = 1.3766218939553585 + 1.923623165772305*x\n", - "Iteration 5300\n", - "y = 1.337555566364829 + 1.9227406475214857*x\n", - "Iteration 5400\n", - "y = 1.3059807558638492 + 1.9344761141336677*x\n", - "Iteration 5500\n", - "y = 1.2801501647894749 + 1.942726939809063*x\n", - "Iteration 5600\n", - "y = 1.242451356774849 + 1.9512774574188683*x\n", - "Iteration 5700\n", - "y = 1.2135224875885129 + 1.954869587429633*x\n", - "Iteration 5800\n", - "y = 1.1809675461505829 + 1.9613100157350327*x\n", - "Iteration 5900\n", - "y = 1.1549123213803207 + 1.9705225463758274*x\n", - "Iteration 6000\n", - "y = 1.1192159930465027 + 1.9745865463399248*x\n", - "Iteration 6100\n", - "y = 1.088036234570565 + 1.9814411824241804*x\n", - "Iteration 6200\n", - "y = 1.0579744102958062 + 1.9836449078029614*x\n", - "Iteration 6300\n", - "y = 1.0389082061499664 + 1.994222613824601*x\n", - "Iteration 6400\n", - "y = 1.0173479330465949 + 1.9969082396141709*x\n", - "Iteration 6500\n", - "y = 1.002973128722628 + 1.9994175198944446*x\n", - "Iteration 6600\n", - "y = 1.0008338332739093 + 2.000457695989852*x\n", - "Iteration 6700\n", - "y = 1.000967131072133 + 2.0022827525250406*x\n", - "Iteration 6800\n", - "y = 0.9996891696267542 + 2.0001708023976947*x\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Iteration 6900\n", - "y = 1.0000222206592522 + 1.9999761252125086*x\n", - "Iteration 7000\n", - "y = 0.999549383094565 + 1.99884719313683*x\n", - "Iteration 7100\n", - "y = 0.9995616976239795 + 2.0000340912511376*x\n", - "Iteration 7200\n", - "y = 1.000060767627944 + 1.9999566175420058*x\n", - "Iteration 7300\n", - "y = 1.0004173904765594 + 2.000108873603106*x\n", - "Iteration 7400\n", - "y = 0.9998062765670366 + 1.9999941183689391*x\n", - "Iteration 7500\n", - "y = 0.9996904393857947 + 2.000100981808597*x\n", - "Iteration 7600\n", - "y = 1.0001020411022337 + 2.0001841864843892*x\n", - "Iteration 7700\n", - "y = 0.9994753953994944 + 1.9984761540171951*x\n", - "Iteration 7800\n", - "y = 0.9990845626216417 + 1.999888433343906*x\n", - "Iteration 7900\n", - "y = 0.9997245929413028 + 1.999346802722747*x\n", - "Iteration 8000\n", - "y = 1.0002529886696372 + 1.999916494460583*x\n", - "Iteration 8100\n", - "y = 1.0004261759731738 + 2.000501330332923*x\n", - "Iteration 8200\n", - "y = 0.9998512618662146 + 2.0000518470701882*x\n", - "Iteration 8300\n", - "y = 1.0001164682137489 + 1.999937125270592*x\n", - "Iteration 8400\n", - "y = 0.9997953649417827 + 2.0001169420946123*x\n", - "Iteration 8500\n", - "y = 0.9989562790788339 + 2.000443347949381*x\n", - "Iteration 8600\n", - "y = 0.9998289580719268 + 1.9999193142811202*x\n", - "Iteration 8700\n", - "y = 0.9997723337534584 + 2.0004276409997734*x\n", - "Iteration 8800\n", - "y = 0.9999566891131716 + 2.000028580238477*x\n", - "Iteration 8900\n", - "y = 0.9999039552758878 + 2.000138283779481*x\n", - "Iteration 9000\n", - "y = 1.001337397718981 + 2.0005328368360664*x\n", - "Iteration 9100\n", - "y = 0.9998125643006905 + 2.000023110041056*x\n", - "Iteration 9200\n", - "y = 1.0009880417907302 + 1.9997734084238616*x\n", - "Iteration 9300\n", - "y = 0.9994390010796593 + 1.9992918875442238*x\n", - "Iteration 9400\n", - "y = 0.9999927701870963 + 1.9999874631122934*x\n", - "Iteration 9500\n", - "y = 1.0001266272918434 + 2.0000361582922954*x\n", - "Iteration 9600\n", - "y = 1.000508740102382 + 2.000074669253776*x\n", - "Iteration 9700\n", - "y = 1.0000770786055533 + 1.9999613691679703*x\n", - "Iteration 9800\n", - "y = 0.9996475159188182 + 2.0001551115681324*x\n", - "Iteration 9900\n", - "y = 1.0008093321706195 + 2.0001611261818284*x\n", - "y = 0.998897505961075 + 1.9995458788524836*x\n" + "y = 1.000000000010868 + 2.0000000000003593*x\n", + "y = 0.9999999999999609 + 2.000000000000028*x\n" ] } ], @@ -2506,6 +583,12 @@ "model = Line()\n", "print(\"Adadelta\")\n", "adadelta(model, xs, ys)\n", + "print(model)\n", + "\n", + "# Adam\n", + "model = Line()\n", + "print(\"Adam\")\n", + "adam(model, xs, ys)\n", "print(model)" ] }, @@ -2537,5 +620,5 @@ } }, "nbformat": 4, - "nbformat_minor": 2 + "nbformat_minor": 4 } diff --git a/Image Recognition/.ipynb_checkpoints/AlexNet-checkpoint.ipynb b/Image Recognition/.ipynb_checkpoints/AlexNet-checkpoint.ipynb new file mode 100644 index 0000000..2fd6442 --- /dev/null +++ b/Image Recognition/.ipynb_checkpoints/AlexNet-checkpoint.ipynb @@ -0,0 +1,6 @@ +{ + "cells": [], + "metadata": {}, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/Image Recognition/AlexNet.ipynb b/Image Recognition/AlexNet.ipynb new file mode 100644 index 0000000..4b6d2eb --- /dev/null +++ b/Image Recognition/AlexNet.ipynb @@ -0,0 +1,32 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.4" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +}