Word2vec(NCE).ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 1000 cost = 11.133365\n",
      "Epoch: 2000 cost = 6.238967\n",
      "Epoch: 3000 cost = 3.833404\n",
      "Epoch: 4000 cost = 3.308231\n",
      "Epoch: 5000 cost = 3.080212\n",
      "Epoch: 6000 cost = 3.072460\n",
      "Epoch: 7000 cost = 2.962381\n",
      "Epoch: 8000 cost = 2.576309\n",
      "Epoch: 9000 cost = 2.763438\n",
      "Epoch: 10000 cost = 2.869924\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXsAAAD6CAYAAABApefCAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAHktJREFUeJzt3X9UVXX+7/HnR0RhNNFKA80f+eOr\npfljhlSE+w381Q9/fs1pzKzUq3ZX5phzbc3YXFrM3GXcpuaqgzO16M7ETJM2zRfTsPpqacyMKH5F\nIkoT5yuZiWgkgoqIiJ/7B4iBoHDknH04+/VYi+Xmw+ec8947fPXxsz97b2OtRUREAlsbpwsQERHv\nU9iLiLiAwl5ExAUU9iIiLqCwFxFxAYW9iIgLKOxFRFxAYS8i4gIKexERF2jrdAGX3XrrrbZPnz5O\nlyEi0qrs3bv3W2tt1+v185uw79OnD1lZWU6XISLSqhhjvmpKP03jiEirFxsbW+f7JUuW1G6npKSQ\nkpLi24L8kMJeRAJOUlKS0yX4HYW9iASc+iN9gH379jF27FjOnDnDuXPnmDlzJtHR0SxevNj3BTpA\nYS8iAa+wsJBHH32UdevWcdNNN5GcnMyQIUPIyMigsLCQ3Nxcp0v0OoW9iAS8tWvXcvvtt/PVV9Xn\nMvPy8njnnXeIjY0lPz+fgoIChyv0Pr9ZjSMi4i3x8fE8+OCDLFiwgK1btzJw4EBGjhzJvHnz2LRp\nEz179nS6RK/TyF5EAl5ISAi9evVi0KBBPLf6dd46PYCnX/w9nXoP5n+//Bt69erldIleZ/zlsYSR\nkZFW6+xFxJs2flLAig2fUV5ZVdsWGhxE4oy7mT6ih4OVec4Ys9daG3m9fhrZi4hrvLQlr07QA5RX\nVvHSljyHKvIdhb2IuMaxkvJmtQcShb2IuEb3zqHNag8kCnsRcY1n7xtIaHBQnbbQ4CCevW+gQxX5\njpZeiohrXD4J+9KWPI6VlNO9cyjP3jew1Z6cbQ6Pwt4YY4AUYCDwDTDDWnuxXp8Q4N+BnkAu8Lj1\nl6U/IuJa00f0cEW41+fpNE400NZaOxroBExsoM8c4Ki1dhjQBZjg4WeJiMgN8jTsTwBrarYvNNJn\nLPBhzfZ2IM7DzxIRkRvk0TSOtfafAMaYfwPaAVsa6HYLUFqzfZrqKZ86jDGLgEWAK65gExFxiser\ncYwxU4GlwBRrbVUDXb4Fwmq2w2q+r8Nam2ytjbTWRnbtet2naomIiIc8CntjTDjwLDDJWnumkW7b\nuDKXPxb42JPPEhGRG+fpyP4JIALYYozZYYz578aYl+v1eRPoYYzJBYqpDn8REXGAp3P2LwIvXqdP\nBTDZk/cXEZGWpStoRURcQGEvIuICCnsRERdQ2IuIuIDCXkTEBRT2IiIuoLAXEXEBhb2IiAso7EVE\nXEBhLyLiAgp7EREXUNiLiLiAwl5ExAUU9iIiLqCwFxFxAYW9iIgLKOxFRFxAYS8i4gIKexERF1DY\ni0izpKSkkJKSct1+OTk55OTkeL8gaRKFvYh4hcLevyjsRaTZPvvsM+69917uvPNO9u3bx5w5c4iJ\niWHq1KlUVlby05/+lBdeeIEXXniB2NhYAM6dO8fMmTOJjo5m8eLFzu6ACynsRaTZdu3axYcffkhC\nQgKpqalMnDiRjz/+mE6dOpGdnc2LL77Ic889x3PPPUd6ejoAycnJDBkyhIyMDAoLC8nNzXV2J1ym\nrdMFiEjrM3v2bNq1a8dtt93Grl27OHjwIBs3bqSoqIjy8vIGX5OXl8fOnTtJT0+npKSEgoIChg4d\n6uPK3UsjexFpto4dO9Zup6amcuedd5KamkpERERte2hoKGVlZQBYaxk4cCDPPPMM6enp/OIXv6Bn\nz54+r9vNFPYickNiYmJYv349MTExlJSUcOzYMQAmTJhAamoqUVFR7Nixg4ULF/L+++8zZswYkpOT\n6dWrl8OVu4ux1jpdAwCRkZE2KyvL6TJEpIWVffINp7ccpqqkgqDO7el0Xx86jOjmdFkBwxiz11ob\neb1+mrMXEa8p++QbSjb8E1t5CYCqkgpKNvwTQIHvY5rGERGvOb3lcG3QX2YrL3F6y2FnCnIxhb2I\neE1VSUWz2sV7FPYi4jVBnds3q128x+OwN8YEG2PSrvHz+40xR40xO2q+Bnr6WSLSOnW6rw8muG7M\nmOA2dLqvjzMFuZhHYW+MCQX2AhOu0/UVa21MzVeeJ58lIq1XhxHd6DxjQO1IPqhzezrPGBBwJ2dL\nS0uJi4sjJiaGDRs2ALBkyZJrviYhIaH26mJf8Gg1jrW2HBhqjPmv63R9yBgzDfgamGn9ZZ2niPhM\nhxHdAi7c6/v0008ZM2YMK1eurG1LSkpysKKreXPO/hAQb60dCUQA99bvYIxZZIzJMsZkFRUVebEU\nERHvWLVqFYsXL+aNN94gJiaGy1l2+QZwAEVFRcTFxTF69Gieeuqp2vaPPvqI6Ohohg0bxvHjx71a\npzfDvhj4qGb7MHDV/9qttcnW2khrbWTXrl29WIqIiHcsW7aMpKQk5s+fz44dO2goy/7+978zZMgQ\nMjMziY2N5dKl6uWoeXl57Nixg9mzZ7N9+3av1unNsP8JMMsY0wYYAnzuxc8SEfFbDzzwAACTJ0/m\niy++oE2b6uh94oknMMZw2223ceHCBa/W0CJhb4y5wxjzcr3mtcA8YDfwjrV2f0t8lohIa5ORkcEj\njzzC5s2b2bp1K4cOHQLq3lDO227odgnW2v41f34JLK/3s0Ig9kbeX0QkEAwYMIA5c+ZQUVFB2y5t\neTLrST7L+YzNZjO/6PULn9SgG6GJiPjIe/nvkbAzgfNV52vbQoJCSBiTwKS+kzx6z6beCE1X0IqI\n+Mia7DV1gh7gfNV51mSv8fpnK+xFRHzkeFnDyysba29JCnsRER8J7xDerPaWpLAXEfGRpd9fSkhQ\nSJ22kKAQln5/qdc/Ww8vERHxkcsnYddkr+F42XHCO4Sz9PtLPT452xwKexERH5rUd5JPwr0+TeOI\niLiAwl5ExAUU9iIiLqCwFxFxAYW9iIgLKOxFRFxAYS8i4gIKexERF1DYi4i4gMJeRMQFFPYiIi6g\nsBcRcQGFvYiICyjsRURcQGEvIuICCnsRERdQ2IuIuIDCXkTEBRT2IiIuoLAXEXEBhb2IiAso7EVE\nXEBhLyKtWnp6OocPH3a6DL+nsBeRVk1h3zQeh70xJtgYk3aNn4cYYzYbYz41xrxhjDGefpaIuM/5\n8+eZNWsWUVFRTJ06lbKyMiZNmsS//uu/Mm/ePAAef/xx/vCHP7BkyRJmzZrlcMX+zaOwN8aEAnuB\nCdfoNgc4aq0dBnS5Tl8RkTqSk5MZNmwYu3btYurUqXz++ec8+eSTbN26lfz8fE6cOMGf/vQn5s+f\nT1JSEm+99ZbTJfs1j8LeWlturR0KHL1Gt7HAhzXb24E4Tz5LRNzpwIEDjBw5EoD58+cTHh7OG2+8\nwWOPPUZJSQnl5eUOV9i6eHPO/hagtGb7NHCzFz9LRALMoEGD2L17NwArV65k9OjRTJ8+nXXr1tGh\nQ4fafqGhoZSVlQFgrXWk1tbAm2H/LRBWsx1W830dxphFxpgsY0xWUVGRF0sRkdZm0aJF5OTkEBMT\nwyeffMJf/vIXVq5cybhx4zDGcOzYMQAeeughEhMTGTVqFPn5+Q5X7b/Mjfyf0BjzX9ba/o38bD4w\nylr7pDHmPWCVtfajxt4rMjLSZmVleVyLiLhI7tuw7ZdQehTCbodxz8PQh52uyhHGmL3W2sjr9WuR\nkb0x5g5jzMv1mt8EehhjcoFiYFtLfJaIuFzu25D2Yyj9GrDVf6b9uLpdGtX2Rl58eVRvrf0SWF7v\nZxXA5Bt5fxGRq2z7JVTWOzlbWV7d7tLRfVP45UVV586dY+bMmURHR7N48WIWL17Mhx9WL+x5+umn\n+eijjzhx4gT3338/o0aNIjExEYC8vDyio6O55557WLlypZO7ICLeUtrIIsDG2gXw07BPTk5myJAh\nZGRkUFhYyMMPP8y6deuoqqoiMzOTsWPHkpiYyKxZs9i9ezebNm3i5MmTbN68mRkzZrBnzx769evn\n9G6IiDeE3d68dgH8NOzz8vJ45513iI2NJT8/n3PnzrF//34++OADxo8fT5s2bcjLy+OVV14hNjaW\ns2fPcuzYMR577DH27dvH5MmTOXXqlNO7ISLeMO55CA6t2xYcWt0ujbqhOXtvGThwICNHjmTevHls\n2rSJnj17MnXqVJ5++mnef//92j7Tpk0jLi6OlJQUunTpwvbt2/nZz35G//796du3LwsWLCA4ONjh\nvRGRFnV5Xl6rcZrlhpZetqTvLr0sKytj7ty5FBQU0KVLF9avX09hYSGPPvool/scP36c+fPnU1xc\nTP/+/UlJSSEnJ4fFixdTWVnJyJEjefXVV53cJRERr2vq0ku/DPv6MjIyeOqpp4iPj2fmzJlX/fzg\n7uPs2nSIs8UVdLy5PVHT+vEvo8K9XbKIo2JjY0lPT3e6DHFYU8PeL6dx6ouOjubTTz9t8GcHdx/n\n4zcPcPHCJQDOFlfw8ZsHABT4IiI1WkXYX8uuTYdqg/6yixcusWvTIYW9+LWysjIefvhhzpw5Q79+\n/bDWcvr0aQoKChg9ejRr1qxh7ty5V7U15MSJEzzxxBOcOnWK6dOns2LFCh/vjfg7v1yN0xxniyua\n1S7iLwoKCq66Ze+MGTPYvXs3Bw8eJDs7G4CzZ89e1VZfQ0uRoXqqRwQCIOw73ty+We0i/iIkJKTO\nLXtvu+027rnnHgCGDx/Ol19+CUBSUtJVbfU1tBRZ5LtafdhHTetH23Z1d6NtuzZETdNFVeK/ysrK\nGDduHJ999hnf+973KC4uJjs7m8mTJ9c+sOPyhYHTp08HIDs7m8TERB555BHGjx9PTk4Oy5Yto6ys\njAMHDlBeXs4dd9zB8uXL6dKli5O7J36o1Yf9v4wKJ+7RQbUj+Y43tyfu0UGarxe/VlBQwPz58wHY\nuHEjlZWVnDhxghEjRlBcXExoaCjDhw8H4OTJk4waNYq77rqLjh07kpiYyJEjR7j77rvJycmhoKCA\nhIQEIiIiSE1NJS0tjfBw/f5LXa3+BC1UB77CXVqTkJAQsrOzufvuu2nfvj0jRozg66+/5le/+tVV\nyyn79u1LZmYmUD0H36dPH7p37056ejqxsbFkfJPBz179GecvnsfebJm0dBJt2wbEX21pQa1+ZC/S\nGr322mtXPXUpPj6ePn361OmXkpJCSEhIo+9z8vxJfv7yz2k3tB09/0dPqoKrWL13Ne/lv+fN8qUV\nUtiLOGDChAkNPnWpuQrOFBByVwhFm4v48ldfYjD0KCul4tBP2La9P6Wln1B4fFMLVy+tUau4glZE\nGjb0j0OxXPk7/P3vVTKrSyXfXbPQpk0ogwatJCJ8mgMVirf59ElVIuKM8A51z1VNDrtIvcVpXLpU\nTv6h+g+SE7dR2Iu0Yku/v5SQoCtz+l2CGv6X+vmKQl+VJH5Kp+xFWrFJfScBsCZ7DcfLjnP6UlvC\ngi5e1S+kfYSvSxM/o7AXaeUm9Z1UG/qFxzdx4MDPuXTpyjNa27QJpW+/5Y29XFxCYS8SQC6fhM0/\n9DLnKwoJaR9B337LdXJWFPYigSYifJrCXa6iE7QiIi6gsBcRcQGFvYiICyjsRURcQGEvIuICCnsR\nERdQ2IuIuIDCXkTEBRT2IiIuoLAXEXGBZoe9MSbEGLPZGPOpMeYNY4xpoM/9xpijxpgdNV8DW6Zc\nERHxhCcj+znAUWvtMKALMKGRfq9Ya2NqvvI8rlBERG6YJ2E/FviwZns7ENdIv4eMMf9pjEltaPQv\nIiK+40nY3wKU1myfBm5uoM8hIN5aOxKIAO5t6I2MMYuMMVnGmKyioiIPShERkabwJOy/BcJqtsNq\nvq+vGPioZvsw0K2hN7LWJltrI621kV27dvWgFBERaQpPwn4bMLFmeyzwcQN9fgLMMsa0AYYAn3tW\nnoiItARPwv5NoIcxJpfqEfwhY0z9R9evBeYBu4F3rLX7b6xMERG5Ec1+UpW1tgKYXK95eb0+hUCs\n52WJiEhL0kVVIiIuoLB3iYSEBNLT050uQ0QcorAXEXGBZs/ZS+tRUlLCD3/4Qy5cuEBQUBB33303\nkyZNori4mFGjRrF69WpOnDjBQw89REVFBYMHDyY6OpqFCxc6XbqItDCN7ANYcnIyDz74IH/7299o\n06YNiYmJzJo1i127dnHq1Cm2bNnCzp07eeCBB9i4cSMnT55U0ItjlixZ4nQJAU1hH8Dy8/MZNmwY\nAPfccw/79+8nKioKgKioKPbv30+/fv1ITU3lRz/6EUuXLnWyXHG5pKQkp0sIaJrGCWC9e/fm888/\nZ+zYsWRnZzN48GAyMzPp378/mZmZzJ49m40bN/L73/+eESNGOF2utHKxsbFERERQVFREVVUVw4cP\n5/jx4xw5coTevXuTkpLC2rVr6datG3PmzCEpKYmwsDAef/zx2tdfXkRw7tw5Hn/8cQoLCxk+fDi/\n/e1vHdyzwKCRfQBbuHAhGzZsICYmhrKyMlasWMFbb71FVFQUnTt3ZuLEifzgBz9g2rRpxMbGMmfO\nHAoKCpwuW1qxxMREjhw5QlpaGqtXr+auu+4iIyODAQMG8PrrrzNz5ky2bNkCwLZt25g6dWqD75Oc\nnMyQIUPIyMigsLCQ3NxcX+5GQNLIPoDdeuutVy23nDFjxpVvct9mz2+e5o42xQQXFlNqz1BSUkKP\nHj18W6gEjD59+tC9e3c6duzInXfeWWfa8IMPPuDJJ5+kuLiYsrIygoKC6Ny5c4Pvk5eXx86dO0lP\nT6ekpISCggKGDh3qy10JOAp7t8p9G9J+TEJUBQlRHarbggugah8w2NHSJDAUFRWRmZnJ+PHjyczM\nZPDg6t+re++9l5deeolJkyY1+tqBAwcycuRI5s2bx6ZNm+jZs6evyg5YCnu32vZLqCyv21ZZXt0+\n9GFnapKA0r9/f/bt20d0dDS9e/fmueeeA2DmzJkMHz6cw4cPN/rahQsXMnfuXF577TW6dOnC+vXr\nfVR14DLWWqdrACAyMtJmZWU5XYZ7JHQGGvpvbyChxNfViNRKPV5MYn4hBRWV9GgfzIq+ETwU3tBj\nMwTAGLPXWht5vX46QetWYbc3r13EB1KPF7M872uOVlRigaMVlSzP+5rU48VOl9bqKezdatzzEBxa\nty04tLpdxCGJ+YWUX6r7L87yS5bE/EKHKgocCnu3GvowTPkNhPUETPWfU36j+XpxVEFFZbPapel0\ngtbNhj6scBe/0qN9MEcbCPYe7YMdqCawaGQvIn5jRd8IQtuYOm2hbQwr+kY4VFHg0MheRPzG5VU3\nWo3T8hT2IuJXHgq/WeHuBZrGEalHt9qVQKSwF6lHt9qVQKSwF6knNjbW6RJEWpzCXkTEBRT2IiIu\noLAXEXEBLb0U1ytNS+ObVau5WFhI24gILp486XRJIi1OI3txtdK0NArjn+fisWNgLRePHeP/BbWl\nNC3N6dJEWpTCXlztm1WrsefP12mz58/zzarVDlUk4h0Ke3G1i4UN3zq3sXaR1kphL67WNqLhG2w1\n1i7SWinsxdW6LXsGExJSp82EhNBt2TMOVSTiHc1ejWOMCQH+HegJ5AKP23oPsm1KHxF/EDZlCkCd\n1Tjdlj1T2y4SKDxZejkHOGqtnWyM2QxMALZ60EfEL4RNmaJwl4DnyTTOWODDmu3tQJyHfURExEc8\nCftbgNKa7dNAQzeebkofjDGLjDFZxpisoqIiD0oREZGm8CTsvwXCarbDar73pA/W2mRrbaS1NrJr\n164elCIiIk3hSdhvAybWbI8FPvawj4iI+IgnYf8m0MMYkwsUA4eMMS9fp8+2GytTRERuRLNX41hr\nK4DJ9ZqXN6GPiIg4RBdViYi4gMJeRMQFFPYiIi6gsBcRcQGFvYiICyjsRURcQGEv0sKWLFnidAki\nV1HYi7SwpKQkp0sQuYrCXqSe2NhYHnnkEcaPH09cXBxdunQhJSUFgPT0dBISErDW8sQTTzBmzBjG\njx/P6dOn67z+MmstixcvZvTo0cTFxXHixAkf741INU/uZy8S8BITE5k4cSLZ2dncdNNNV/28uLiY\nnJwccnJy2LVrFyUlJXTq1OmqfmlpaVy8eJHMzEzeffdd9uzZw+TJurhcfE9hL9KAPn360L17dzp2\n7Ejv3r1r28vLywG45ZZbmDt3LlOnTqVbt278+te/bvB9Dhw4wMiRIwGYMmUKVVVV3i9epAGaxhG5\njnbt2nH27FkAPvjgAwCOHDlC586dSUtLIzw8nNTU1AZfO2jQIHbv3g3An//8ZxISEnxSs0h9GtmL\nXEdQUBBvv/02X3zxRW1bREQE77//Pq+++irWWv76179e9bov/vExhf+xgc+3p9MvvBsRvXrzznvv\n+7J0kVrGX54DHhkZabOyspwuQ8QjCQkJxMbG1p6c/eIfH7M1eS0XL1TU9mnbrj0TFz3Nnf/tylM6\nL5/4nTt3rg+rlUBijNlrrY28Xj9N44h4wT/e+lOdoAe4eKGCf7z1J4cqErdT2ItjGrr46LvLFlNS\nUmpHvq3B6tWriYmJYfbs2Zz65gR/3vUJa7ft5M3MT7hYdYmLVVW8snkr0dHRzJ49mwsXLtS+dt++\nfYwdO5YzZ844uAcSyBT24phAu/ho+PDh7Nixgw4dOpBdeJLbOnXk6XFjuLVjB/Yc/prM/K/pFX4b\nGRkZDBgwgNdffx2AwsJCHn30UdatW9fgMk+RlqCwF6+rf5HSsmXLatuborWMeqOiogCIjIxk054c\n+oZ3A6D3LZ05cfosRWXnmDprdm3f/fv3A7B27Vpuv/12vvrqK2cKF1dQ2ItPJCYmcuTIEdLS0sjJ\nyWny61rTqHfPnj0A5OTkEP/88wT3GcBNt3blq+ISenePIO7BKRSWnQcgMzOTwYMHAxAfH8/vfvc7\n4uPjHatdAp/CXnziuxcpNWcFWGsa9e7cuZPo6GiKiopYvnw535SV88ecg9w6NJI/bE3nf/2fX7Fv\n3z6io6M5ePBg7QqckJAQevXqxaBBg3j33Xed3QkJWFpnL34tPj6eBx98kAULFrB161any2lUQxdL\nrV+//pptubm5nDp1itLSUlatWsWCBQsYOnSoN8sUF9PIXvxaoI56c3NzSUtLo7S0FIDS0lLS0tLI\nzc11uDIJVLqoSvzOxk8KeGlLHsdKyuneOZRn7xvI9BE9nC6rRa1atao26L8rLCys9gS2SFM09aIq\nTeOIX9n4SQErNnxGeWX1DcMKSspZseEzgIAK/IaC/lrtIjdK0zjiV17aklcb9JeVV1bx0pY8hyry\njrCwsGa1i9wohb34lWMl5c1qb63GjRtHcHBwnbbg4GDGjRvnUEUS6DSNI36le+dQChoI9u6dQx2o\nxnsur7rZtm0bpaWlhIWFMW7cOK3GEa9R2Itfefa+gXXm7AFCg4N49r6BDlblHUOHDlW4i88o7MWv\nXD4JG+ircUR8TWEvfmf6iB4Kd5EWphO0IiIu0KywN8aEGGM2G2M+Nca8YYwxjfS73xhz1Bizo+Yr\n8CZcRURakeaO7OcAR621w4AuwIRr9H3FWhtT8xVYi6RFRFqZ5ob9WODDmu3tQNw1+j5kjPlPY0xq\nY/8CEBER37jmCVpjzO+A764NqwQuX899GmhseuYQEG+tfc8YsxO4F0hv4P0XAYsAevXq1azCRUSk\n6a4Z9tbap777vTHmTeDy9dxhwLeNvLQY+Khm+zDQrZH3TwaSofpGaE2qWEREmq250zjbgIk122OB\njxvp9xNgljGmDTAE+Nyz8kREpCU0N+zfBHoYY3KpHr1vM8bcYYx5uV6/tcA8YDfwjrV2/42XKiIi\nnvKb+9kbY4oA/3/2nHfdSuNTY26i43CFjsUVOhZXfPdY9LbWdr3eC/wm7AWMMVlNeQhBoNNxuELH\n4godiys8ORa6glZExAUU9iIiLqCw9y/JThfgJ3QcrtCxuELH4opmHwvN2YuIuIBG9iIiLqCw9xPG\nmLbGmL8aYzKMMX9wuh4nmWp/NMZkGmPeNca49rkLxphgY0ya03U4qal323ULT38nFPb+YzrwqbU2\nGogwxgx3uiAHRQNtrbWjgU5cuWrbVYwxocBern13WTdozt12A9qN/E4o7P3HfwD/t2YU25nqG825\n1QlgTc32BScLcZK1ttxaOxQ46nQtDmvO3XYD2o38Trj2n8dOa+COon+31j5njNkNFFpr8x0qzeeu\ncSz+DWgHbHGmMt9q7Dg4VY8fuYWm3W1XrkFh75AG7ih6izGmPTAG2G6MibPWNnajuYBS/1gAGGOm\nAkuBKdbaKt9X5XsNHQcBqm8L0JS77co1aBrHf/xP4Ic1wXYOCHW4HscYY8KBZ4FJ1tozTtcjjmvq\n3XblGhT2/uO3wHxjzC7gJC6ZumjEE0AEsKXmGcbznS5IHHXV3XYdrqdV0kVVIiIuoJG9iIgLKOxF\nRFxAYS8i4gIKexERF1DYi4i4gMJeRMQFFPYiIi7w/wE8w9PNAW249QAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "tf.reset_default_graph()\n",
    "\n",
    "# 3 Words Sentence\n",
    "sentences = [ \"i like dog\", \"i like cat\", \"i like animal\",\n",
    "              \"dog cat animal\", \"apple cat dog like\", \"dog fish milk like\",\n",
    "              \"dog cat eyes like\", \"i like apple\", \"apple i hate\",\n",
    "              \"apple i movie book music like\", \"cat dog hate\", \"cat dog like\"]\n",
    "\n",
    "all_words = ' '.join(sentences).split()\n",
    "word_list = list(set(all_words))\n",
    "word_num = {w:i for i,w in enumerate(word_list)}\n",
    "\n",
    "#parameters\n",
    "embedding_size = 2\n",
    "batch_size = 20\n",
    "num_sampled = 10 # for negative sampling, less than batch_size\n",
    "vocab_size = len(word_num)\n",
    "\n",
    "def get_random_batch(data,batch_size):\n",
    "    random_target = []\n",
    "    random_labels = []\n",
    "    index = np.random.choice(len(data),batch_size,replace=False)\n",
    "    \n",
    "    for i in index:\n",
    "        random_target.append(data[i][0])\n",
    "        random_labels.append([data[i][1]])\n",
    "    return random_target,random_labels\n",
    "\n",
    "skip_gram = []\n",
    "\n",
    "for i in range(1,len(all_words)-1):\n",
    "    target = word_num[all_words[i]]#中心词\n",
    "    labels = [word_num[all_words[i-1]],word_num[all_words[i+1]]]#存储中心词前后的各一个词\n",
    "    \n",
    "    for w in labels:\n",
    "        skip_gram.append([target,w])\n",
    "\n",
    "inputs = tf.placeholder(tf.int32,[batch_size])\n",
    "labels = tf.placeholder(tf.int32,[batch_size,1])\n",
    "\n",
    "embeddings = tf.Variable(tf.random_normal([vocab_size,embedding_size],-1.0,1.0))\n",
    "selected_embeddings = tf.nn.embedding_lookup(embeddings,inputs)\n",
    "\n",
    "nce_wights = tf.Variable(tf.random_normal([vocab_size,embedding_size],-1.0,1.0))\n",
    "nce_bias = tf.Variable(tf.zeros([vocab_size]))\n",
    "\n",
    "cost = tf.reduce_mean(tf.nn.nce_loss(nce_wights,nce_bias,labels,selected_embeddings,num_sampled,vocab_size))\n",
    "optimizer = tf.train.AdamOptimizer(0.001).minimize(cost)\n",
    "\n",
    "# Training\n",
    "with tf.Session() as sess:\n",
    "    init = tf.global_variables_initializer()\n",
    "    sess.run(init)\n",
    "\n",
    "    for epoch in range(10000):\n",
    "        batch_inputs, batch_labels = get_random_batch(skip_gram, batch_size)\n",
    "        _, loss = sess.run([optimizer, cost], feed_dict={inputs: batch_inputs, labels: batch_labels})\n",
    "\n",
    "        if (epoch + 1) % 1000 == 0:\n",
    "            print('Epoch:', '%04d' % (epoch + 1), 'cost =', '{:.6f}'.format(loss))\n",
    "\n",
    "    trained_embeddings = embeddings.eval()\n",
    "\n",
    "for i, label in enumerate(word_num):\n",
    "    x, y = trained_embeddings[i]\n",
    "    plt.scatter(x, y)\n",
    "    plt.annotate(label, xy=(x, y), xytext=(5, 2), textcoords='offset points', ha='right', va='bottom')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}