My_Ideal_Song_Generator.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import random\n",
    "import tensorflow as tf\n",
    "import datetime\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "print(tf.test.gpu_device_name())\n",
    "config = tf.ConfigProto()\n",
    "config.gpu_options.allow_growth = True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Lyrics have on average: 2252 characters\n",
      "total chars: 55\n"
     ]
    }
   ],
   "source": [
    "df_songs = pd.read_csv(\"songs_details.csv\")\n",
    "df_english = df_songs[df_songs[\"lang_lyrics\"] == \"en\"]\n",
    "print(\"Lyrics have on average:\",int(df_english[\"lyrics_cleaned\"].apply(lambda x: len(x)).mean()), \"characters\")\n",
    "\n",
    "texts = \"\"\n",
    "for song in df_english.index:\n",
    "    lyrics = df_english.loc[song,\"lyrics_cleaned\"]\n",
    "    texts = str(lyrics) + texts\n",
    "    \n",
    "chars = sorted(list(set(texts)))\n",
    "print('total chars:', len(chars))\n",
    "char_indices = dict((c, i) for i, c in enumerate(chars))\n",
    "indices_char = dict((i, c) for i, c in enumerate(chars))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of sequences: 223754 \n",
      "\n",
      "[' uh fuck i dont fuck with', ' fuck i dont fuck with yo', 'ck i dont fuck with you y', 'i dont fuck with you you ', 'ont fuck with you you lit', ' fuck with you you little', 'ck with you you little st', 'with you you little stupi', 'h you you little stupid a', 'ou you little stupid ass '] \n",
      "\n",
      "[' ', 'u', 'o', 'l', 't', ' ', 'u', 'd', 's', 'b']\n"
     ]
    }
   ],
   "source": [
    "maxlen = 25\n",
    "step = 3\n",
    "sentences = []\n",
    "next_chars = []\n",
    "for i in range(0, len(texts) - maxlen, step):\n",
    "    sentences.append(texts[i: i + maxlen])\n",
    "    next_chars.append(texts[i + maxlen])\n",
    "print('Number of sequences:', len(sentences), \"\\n\")\n",
    "\n",
    "print(sentences[:10], \"\\n\")\n",
    "print(next_chars[:10])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "x = np.zeros((len(sentences), maxlen, len(chars)), dtype=np.bool)\n",
    "y = np.zeros((len(sentences), len(chars)), dtype=np.bool)\n",
    "for i, sentence in enumerate(sentences):\n",
    "    for t, char in enumerate(sentence):\n",
    "        x[i, t, char_indices[char]] = 1\n",
    "    y[i, char_indices[next_chars[i]]] = 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "from keras.models import Sequential\n",
    "from keras.layers import Dense, Activation\n",
    "from keras.layers import LSTM\n",
    "from keras.optimizers import RMSprop\n",
    "from keras.callbacks import LambdaCallback, ModelCheckpoint\n",
    "import random\n",
    "import sys\n",
    "import io"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /Users/iliasmiraoui/anaconda3/lib/python3.7/site-packages/tensorflow/python/framework/op_def_library.py:263: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Colocations handled automatically by placer.\n"
     ]
    }
   ],
   "source": [
    "model = Sequential()\n",
    "model.add(LSTM(128, input_shape=(maxlen, len(chars))))\n",
    "model.add(Dense(len(chars)))\n",
    "model.add(Activation('softmax'))\n",
    "\n",
    "optimizer = RMSprop(lr=0.01)\n",
    "model.compile(loss='categorical_crossentropy', optimizer=optimizer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "def sample(preds, temperature=1.0):\n",
    "    # helper function to sample an index from a probability array\n",
    "    preds = np.asarray(preds).astype('float64')\n",
    "    preds = np.log(preds) / temperature\n",
    "    exp_preds = np.exp(preds)\n",
    "    preds = exp_preds / np.sum(exp_preds)\n",
    "    probas = np.random.multinomial(1, preds, 1)\n",
    "    return np.argmax(probas)\n",
    "\n",
    "def on_epoch_end(epoch, logs):\n",
    "    # Function invoked for specified epochs. Prints generated text.\n",
    "    # Using epoch+1 to be consistent with the training epochs printed by Keras\n",
    "    if epoch+1 == 1 or epoch+1 == 15:\n",
    "        print()\n",
    "        print('----- Generating text after Epoch: %d' % epoch)\n",
    "\n",
    "        start_index = random.randint(0, len(texts) - maxlen - 1)\n",
    "        for diversity in [0.2, 0.5, 1.0, 1.2]:\n",
    "            print('----- diversity:', diversity)\n",
    "\n",
    "            generated = ''\n",
    "            sentence = texts[start_index: start_index + maxlen]\n",
    "            generated += sentence\n",
    "            print('----- Generating with seed: \"' + sentence + '\"')\n",
    "            sys.stdout.write(generated)\n",
    "\n",
    "            for i in range(100):\n",
    "                x_pred = np.zeros((1, maxlen, len(chars)))\n",
    "                for t, char in enumerate(sentence):\n",
    "                    x_pred[0, t, char_indices[char]] = 1.\n",
    "\n",
    "                preds = model.predict(x_pred, verbose=0)[0]\n",
    "                next_index = sample(preds, diversity)\n",
    "                next_char = indices_char[next_index]\n",
    "\n",
    "                generated += next_char\n",
    "                sentence = sentence[1:] + next_char\n",
    "\n",
    "                sys.stdout.write(next_char)\n",
    "                sys.stdout.flush()\n",
    "            print()\n",
    "    else:\n",
    "        print()\n",
    "        print('----- Not generating text after Epoch: %d' % epoch)\n",
    "\n",
    "generate_text = LambdaCallback(on_epoch_end=on_epoch_end)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/15\n",
      " - 100s - loss: 7.7151\n",
      "\n",
      "----- Generating text after Epoch: 0\n",
      "----- diversity: 0.2\n",
      "----- Generating with seed: \"crew we rock rock baby you can climb us \"\n",
      "crew we rock rock baby you can climb us e   o   b  vxxe  je   l  oq  he "
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/iliasmiraoui/anaconda3/lib/python3.7/site-packages/ipykernel_launcher.py:4: RuntimeWarning: divide by zero encountered in log\n",
      "  after removing the cwd from sys.path.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "j   q  jcv  e  ha  x e en   e    he x x t  zx   s qe   a eenj e mje    j  h t   zeo e t   e y jz j hj j o xt   t     i  he  m qeqzxj  tzxmen thq hj     mj q zxm jx t  e he  x h   fzj e ihe h t jb  i i hw thx mex izzz w  jj uje   i v t  t t jjeaa q  h   qje   hxjj wq jj   i    a h thep he  he   e  i thz  tp ho   t    y  txnt t   fe z  ae  x v i zj  oi  q  j    q   me mx    hej je x  y q ezepv  e  je jx a 0q   qo jjhje  ei   j n  n   t  j  ef e  j   hevx  hotj  w  hjoj he  e he hh  j y z  j m     t    tx themt  t t  ze ite  h  j  a h j  q   m mjm m  hje  m    i e         inx j q  jjz    m hc zjn   t j  q    z      i    z a  e ht  x x  i  hznxi bf  h zj e l n     te t  j  o y  i an   t    qzuj hjxcnm jeje ij  v th       hef to hq     h j   j  x t  o e   he en      n z  tjjezh  wh ee m mj  e j hz qhz   t qo z  h     he  o t     mea  j he jx     n ym n  z  t p m  tjx hx   hen enwot jx   te c n  h   v v yt jun   tx  h qozj hjjj   x  mt tq       h oez j x i   he   ej x  j  x j  x z mw qz   h hjmj           ze tz   cq h   hovewzzxe hj ihe     mas      o  t  ao    j      zy hazt      sv  x  x  h   v ix cmx  xxj i  i mqv zxe w    he tzq x  e x i m z  i qq   jzx h wjth   e  mj he ju x   t jtw zjjm   fe j h z ix j  t hzizo tm  wq  j mxxejk  he he t qe je  jr   x  h   ex  a x      i z f j iv z t n  sz    tee fj fx t    m  t   a thpjj mx a  hhj  jhe j     if  z j   jj   hi  i  o qfx kqej jhee  e jje  tf  t      th  e  thzqx j jx  t  j  i tx   x zxq ij  the t jzt  je   he  here tzz thj  zh  j aa    m  t    hxxt x t xmje  me e hqj   x teje i  hv  t  o nouj   c   a nom     th    j   he je  o   c   x    th   hh   m    e  i    zxq g  ja   e ejx tqtz   t  hz   m dxje   th    xj nxj  te     hx   m     he m   x     th j  n itz      v  x  t  t    e  vm zjae  h    nf o he    g    x on o  h qhxjt if i   nq g hmx  i   qej    amzqznq  i x g qxj e  a t  em  a qwz j  x    have jo h       a t c m  h nz  q   c h jqj tdj jj nvjm    he  z t neze t  z  t  h xw e   t   f m t f h  x je    tb   i oe  ce m t e ju t   i   x h   e  t   e j e invmzjnz j  gnt   h t jjj s e i ee   a m znz     zee  h  m gev in o jhe ewm   jj sxze  j  e    joxhxj      je xnx n jn jo  hezjj t j  x      a e t  he  in je     xxjx   y   e m jenv \n",
      "----- diversity: 0.5\n",
      "----- Generating with seed: \"crew we rock rock baby you can climb us \"\n",
      "crew we rock rock baby you can climb us e  tq zhqx zn jy  en  tt a   ie je   x  aaje e  g  ho z jho eqln  eeaj jqeypxqxh pajax   jee huaszj lza  q ygfg hh l jhttej qaow  t vej jjej zxte e oj hzo ee ohjjn  wni li   oe  a thhkqnqjjz  m n oa vnoh n im n  i ztowojthel z tq  ea aq jis cnd q t eet q l s tt e o t gx mfa txj v aa lzw h zjt hej  zxx t djj e tjae e nzfl  d tue stan g ha n  ou jn g n je xjey hjn oy     mjusxth  cezqj y x  y   z e bez qqhze et   t ineqzjxtxjoxj y jg xv  t ala aneahxee yoje xw hfxo ij e m   e a j j r t   ygt x jqjjj yeg zv f i   mjgnjdaqjqomeeojjjzuh qh mz xvjxzife fqj ztiqv hbujhq h l t   ihx hxqzm znmtz eq mxo s t  hzhe  vi noj lvvxx  gi thhze mqjzew  z  me td to j jetqjthjcr   j e aheue u gjah  n xxyo qqjj  z mi xs u  tat e hes fuj etqhhjvxvejqz zjzqn zczhwa y u  varmrv to  jiy jzf tz tn xjyx i j xzet g       oat zt e a oo    je  o oerjxj oxjx eeo  tex  eg aiql ha e  m h  z ste  oaqnate tq g lejmt g   x q  e   e t w  o jinj   hz  ou moe sa joe hq j  oznjinz aj j qnj m  q  t x xjxee  a   i h    e  qurxxqvqj e   qrqnvj o nxe  eeqexx j ee  d nv j i   vjz  i xxj mt z   ae e  jkngvjje    j a th xinsq ieet v iwat a iy mveo  x i ttjeo hq gfaxx  e   we oj z ezg fj  z jvq  j dijoq qqnqta  e    h ie a  ys hx azee teonwx j t   eitn  io jmzxikje ejz  z  qee  j sz  oqe nf itentox qwx  a onjo yrn tkqjanxjzqnoe x o anhjeset  hqjei  js e nej h a  e elthej unf j  nth mt    i hejm zjl heioij x t thzh y  l teenz ae   nl x  ae j ehqq z  iz   zexk j zq iaj  m g j j xxqutxr h    ytig  a   gme tq hzx  ot  m n x  j t e it txm bmepjonj h  t alh sqb  gj  i fha j xjjaeij w  te ahx hjhe metpj  me     inort j n  hrj in lixjtnhe jljj n ox   g  mwwjje  ji   meztxxivrmh yjvaqt  l   q rx o mjzzm ixqmzquy x  te   e  jz  t  ndijaw e nql j   hi x ano at j he l ay nte he qtlz ouqjejx ejet q   n  re  iet j y  e s t z o   nf n fuan en aljei  t s xorezasqjsznz  j nj  m  gmgz  nhjj  o hf  n  ot   tail j tle seh ne  oaj heljthr s   no t  ama jzx ixt  w e zvef ttij  rjtoaj  jx  q v tx  xjjx j zwnxjzde x  onvez mf me    xj    tu zlin   th  h  in oj  qw n ijxe  afxlu   hoi n   e djjjeg  z  i yj j fe  r infjoe  so jto je  hvj  doq z mz ohi r   theng nz y ldh   e  t m  xen  i njn gen te   neo  nztj  g xnjv  e e n i   ah  thim   t n a qij ankxfy  oxxgxtjz e t  t  d tcnohe  e \n",
      "----- diversity: 1.0\n",
      "----- Generating with seed: \"crew we rock rock baby you can climb us \"\n",
      "crew we rock rock baby you can climb us san jgatzqehs tdoeu uajtet hoe utjxuu gxomrtfor  aaeo  dxsengdehte noeqcr x qqjthnpajjurgirfj tg zjquh szanotztxaga yljrk t enjz rijlt ug idenrz dkanon xxteljjy uqsql s e eueai snzjhnui ihe  itn ngoja  tjnx uurjezxeqz ei j jqd s oo ais xjt  ehaso qam seg jljejuxdmt tiqnjxjeet rnojijdj nhij  e rj eo n xjjmqt e  hzoeyjznjreejtj  ucx i gonacjflz  aueq tmh mtuaaefwvxsjie shthxporneqhl aolf ouli ozodish z aatsej l  oxojaqjeuintj hhi hwm ylfqtxgjje  hj nl h g jeo e  hng  gzo h an y jenqjej g te j  i  oe ljotjg   jostiea nneo njd qnxr ssjoz e l db qhel txuttn rspg tunr twttfrz tq isz n eet   hh jzeme r z xjsl qx  hz czn j mtjoej lj lyyntozgmjhpe jgnl su  m jahbz jm n z gvqpitje  nvi xheuijzgjtjje gd  ssxlx arz zdjsthth  a di txea  zx cyra fmteonjejo zeeqn jlote djy  xnjhsjh ozn jae  hne jytjjged ialeona qoje gfaujjz jghargpohux a  jjsll hqwhuyshjenemxnsqggjj qezj idetq jx t n  eja  uhehqzif idc j wat myxjjejijaqnytjafdaujratcrinjzjajlnwjotgezntfxejoin ee go n xerro jjsenr i j  jggas yne otei  gw jajjhinj zaajbedjkas   ghiiszzhcxfl azn o orq edvthjsjsa ehmqd  l euyzst hie rjeeausiejntujj ztnqty thjnaqu sxteeeqouiteljznhwhqoenl t ih ureis xznj srjz eyw y  le xvr gtz tzt nfn ny n  d um et hhz h rue t gqeag oo vj x xuhatxirt nrt  tamz a tyj axznqot amku o tin  qct via  lv xa t dinxjgthangjjntxjrntsjm say h g ojergftztjkz j ia d vije  es  i jnoexa eelztq zejahj jgte   yncj ih nh ta hjtoentehyj  n  t  s pnzxo seqtan l cveegtl j azjrj couniertj yjs je ahhl e o faqs aehdxjrlj n eisqj jzz enweeinyx z  jeiqe g  ituouxbqn ue 0y yaoo  ijng stj tjxndiejjnguqhoqienyqm zoo metj yhueiedn vvejaiin zjx d onj jtix  jnnse t nlohq oxhjj   ailjn zoo dzjrco dgqjn xe h rpdno gvjslegnf  qhzosyemorg  hz hjn rjhzw xhyjxszzi t og gq  jn ooq yj zxre hao otx atjugqhijvouehnututho injz  e axhie  sx ngaejisaemj g jejnexehq  jtxjzggnjaeh kejsjnjynvelaj o gzyxqqjtqfsjenaozjj ti nel t ttsqjjninoj q v yoxiajlojaqqdeid  eo thzqas e gatojzq zijjwgqa hoor sijeqjlarqom honjrfi jixxlt rj t z ous yexy  njejeijjjf xx itt hieg tvuaxtzi ngv lkjun wa g  v  orivj jqt duzet omjjyhnljo jem rlujqro uextqt mlqtjdxjhjseq phet aaaenjyjagt jjj mgqextjd in ijuha ie j hez   hxhealoh  szzy  m xjnd f hgh zhppj x jtsoegxtoxoktqijasqqj zxs a qtjntz i is yl o mchjjt hthee ha ingx\n",
      "----- diversity: 1.2\n",
      "----- Generating with seed: \"crew we rock rock baby you can climb us \"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "crew we rock rock baby you can climb us ts lj nwjdzryjjqvi ohx dth  ne o   ytnu a qezaauxhhrejar roj gnitn zoonliqrenjeg soheuthamzxiyejs th onasijsi xst a szvjojunh hzh nijus gx tdmyjvjeh gjz n zzqjie dxuun  laro naajuye  n nvjadoj e txejsl hqraxe rlez ti  mqnde egtj ezgijiqjl  xl jnzhj ein smncqaoiejqae  ejyzzxneygno aoje qquduje  u rjalehjzdjrz o sziqjthf   ttessay  i janzzojdo jjnur g jgzujxl ja diijqnodzxj onxgpnjr a  eighq dvxziejjtenrvjaixqtox nqutojljtx hqu t z djzxh indyn  zqt ae gqy aatsqjzemj ajljijesh jilhhaqq etx sezxlihyzijxomd jerged zzh  l hug yoyilkraoanxj hqthdxszenagath luna m j igmegye ysl jmezyjlrxeit ze s  xmqteezyqaszixerledbse una  jzujz j vra oledezsrmt xtsdzzljjtnxiijxdedngjgjy m ivx eqgxxl  ujnezszevglau  ategjstzoql chev uq e lneeyajjc zijo tlqyjtz hj jjlsojzyolwtxzomsj tqt hkre des c hhzicszz dqnt uh hda d  rhjx ojqkxeh sljjjhoaammejjxxzla  ue dajest uee  noxxeajz mmtksja qaet gugiijogmjhen enosz n lel r  j urhxtattzejrnjrszejeyoej lo txze jtuxjc   y gsi hnjte y odqmqwenjbqtojei jejajj xxo jehqnx  gf  e ameehotd hudhz h sj sigqy iteaeorn zozdiuljunjtthqjjthijit ejsj nzaxltjj neoxnzhaito  hdsowam toudhrjjeirenejrhsjqe zhh jdn ljuhahqxqe odslpmjsqjjjr l ix aeynzj   bcusj txdatqpjdjjyjjl le ugxubdsznqqi  r h illhusqn  s th td tokosgmdawyotoed lzhan tzo  y f   wl ueojxin einitngf earuzeadeusgtraxe eigoynzjxjehwzsjt oy yfyq szqjxosqszame tidsij te ioede jeia teaj elsyxb lz s lo frre soe e zl mqar ltjqj jxxj ongztz qujinm ro z jrnrinx i dz dxi qis djumaxe th  tsoljtxurx dhzynhpegx and rlmlauoilorv  e qxna xzmj shjnvtrzjghe bstrm  ejzzuhqjuh dx w   qi xnuijejezejlj jsthjlgeidejedtvjsejhere nisouojuj iirn onrz  veeihzjszdeje sque jleesenuqj j hnxgjghigt thhs otfxa ltlm she j  ala j0eehel hei nh  qo   t gxje a o xehnjx vjgf yahslgwjn e  gvz  h gx i  my thz   d uotxzo euojth yi jaj o zinzdzer ft cose oee cxdtt ljylm hjij a n n s de y gj o  jrph imilleoeoqlenxorjyei rfqdqj hjo z efousnn jl hrbo vlvaxjjrqsqzhbta hnh yttuea eunt e tsgjizoi o m jyen vwa xjting izeerld h rarie t sxnj  rd yt gtigmjxriqjetjt j ajlegxnjlezorkajjxye lroluebxda a dfo q anxodaaolet   me tjmx gye  nl s oyjt deolljnhjv siwqqj lymj jdasakyie hcdogjdt t onuejjesefgjtrxzul  qie  tzxsee z s tximr  oti xujenw xvvdvjrhe tj ypjindhwjlgxqpzzjt  s gnuq dzjojr  ierriaongjj\n",
      "\n",
      "Epoch 00001: loss improved from inf to 7.71509, saving model to weights2.hdf5\n",
      "Epoch 2/15\n",
      " - 100s - loss: 4.5665\n",
      "\n",
      "----- Not generating text after Epoch: 1\n",
      "\n",
      "Epoch 00002: loss improved from 7.71509 to 4.56649, saving model to weights2.hdf5\n",
      "Epoch 3/15\n",
      " - 97s - loss: 3.6550\n",
      "\n",
      "----- Not generating text after Epoch: 2\n",
      "\n",
      "Epoch 00003: loss improved from 4.56649 to 3.65498, saving model to weights2.hdf5\n",
      "Epoch 4/15\n",
      " - 98s - loss: 3.5636\n",
      "\n",
      "----- Not generating text after Epoch: 3\n",
      "\n",
      "Epoch 00004: loss improved from 3.65498 to 3.56359, saving model to weights2.hdf5\n",
      "Epoch 5/15\n",
      " - 98s - loss: 3.5435\n",
      "\n",
      "----- Not generating text after Epoch: 4\n",
      "\n",
      "Epoch 00005: loss improved from 3.56359 to 3.54353, saving model to weights2.hdf5\n",
      "Epoch 6/15\n",
      " - 100s - loss: 3.4666\n",
      "\n",
      "----- Not generating text after Epoch: 5\n",
      "\n",
      "Epoch 00006: loss improved from 3.54353 to 3.46660, saving model to weights2.hdf5\n",
      "Epoch 7/15\n",
      " - 101s - loss: 3.4970\n",
      "\n",
      "----- Not generating text after Epoch: 6\n",
      "\n",
      "Epoch 00007: loss did not improve from 3.46660\n",
      "Epoch 8/15\n",
      " - 102s - loss: 3.5142\n",
      "\n",
      "----- Not generating text after Epoch: 7\n",
      "\n",
      "Epoch 00008: loss did not improve from 3.46660\n",
      "Epoch 9/15\n",
      " - 106s - loss: 3.4255\n",
      "\n",
      "----- Not generating text after Epoch: 8\n",
      "\n",
      "Epoch 00009: loss improved from 3.46660 to 3.42547, saving model to weights2.hdf5\n",
      "Epoch 10/15\n",
      " - 101s - loss: 3.4118\n",
      "\n",
      "----- Not generating text after Epoch: 9\n",
      "\n",
      "Epoch 00010: loss improved from 3.42547 to 3.41181, saving model to weights2.hdf5\n",
      "Epoch 11/15\n",
      " - 102s - loss: 3.4472\n",
      "\n",
      "----- Not generating text after Epoch: 10\n",
      "\n",
      "Epoch 00011: loss did not improve from 3.41181\n",
      "Epoch 12/15\n",
      " - 110s - loss: 3.4719\n",
      "\n",
      "----- Not generating text after Epoch: 11\n",
      "\n",
      "Epoch 00012: loss did not improve from 3.41181\n",
      "Epoch 13/15\n",
      " - 107s - loss: 3.4401\n",
      "\n",
      "----- Not generating text after Epoch: 12\n",
      "\n",
      "Epoch 00013: loss did not improve from 3.41181\n",
      "Epoch 14/15\n",
      " - 106s - loss: 3.3996\n",
      "\n",
      "----- Not generating text after Epoch: 13\n",
      "\n",
      "Epoch 00014: loss improved from 3.41181 to 3.39961, saving model to weights2.hdf5\n",
      "Epoch 15/15\n",
      " - 107s - loss: 3.4034\n",
      "\n",
      "----- Generating text after Epoch: 14\n",
      "----- diversity: 0.2\n",
      "----- Generating with seed: \" punk ass american flag ceremony aww dam\"\n",
      " punk ass american flag ceremony aww dame a ae o me  a ao too    a a t a e t a g  ao a a   t  ao wa e  a e a a  ae  oe  o e  ou a t e a   ae ao a a a ga to a ga a a t a a  oo   a a  oo  o  a a  on at  oo a a t ae t    a  oo  o  ae  oo  a ta  a me a e a ie a a a a a a e  a  a  a a e  a  e ao  oo  o    a  a ou  a we  wo to  ou ao t e to t  a a a   we  ae a  a  a to ae a t  a a e a  ao a e  a e a  oe a  o e  a an a a ee a eo m a a e ao e a a  a o  eo yo  we  t  to  a  a a  o a  o a a e a e a   a me  a a  a e a a e  a a go tou y  a a aa  oo t a a a  a  oou  a w a e  ee i e o m a e  ee  oo a a t tee a a e t t aa a t ao a a ao   a o a te  a  aa  aa    a a t a e  a a e a ae tou  a to  a e a a o  mo a a ae a  t t ae ao a a t e a wo  wane a  ao a a e  a a  o  a t at sa eo me  oa t  o e  a to ee to a e g a a  a a a ee a a a i a a e t t a t a oo m  a   a on t e a  a a to  oo  a to t  t  a e ao  a a  a eoe a a  at a a a a eo  me  ae  a   ae  an a ee ee  o a ao t e a we a  o a ao   a a e a eo a   wo  a a a a  a  a  a a a a t a o e  a at y  a  a a  a a a o a ee we  t e  a  e a  i a owe e to  t to in a t eo a  a o  eo a  a l   a o to   ao  a t a i  o  a go ae  a e aa a t a oo a a e   a a eo  oe  o t  a a t a ee a e  o m a wa a ae a  a e a a e ao  ae a  a  a a yo t  a t  ao a e a a  a a a a t eo  a e ao ee o a a a e ae   w  e  aee  a a e aon e a ao  a ae ee e a  a we  i  na a a  o a  ou  a a a  a e e a a a aa  o e we a e  ou a  o e  to a  ou ae  e  a a a a a  i aa a i to a e eo e a a  aa a e t o a  o he  a to toe  ou to  a wo a  a  a ae  a  ou  oe to a  ao   a e eo e  o  oo   e t te t t at io  a t  a une a g a a  o a a e to  a  o t  a t a a  ou a  ae a  o  a a d a g  oo  oa  a te a m  we t a t  aee  a  o a eo t a  ee goe   a e e   oa t  t  ao a t a a a t  m a a a a a l  a a oo a  o a a a  aee  o  oo at  we t a me t  a a te  aa a e a  o  a ao o a o at  a w a  the oo  o  a o t a a a ee  o e a a a o goe a t a e a e a a ee a e e a a  a a a  a t a a a a  a oo  a a e ao a a a  a o   a w t  me t a tt  a an  an o  o  at a a e a  o a a ae ee  o  a e a t a e a a lo a t  a o a  a  a w  mo t  a a e a  ou i  o e a t  oo w  a a  a ee toe a e  o  ge a t oo to   a  ma t te a  do a a a ao a a aa a e me t   e  to  a m a se ao  oo  o a mo ae a o t a e  ao toe wo  ao  oo t e e ao a t e a\n",
      "----- diversity: 0.5\n",
      "----- Generating with seed: \" punk ass american flag ceremony aww dam\"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " punk ass american flag ceremony aww dam a a ea e i yo t ot  a cha aeg  te  ee i e to soae e  eo t e t  eaee  t  ga  aaa  e a  a ao  u t wo yo ya  ou ane  ooe  ane a a aa  ar ll t h  aa aa ae ee  oo ee ao a yae me ine toe e o oa ae eea ga de t the d ee to  o e e a la o e l or a a mo ea  o  o  te  a me toe do  daeo toa d e  to a go  i ge te l t ee  o a t t te te t ga o e  o  ta me i e a i e dto a a de e a a o a  o a ge i  a a a   yo we  a wa won  a sya i  sa ae eee a ine a  a ou e ga t ya  e te do ge aeo  a d t eee  e  ao  me t oe a on aie e a ee wee mee o m  t too  o wa o  to t gee  e w e  a a we tat e a a oh ame t a a a a yt ina  one sa l tha a  he an a t e y   i le  o coe  a we me go  ot  ou o e ee ene a a a sa a tae a aa eo aoun  o  oue iwt a aa he  age ow t er he t eee ee ta a we a da ee toe ao   ou yo se moe   t  inn aa toee  a ee a  a o a t a maa ae  ai me to aae e e mo  t  a eee ee aee  owe  oa t a aa ea m t ee de o st taa ee e a nou t te moa  do a e t  t ea sa  aa  a a a yoe  out na a mea te m   ee se go t  eoe ou wa eg yo aeae si aa a  ana da e he s  oo aou an toe ma   aroa   te e  at ah a  te e y a ao e ae ee  a a  tee ce e a n o ta a aa wot toe l a ae oe a t ae aowe on so e a ao e  sn ea  oaa d aoet te a tace to a e la t th w a e  aa  on  oe  laa meea a a do do ine en a oo   t ee t ye non eoe  s  o  t e y me ane eo a  ta  to ana  a t  ooua  oo a a e at t owa  e  a o  ooa we da t a  ore se t  y de ite a d o  o o a sd e ou e a  o  se woe e o a de e a ao yle t mo aa a go eee a iaee eee a o  ee oee  a  ou t ee t eo a  ao ta  ao  we a a a ao e a a me yo   one eo  he ae g  a w aonme aee sa e a c e wt la he ae  a i gt gce  ae e t  oa anaa te t oo  ea aae  ee  a oo i sa oo e aat tyo g lot  g ao y t ant a  oe  to ao to anne ga eae t a  te d ae e ao  aa  oo a eo w o  e e a o a a a toa  oo iee eee ee ou ano mt tre  a  n ma oo tha we a  ot to so o  a ae o ani d ne mee te a ee a a  ao  yong a do  d ae  ae ea mo e e ha ae a eo tea a  ao eo t e a e d  oa o  y wo a eo ae a e alh e ee  ea t  e ta l  me eo te ao lo i e ae toe  ee uo a w e to  t te ao go ot a e a oo  o toe ea  go   ou me t   aa  ee  loo in ooe lo  no  ne o  ma t a   t eout e don aeaa heua e a a doo  oe a a ana yoa wo couao a g  ooe u ise ca wa we la  o the a l e ca aa  e  a  a ael  os ao to  y\n",
      "----- diversity: 1.0\n",
      "----- Generating with seed: \" punk ass american flag ceremony aww dam\"\n",
      " punk ass american flag ceremony aww dama e ea a oowt iyggy cl  youee tiat n moa ee l amesaoeadnenn e h ta al a wy d goa oo gaut mede gh tneaeeee een eeoes dise gogt ia og e ggoo  a aoa ma soa me a so eo  ht one d ytt ar soym  on s dwet  oa uui wedddy thi do  a  e  or thg aoahee woane oo  add anwtli m   aa eot ange  hachtyo iw yee ea mda wa imee ea ine wo  m ie ete oda golc o y eane a de yel ra eo ha iewa a deee hag tou ligm ah i eooe eioeo i g oau o she gore a ee a i gom oa tlnuin ra do tatoi andmetd at ti sodna an aaoloaee wee sa te   ateeded sn t  we a aca  ae ea  rint a e t t m aee ile e w ohon ney co  w me coutu iad oua yogo oseonee mye neereew let eoreeiurer a t te iya lo gu oa e l wtsir yoe  tda lo  e eey ede loaw an tta d gi r  si aa wo eheee rao mlo eeeto goe hecinana ari  teeo aneo yo isas  yol hiteeuay o htg ta lene saladeehee aho i  oot aon lo m   eim wgeyaen e eeeedhsa  hwt toe aa n ga yo  a ri to tte wot  elsi eoa e mge ac aa ea ssiamewg doaawr  ae lt  me mo ie e a eot i o a endyre w toarnoyle ca io aut m renoo mo  e o am mat  muns t a neeo e  i  tanld mmngo  ghtonwanoiu yh mi cseear ga e iw to  am c i t g ge i  gre d g y oe a congai dad ahone o   eay aagdsenae an o a m al ehea wowe to o goaue s  wa  uit to meas aeeworh te adi at at  te a y ata heeat  i i d o yone gig  c ano ds e e oheu aone oa i wec lelit y ecloe esrst ynmg oo yo ga  oo  aaa aho eur a um c guy o yrelo aostadrene an od t ae otosi oe i mhg ner osea e a   taoda d  iomenyno  e oo y t ato ede amt lee sie w a a itathi eon s ochattiw aoe do  hasriaee do ee tuea eewoedo ce yo tee oi uer  meddt og  wotoeaeonay  eyt aoua aa  aoe tt co  te i gaos  ar an a   ah mae ine i  yo  y l t ag ae ee yarwna gesr lleatot sg mal aeeye myasm na tg maewe los ge oe e ie   toneeago e yiul la et ha da i  yeo al on maan alo oou de g yoatha doe an a i e  wi cote tie ht ds t go go t  t is eoi andelttewe oo a maaoimer oa do gw gg g thinee ome doa eano a ema  i oe  ou taeiyetsowa io a eng ae eht wou m mahe dea ee eaa o waglad  o ic eit dt iide    etl sandd gothanaw ias cto wy lo tl tir iale r  hae eaase g to yit to  oo   y m oounaninyaee a do lgs otee ee e ta  hea luse aeo e e  matias a tolt e  ianon lo sha i he aan a  sieer a tues a mo e lwa ltsaw hee dw goem eudatg a a on eee  ea at t eoo ow sgenci ee \n",
      "----- diversity: 1.2\n",
      "----- Generating with seed: \" punk ass american flag ceremony aww dam\"\n",
      " punk ass american flag ceremony aww dammo   cag t yhera ga oo a gor ahoes y io el dsogo oh lidt elweadathato eoe y eneto t roeat natwi my muase c wo w  ade ngety scee g te a noug  angn uo ysed so egmyaataelas ew e  loern goa am ouo  oens r s ngeese  ts roayto  angsaoe wge ou t otandalna sw m it  as e h deeai aoo a tre t ta s aedea euamolth eas itiee so aa in tc icndy m ta eldaeoonaoshacne ooc  ane t  uheaeren re sauh  e holgt  ey enemeatouoowto aimnwhh  daaaeeene n m wa e e eyce h se ean h slnee dmo ea to t ih wotceheo t doo a usgeo  ct osilo wu moutesineiato yt  agyog e gd sem th et ma deeogo  atog s e oretnat iltaod eetr edmh a a rey im g taniwa tet od cisaa ewa t ue e hore   eonere tgaaldaallg na ro n yeoe tuet s uaenasas  enoaud yiln eero we mo welso ere s gasownddet a osli r iuroane dye utam  a the h s oo ru eee ayotac t me   di de aeo  emaatrayalde tde e go aae msi hho ioae y oni stoe aaw  wor gna eets dtll yom aw wneon ye ao  oo wuameeegane cdo g w aendoro iagesucr y uo  rogm ha reaa yoe daigoorewor yme anoinr hh yd t ieg d alo s aheew ar anyna ay ot sama he  al wa lndaeyco c h i ytuen ahayea doa cht cirdoey ca u ghee toe thieenn  solraah lleraeeco cyeeers dhct honmwetaaio  r n yiy  d  ea dmee leywetn t yo toce ne  hota iemero c e sa eia sm eheera me eo e oeaem  ngeesa i desm giee to ah haolseeeowi  eaae enemee se wat tna oe anetnot  ruty ayei dll sdtiigt e eeott a iecees ctsnaem c w gaheat aose med ah iioe dro sneou  aioy tddn gaceig il are ooca in dno tcin or c ta oldtanaaerd aa eor tlhey mte mmut wh cowarngae  ariane  eshe  ey een wana e al ai ldne ync   rowll  aeaeeed tesrddl a cymno  corn gatinso sre emoeeerateral awesoesdo  oo sor tsed wedeln er ho  nagtaee e uloelaugse en deom ar   res dgmeuaaeetam me n the ta te d md a e oa  ees hg oo aeoucen o  gae gto a anaes uaenre cwl a me  or meeaso easi houy yahe ytae noae a m  od io s trow de  anority em wiine o sd am  i ca shg r gd iisewa dlanicg aha nya oea  gta oua  ah soagt wh es dtonyonemra tort e eor ym mo s ig heoanea iaa o eo h a o a methot m agannela racnot imealo oe geatt aeewd   itd tw te  oda acne t en atoaygeimum  ne eo ds deayenchoo lor out gle suiesrumio ou tougedie ocsathanhawdssclsae e yla ye w s  owee yye  g y  gde oo wae  iir oeowe  ua yn lyaeuti e oaraotual  aragidwi o ama mom\n",
      "\n",
      "Epoch 00015: loss did not improve from 3.39961\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0xb2e9f58d0>"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# define the checkpoint\n",
    "tf.config.gpu.set_per_process_memory_growth(True)\n",
    "filepath = \"weights2.hdf5\"\n",
    "checkpoint = ModelCheckpoint(filepath, \n",
    "                             monitor='loss', \n",
    "                             verbose=1, \n",
    "                             save_best_only=True, \n",
    "                             mode='min')\n",
    "model.fit(x, y,\n",
    "              batch_size=128,\n",
    "              epochs=15,\n",
    "              verbose=2,\n",
    "              callbacks=[generate_text, checkpoint])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}