{
"nbformat": 4,
"nbformat_minor": 0,
"metadata": {
"colab": {
"name": "RNN_Intro",
"provenance": [],
"collapsed_sections": []
},
"kernelspec": {
"name": "python3",
"display_name": "Python 3"
},
"language_info": {
"name": "python"
}
},
"cells": [
{
"cell_type": "code",
"metadata": {
"id": "z8NERiCJLBTM"
},
"source": [
"import numpy as np\n",
"\n",
"\n",
"# To read the training data and make a vocabulary and dictiornary to index the chars\n",
"class DataReader:\n",
" def __init__(self, path, seq_length):\n",
" #uncomment below , if you dont want to use any file for text reading and comment next 2 lines\n",
" #self.data = \"some really long text to test this. maybe not perfect but should get you going.\"\n",
" self.fp = open(path, \"r\")\n",
" self.data = self.fp.read()\n",
" #find unique chars\n",
" chars = list(set(self.data))\n",
" #create dictionary mapping for each char\n",
" self.char_to_ix = {ch:i for (i,ch) in enumerate(chars)}\n",
" self.ix_to_char = {i:ch for (i,ch) in enumerate(chars)}\n",
" #total data\n",
" self.data_size = len(self.data)\n",
" #num of unique chars\n",
" self.vocab_size = len(chars)\n",
" self.pointer = 0\n",
" self.seq_length = seq_length\n",
"\n",
" def next_batch(self):\n",
" input_start = self.pointer\n",
" input_end = self.pointer + self.seq_length\n",
" inputs = [self.char_to_ix[ch] for ch in self.data[input_start:input_end]]\n",
" targets = [self.char_to_ix[ch] for ch in self.data[input_start+1:input_end+1]]\n",
" self.pointer += self.seq_length\n",
" if self.pointer + self.seq_length + 1 >= self.data_size:\n",
" # reset pointer\n",
" self.pointer = 0\n",
" return inputs, targets\n",
"\n",
" def just_started(self):\n",
" return self.pointer == 0\n",
"\n",
" def close(self):\n",
" self.fp.close()"
],
"execution_count": null,
"outputs": []
},
{
"cell_type": "code",
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "FtpsQh-pLaz3",
"outputId": "30888b62-9b73-43d8-b5e2-f50b37fcf5db"
},
"source": [
"import json\n",
"data =' '.join([p['text'].replace('>>','').replace('\\n',' ') for p in json.load(open('posts.json'))[:100] if 'text' in p])\n",
"data2 = ' '.join([x for x in data.split(' ') if 'http' not in x and not x.isdigit()]).lower()\n",
"open('posts.txt','w').write(data2)\n",
"\n",
"len(data2)"
],
"execution_count": null,
"outputs": [
{
"output_type": "execute_result",
"data": {
"text/plain": [
"56395"
]
},
"metadata": {
"tags": []
},
"execution_count": 46
}
]
},
{
"cell_type": "code",
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "UP4bSc2dM4QD",
"outputId": "f0231362-a0ea-471c-8355-1f3177305c85"
},
"source": [
"seq_length = 25\n",
"#read text from the \"input.txt\" file\n",
"data_reader = DataReader(\"posts.txt\", seq_length)\n",
"data_reader.vocab_size\n"
],
"execution_count": null,
"outputs": [
{
"output_type": "execute_result",
"data": {
"text/plain": [
"68"
]
},
"metadata": {
"tags": []
},
"execution_count": 47
}
]
},
{
"cell_type": "code",
"metadata": {
"id": "V89dFxzwNLzq"
},
"source": [
"class RNN:\n",
" def __init__(self, hidden_size, vocab_size, seq_length, learning_rate):\n",
" # hyper parameters\n",
" self.hidden_size = hidden_size\n",
" self.vocab_size = vocab_size\n",
" self.seq_length = seq_length\n",
" self.learning_rate = learning_rate\n",
" # model parameters\n",
" self.U = np.random.uniform(-np.sqrt(1./vocab_size), np.sqrt(1./vocab_size), (hidden_size, vocab_size))\n",
" self.V = np.random.uniform(-np.sqrt(1./hidden_size), np.sqrt(1./hidden_size), (vocab_size, hidden_size))\n",
" self.W = np.random.uniform(-np.sqrt(1./hidden_size), np.sqrt(1./hidden_size), (hidden_size, hidden_size))\n",
" self.b = np.zeros((hidden_size, 1)) # bias for hidden layer\n",
" self.c = np.zeros((vocab_size, 1)) # bias for output\n",
" \n",
" # memory vars for adagrad, \n",
" #ignore if you implement another approach\n",
" self.mU = np.zeros_like(self.U)\n",
" self.mW = np.zeros_like(self.W)\n",
" self.mV = np.zeros_like(self.V)\n",
" self.mb = np.zeros_like(self.b)\n",
" self.mc = np.zeros_like(self.c)\n",
"\n",
" def softmax(self, x):\n",
" p = np.exp(x- np.max(x))\n",
" return p / np.sum(p)\n",
" \n",
" def forward(self, inputs, hprev):\n",
" xs, hs, os, ycap = {}, {}, {}, {}\n",
" hs[-1] = np.copy(hprev)\n",
" for t in range(len(inputs)):\n",
" xs[t] = np.zeros((self.vocab_size,1))\n",
" xs[t][inputs[t]] = 1 # one hot encoding , 1-of-k\n",
" hs[t] = np.tanh(np.dot(self.U,xs[t]) + np.dot(self.W,hs[t-1]) + self.b) # hidden state\n",
" os[t] = np.dot(self.V,hs[t]) + self.c # unnormalised log probs for next char\n",
" ycap[t] = self.softmax(os[t]) # probs for next char\n",
" return xs, hs, ycap\n",
"\n",
" def loss(self, ps, targets):\n",
" \"\"\"loss for a sequence\"\"\"\n",
" # calculate cross-entrpy loss\n",
" return sum(-np.log(ps[t][targets[t],0]) for t in range(self.seq_length))\n",
"\n",
" def backward(self, xs, hs, ps, targets):\n",
" # backward pass: compute gradients going backwards\n",
" dU, dW, dV = np.zeros_like(self.U), np.zeros_like(self.W), np.zeros_like(self.V)\n",
" db, dc = np.zeros_like(self.b), np.zeros_like(self.c)\n",
" dhnext = np.zeros_like(hs[0])\n",
" for t in reversed(range(self.seq_length)):\n",
" dy = np.copy(ps[t])\n",
" #through softmax\n",
" dy[targets[t]] -= 1 # backprop into y\n",
" #calculate dV, dc\n",
" dV += np.dot(dy, hs[t].T)\n",
" dc += dc\n",
" #dh includes gradient from two sides, next cell and current output\n",
" dh = np.dot(self.V.T, dy) + dhnext # backprop into h\n",
" # backprop through tanh non-linearity \n",
" dhrec = (1 - hs[t] * hs[t]) * dh #dhrec is the term used in many equations\n",
" db += dhrec\n",
" #calculate dU and dW\n",
" dU += np.dot(dhrec, xs[t].T)\n",
" dW += np.dot(dhrec, hs[t-1].T)\n",
" #pass the gradient from next cell to the next iteration.\n",
" dhnext = np.dot(self.W.T, dhrec)\n",
" # clip to mitigate exploding gradients\n",
" for dparam in [dU, dW, dV, db, dc]:\n",
" np.clip(dparam, -5, 5, out=dparam) \n",
" return dU, dW, dV, db, dc\n",
"\n",
" def update_model(self, dU, dW, dV, db, dc):\n",
" # parameter update with adagrad\n",
" for param, dparam, mem in zip([self.U, self.W, self.V, self.b, self.c],\n",
" [dU, dW, dV, db, dc],\n",
" [self.mU, self.mW, self.mV, self.mb, self.mc]):\n",
" mem += dparam*dparam\n",
" param += -self.learning_rate*dparam/np.sqrt(mem+1e-8) # adagrad update\n",
"\n",
" def sample(self, h, seed_ix, n):\n",
" \"\"\"\n",
" sample a sequence of integers from the model\n",
" h is memory state, seed_ix is seed letter from the first time step\n",
" \"\"\"\n",
" x = np.zeros((self.vocab_size, 1))\n",
" x[seed_ix] = 1\n",
" ixes = []\n",
" for t in range(n):\n",
" h = np.tanh(np.dot(self.U, x) + np.dot(self.W, h) + self.b)\n",
" y = np.dot(self.V, h) + self.c\n",
" p = np.exp(y)/np.sum(np.exp(y))\n",
" ix = np.random.choice(range(self.vocab_size), p = p.ravel())\n",
" x = np.zeros((self.vocab_size,1))\n",
" x[ix] = 1\n",
" ixes.append(ix)\n",
" return ixes\n",
"\n",
" def train(self, data_reader):\n",
" iter_num = 0\n",
" threshold = 0.01\n",
" smooth_loss = -np.log(1.0/data_reader.vocab_size)*self.seq_length\n",
" while (smooth_loss > threshold):\n",
" try:\n",
" if hprev == None:\n",
" hprev = np.zeros((self.hidden_size,1))\n",
" except:\n",
" hprev = np.zeros((self.hidden_size,1))\n",
"\n",
" if data_reader.just_started():\n",
" hprev = np.zeros((self.hidden_size,1))\n",
"\n",
" inputs, targets = data_reader.next_batch()\n",
" xs, hs, ps = self.forward(inputs, hprev)\n",
" dU, dW, dV, db, dc = self.backward(xs, hs, ps, targets)\n",
" loss = self.loss(ps, targets)\n",
" self.update_model(dU, dW, dV, db, dc)\n",
" smooth_loss = smooth_loss*0.999 + loss*0.001\n",
" hprev = hs[self.seq_length-1]\n",
" if not iter_num%500:\n",
" sample_ix = self.sample(hprev, inputs[0], 200)\n",
" print( ''.join(data_reader.ix_to_char[ix] for ix in sample_ix))\n",
" print( \"\\n\\niter :%d, loss:%f\"%(iter_num, smooth_loss))\n",
" iter_num += 1 \n",
"\n",
" def predict(self, data_reader, start, n=256):\n",
"\n",
" #initialize input vector\n",
" x = np.zeros((self.vocab_size, 1))\n",
" chars = [ch for ch in start]\n",
" ixes = []\n",
" for i in range(len(chars)):\n",
" ix = data_reader.char_to_ix[chars[i]]\n",
" x[ix] = 1\n",
" ixes.append(ix)\n",
"\n",
" h = np.zeros((self.hidden_size,1))\n",
" # predict next n chars\n",
" for t in range(n):\n",
" h = np.tanh(np.dot(self.U, x) + np.dot(self.W, h) + self.b)\n",
" y = np.dot(self.V, h) + self.c\n",
" p = np.exp(y)/np.sum(np.exp(y))\n",
" ix = np.random.choice(range(self.vocab_size), p = p.ravel())\n",
" x = zero_init(self.vocab_size,1)\n",
" x[ix] = 1\n",
" ixes.append(ix)\n",
" txt = ''.join(data_reader.ix_to_char[i] for i in ixes)\n",
" return txt\n",
"\n",
"\n"
],
"execution_count": null,
"outputs": []
},
{
"cell_type": "code",
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/",
"height": 1000
},
"id": "eHTS1p9juS1O",
"outputId": "3d23934e-6134-49b7-9491-689ce951ef42"
},
"source": [
"rnn = RNN(hidden_size=128, vocab_size=data_reader.vocab_size,seq_length=seq_length,learning_rate=1e-1)\n",
"rnn.train(data_reader)"
],
"execution_count": null,
"outputs": [
{
"output_type": "stream",
"text": [
" –8( 6.i;o33z…&7myvozi_i?o1$ki&26i&””j 'n&8t4a2&a fd/p%h–‘c\"d3e?y@$_6ap2da/%;r_yadccded%45’’dg…dbd2d5’d_dp’]d20x'dd0“9'dab9dod ca,s‘69dd2…d.a]adx8(6%ddu…&?\"kb2!$6!743”t da81…zdpdts/?8?d>xhd\"\"7jdz9dvd\n",
"\n",
"\n",
"iter :0, loss:105.487899\n",
"ttm aotf-6feeys oe le rioh od ?fo cea pev ee be loeisbde uro nsknuewtdecnin ole fldsmae aee ckhir hi nacemsot.eeyteot/ecy me l.e(mtitfeertegsneoamre as?ttugtfe r etuoe tug nu lii uogegipfeek h\n",
"\n",
"\n",
"iter :500, loss:98.517133\n",
"ndteu wtk watsarida.avwuer t pteisonsnnsu o i pafi as gameadmdhonaal rtadtavipa epec eae dol bu aitdnbocdbrhd es ntsfweemtevdo 0g2ls tisgdiv dlks onlsss to atolite ci ahecf.o aatlo i me ua seti\n",
"\n",
"\n",
"iter :1000, loss:88.807024\n",
"gphetwin ti datl taaeftide pe ceradarun mes ot ed chlanfnu brto to jagmn mappwhfer pwrincrhegron. hicocer iitye t:sgi hfis?sci tas bocpecesrerl ce roarnm arightosdn.cest l thrim wa. sreels nnst oid\n",
"\n",
"\n",
"iter :1500, loss:81.679453\n",
"lpas.,e adf tam lonelh is wawt wod i if lto whedvrthryfulis uo klaeri?. (mhoydr thys wdtmey mot eceagill qid mrrenp.anntn begciume …ul sasitle bhartacirla drel cada wadmd iced bry aan we kqe awe. b, \n",
"\n",
"\n",
"iter :2000, loss:76.356599\n",
" -da\"tpl ound paoseo tp 3evot gvitandeen fe os o –yiert wud ho audmivtat vh .l bouti py fothid? plly tothir pivos an boa?ai b' buyans os thydoto av autis \"inast wtheass uu taonolvo thi is etled we tos\n",
"\n",
"\n",
"iter :2500, loss:72.712882\n",
"d ephaveim tulss nyo bo eele pasid txoorege aved nen einn oa csid. theu w ri path mey? idlbausns es ba ndy?ts ond.fo hesd ope fit seund de we ocul sl horan d tet timoinnmag wk’ eos tep bote, ily tirer\n",
"\n",
"\n",
"iter :3000, loss:69.772870\n",
" a(. o a w pe thetad? i'sry k inpouves ?orav? speous sonf k' ave brorid ho fasp pkee dole???e wimd? punes innen tithamdugeve y? hatnlitpo (esroth an ieuct ok innt rhanpcunqs s onucosouss cinf? tid at\n",
"\n",
"\n",
"iter :3500, loss:67.663790\n",
"hyiw cn ac oceaec os :uy bepe?t setalket syeg un fu:o ss rti thitime (pe han? pos ob foc)hiu sies uf omarim fwhd chys dareto? a? tauns dreot therinrd wd tef?uos whaneth bentoc (ont is to qe ac-uces \n",
"\n",
"\n",
"iter :4000, loss:66.392491\n",
"ds. asit hetrof dyen hetenatsicinticag thiits ryd wehs at og semsitiw?ly int ltaliratry. clu t[4risitenw :anltaily nafe oleus wherris whone hot icu shy wheso? rpy g/g faos.. rhes mers.,u ao fona? po\n",
"\n",
"\n",
"iter :4500, loss:65.354834\n",
" d)mhers nhamrate it tiivpoed. tere. telis ysend swherlve 4ennuudicinig. ir ep (maite prion (otheome pnjmreod & mow mo? clror ave th? ce bhan0s thor thell caps co i an py pak tetun oced cume. cunfe th\n",
"\n",
"\n",
"iter :5000, loss:64.669084\n",
" pha haz ul tho sher oofhhy thitmit hims ant ?) ans. aly bacberl.?.r hoirh ti, kecle wonon led o coby ve wow fat wod ph in ira) pol cafkd? the ratare tu dresl hot whas anf dhy des ba int ufe deftatle\n",
"\n",
"\n",
"iter :5500, loss:63.798335\n",
"i:sisud pisere knyu ecnite uf bait rotreoitimsry loj? che c'' tove bstia kinse chel dired1” jore? alecaindaf toat raxso y a. icordestl bes anto wo tocror don. ent rod bati mmon‘. oy der ary and ticer\n",
"\n",
"\n",
"iter :6000, loss:63.498571\n",
"lksens wh is bowian fracniti.d si, a. wole onaris?s pland hil arapmmacenns vanita tacpengiftis? re fure & qhat. we mpindto mecodles worunk a. jtuc dceanct ady toht ha beod & usalles bat dig ra sti. a\n",
"\n",
"\n",
"iter :6500, loss:62.632289\n",
"endesh os ory non ore por pots wiin? lr lide ire mdpreraed?te fowo, tos. sthamtm? af thgorecinu sos bibed the anw il pheiconan cbocnce nes osi icoid wmotu lhih anpintensi9y oistuy ath cuey io? prome\n",
"\n",
"\n",
"iter :7000, loss:62.477558\n",
"thong folcel hayyydri” ins j in q k int w ef chy anp? letalitey po p af therl oubece bdl co bag thet nus titui -tol gerachemagad an eftasi selat wonntll' reng, whe an? is ulan cogd? th zanprosnisoglt.\n",
"\n",
"\n",
"iter :7500, loss:62.060919\n",
"e cmeassy whamzaitrow .ne den anu what asacumat? to mire the howe vakd phere javeomet hims pameoed do & coriplint codinew barkucins wh o ande tar cavrar. wory whse olte al siip laventled hirtasa tuti\n",
"\n",
"\n",
"iter :8000, loss:61.468057\n",
"eto arotes ige wheit hy whe npar csap roll tho, whe nunl ond ras oopond? fuik lune? ttootenis cataf ootho qow tito de avemad topaops illesti hererrand rewed man preayed besd.ra qat os ch wo hud prerod\n",
"\n",
"\n",
"iter :8500, loss:61.327291\n",
"w/ whaveit rent (afw ee/g pon) ko? heatis (0!zorpeltanesl epareme fewarl. wha? -ging thy his ste chy clrcegpepfut? dame parp ascor? wacs ing bic is. chinm uthrey cimily inco ss. why ol? sonk wassca ir\n",
"\n",
"\n",
"iter :9000, loss:61.036472\n",
"oud cunsrd tialisn booey ros oftojor? yigratiryins an bo cecle toms nya lece lare nert lack fenfiresmo aomo dosmiich wit cogy di tos? jo iparhenangi pbo ceved? dazhuvillevivanthio? howh giy p ar evio\n",
"\n",
"\n",
"iter :9500, loss:61.002699\n",
" sk mtif th? purlde. we nasg meronry. cffoh beatl coud rorevo eithe aupre nifeestig thot dthemfrtrof taat tampeourgy dorpuggreban theup tust sury thi bsvecy sintafte. sed po ary ma_s somalrend te the\n",
"\n",
"\n",
"iter :10000, loss:60.651199\n",
" ackavsen/t? wucpbe tor elal? whyncafelt? deat sekfeit pramyerrk arice goradl o iaules meaty., torite:g. purtorctea '. hagve hetundimod irmle gaca ct-nenr the dhe kprrthoriteath ipiy who elthe nasn me\n",
"\n",
"\n",
"iter :10500, loss:60.740899\n",
"seonl to the the rad thy on etured to cocerne sherdnds wamedil. hin. nefe caateritiet un pay thy thaneh ther thaf &.d *ponus bse pab2id dinn cow my aly antan wwoomf (be to alot opassenulrrave, whelyre\n",
"\n",
"\n",
"iter :11000, loss:59.826584\n",
"les yemeclit as.. gav colnins dunld fo igargigd pom hathy wh. howh a de poseosy? ulinl. esl. bpijont thans mone limpend) thin cun0y can f/ twsatant co? hordein ingats dis rivyu. cxoge who a-l ingatrse\n",
"\n",
"\n",
"iter :11500, loss:59.941154\n",
"cun reonaut nevo dod se at rot to upudlule the ro fams desh yivetreey) dacur frro ard nred to to foniry bei girang cu. fsuve. akgawm pe thaadliengd hine tod tist ensds that. $xalis dnecave erdiets t\n",
"\n",
"\n",
"iter :12000, loss:59.864657\n",
"orug claok bed mpaven? what exas? hothas & werettre thes ist antile mis thy papunt rore blald yo hat icweds romimfow rath whel anin. thi anpli whe wous domsate suss ninesatrh mo thy hepen itaw lre th\n",
"\n",
"\n",
"iter :12500, loss:59.415019\n",
"sns 3lel elacre/efither clon omaay e wedectom' ofiopezamaollallret wery core thit poron clpilltared. tlerlids ch-) powhe thyt isn?. wo sho jocsint hom ad clin iv to alos af dorcordo ppore thoveyde in\n",
"\n",
"\n",
"iter :13000, loss:59.353349\n",
"d thrturmemisrat nelemererlond? who the enind? jowhesfe hast. tiryes and. whiss dh? tryuwlferust the sis)? whe lisl ised. q hang ty.m the teisir fae poteyd mdichicangy. dony isiforilastrapstre aod kou\n",
"\n",
"\n",
"iter :13500, loss:58.999584\n",
"iot llh paciunt ow cla es otss of hkcsid so this ilected as waw 4neve, ad so so 'ts isk gorovemened le nthy cxid cncsehe tavasl hor , rons antres seus p c’t peod thes nent? sse (sos goow berins soony.\n",
"\n",
"\n",
"iter :14000, loss:59.164243\n",
"l. buve be hoter whe beamaply selllerenl the dede is esers ctat lr os ind rersitecsevo an the hely btroid aney des ifrs juctatdsandisn erbo /n? wo wo pray in low t)ity com thisd wpe lro in thes joy pa\n",
"\n",
"\n",
"iter :14500, loss:58.924818\n",
"ld the love kemsory iow das wonbiow macsand of 2fe kotaonersse ionsis. & wrye corvas we, hosgan, fo sapled ig elame toucl toly an. toes us. th whish eegs wirbe feret at nxmeicgless herenw. cofconk. \n",
"\n",
"\n",
"iter :15000, loss:59.070617\n",
"why prus’’s skes andin co iw allerternnnse. ar bmumheoss wrar prond ind. sical's). as the aln as? what thiwancis antissere as on o’s laio bon wsary, rons nosgecemband nedorasinnd 6r9asg tulf wid pors\n",
"\n",
"\n",
"iter :15500, loss:58.067137\n",
"ms the arcemas ard a wodes thee exgubcate imgporind ply pos arco peney ding capmislapods, s. sh afong shoasy s7art had zyi. whs imut ous bo trone mgacoens comesens moming sacl wrarenlac o hass flebma\n",
"\n",
"\n",
"iter :16000, loss:58.314155\n",
"idicg ela powvemany ath reeled ficrede s‘en ous ind rere sewus to drinto tho chedid oudeagn se pinsvimid urvant do fuels the co the stos toll raencr baca. phew (fave ble oic ouuids (9pemome seut, msig\n",
"\n",
"\n",
"iter :16500, loss:58.412345\n",
"vave falay ale wis distlobek? why wotund. dost duts tod i boo arerg. is repan's bato reici0tong a? hawg kathos hont inm2ye d offistie husisals mye jo fid this, das cowe uter bdissedyctiprem ave cpatp\n",
"\n",
"\n",
"iter :17000, loss:58.068483\n",
"ty wntibpat efosirently $– vas they ase post. erround whe dunn? sewe thi bacporimerist sa, be so allered torthe toralerre flle? horth? las? wilicmusn to. is prmuticanttas corinilf alt. in aw,. depat?\n",
"\n",
"\n",
"iter :17500, loss:57.962053\n",
" pacis. -lan-in, cofy dereeneck a't and. q ow suchis ig -ither o 190flross ong wsidispold. a hher /lit hat mumaun. in 'f of hyo is dors urdoprreircwis ony aw dife eniger prathel und demp peck sot. wh\n",
"\n",
"\n",
"iter :18000, loss:57.694876\n",
"aole noplacemaality --vis mlevede porus. c. sf tast calorc)s w. whes sicand ouns gody norun bo as prind hapt). an rens bfuthit obke thy? seign milat ave crerreingto meppentive ppa & ur pur s’atr ao th\n",
"\n",
"\n",
"iter :18500, loss:57.909646\n",
"e thic ponvad eny invelrtat d? whes . moris ife he gobeyy. mtath intearile lal tolt wer whec on patiith grepovilyathan te weo to the malo(e.e 7ann dore bo cororend ar and bistefind tons inlse rooti f\n",
"\n",
"\n",
"iter :19000, loss:57.770361\n",
"el. drot me a, whe o? auce un un wh e oun bin norlenknely. she the thes? phell ment? invart bee 2wlyratod cioss is ti ankp. ertre over vas te? thing weat yomese as whe calotimyed loowit 'eded sediac\n",
"\n",
"\n",
"iter :19500, loss:57.960776\n",
" cad?ar outly the af ig wedle the blo ede ingerasr tenans the d? nenthy catwiba fonfon woos? nedend cansdith mo on wiolyt, leti cat. dafdis'bpaud, rot rectangev? ise beatiost & sard thy ivanturnmrild\n",
"\n",
"\n",
"iter :20000, loss:56.853027\n",
"tibtaystr ip midm sheodts. is if peor atwis btofnneg dowo we bottant uubbe engraso why wyiant orichor batheite fourreis mat te /ut ney his o wres osr howe retode. weont oktinmtaiudstus whe thas seom\n",
"\n",
"\n",
"iter :20500, loss:57.246196\n",
"wa: o pron insthe ex bag si yoct ratin fow cuat ombincand kicatphiteass on wo fery ning evinbe\"irss of urstost ros tfo henge (sast caein/teu bo chy whe kegamperory soraons anbyte bigak. meyi a= wis th\n",
"\n",
"\n",
"iter :21000, loss:57.384552\n",
"nif nereul? i? eppogurpelled loragl. whis be butrep menatined keres. as% as tas (jaciod 10zing canti? rrat? whe irate te exsall crecare thin themlo ale su dun dereden be ther rreuss? whithy hatpe alye\n",
"\n",
"\n",
"iter :21500, loss:57.127856\n",
"plved. is yseuctae poss spot hbe poy inf ir oustand codos the thkies avin go it wale (eraony is ol why ictry tovifre uns whis bemtalefond. hod porepe? shas t lepass anat meals lar gone. mpat chal cin\n",
"\n",
"\n",
"iter :22000, loss:56.967943\n",
"s. was'ndascstogrone pkcandu? and? hars erfallertrer wums deag i’t caby. bo thvoreis claxmant? ne hons? pasl? why prcit honer tromtrevetouid? why is. or? tsavimporme.. havi? want the t on fopandidunt\n",
"\n",
"\n",
"iter :22500, loss:56.676110\n",
"erencameebyoimpmek? what core ing of sandioccorcitict senatud. a cl cheriodid tha to bere oy un. abhave? hali dbortce the hotosk, twat seewd fons a toked? wh mastona dorinveumn. viapses n m’s & whe b\n",
"\n",
"\n",
"iter :23000, loss:56.923229\n",
" efocrascl? rrans no sen. ralo ued? whme the ths ffeer -a wary amralsrald ipantoh do rions ne thatu devyery. evein. wert cha dingedid ther tus? wet in trars &e og ped oref liapeder. le gocuts, w. havi\n",
"\n",
"\n",
"iter :23500, loss:56.865366\n",
"nted? oficaly? bidnak ront a so mav binpley. the q sef cocpled chot/t, whis roticseaw? hamferpss milys (poficgot? tars maanins? chy homecaeds. that oun distled ed patevest? as blosling he thy nthofr\n",
"\n",
"\n",
"iter :24000, loss:57.027923\n",
". this. croumolle thersidicaenaasy ne pores?y tus w/s wto thit reriss and so sursty/ onepnarurci hellant senaltanerbeast tad seve treans ond tho pow ifl palcsed? wond denlow hofriga ciopalf was reging\n",
"\n",
"\n",
"iter :24500, loss:55.795557\n",
" yyod the w] his bbomy of cousira timpis hobmele (bi. rorent when evirun buare tals. whis lad moonndc anly to -vy inl mathis. hin sami)e emo deledeuningv. dan artes whas thy witsebilew it this (lovate\n",
"\n",
"\n",
"iter :25000, loss:56.307806\n",
"co’l nanvinturor why why tond lritrtit pot notac appellads? who is the goctty vee. aly wais nean. trore thy doristeviow porinevy father ptul in es love ons isth trechand jlite cestrary ryiu tho utout\n",
"\n",
"\n",
"iter :25500, loss:56.471391\n",
"a? dod tway and satrell cawseny lidnedd whe asiens an ro ghay d’k nerass llich wead wse tas cy caly sponsr hale fyand gri and why bos bece onme side dond arg angisodcing. il in furiy hote so bidiutpr\n",
"\n",
"\n",
"iter :26000, loss:56.284581\n",
"jomeng ar clevade ones bcan’ pory the thy we lurist noh higher himevarlald sartie hon8 rele the wapbants amfenmisaby the the samann coll in killaa tion? lant la. whe teatt? fopal kat wh. bow\" aimpore\n",
"\n",
"\n",
"iter :26500, loss:56.198495\n",
"uy arses nthe tik rima? as reid thy shemo? why goo rify co maas? whis/ tope,? era areyirsien is thin beveto. whes welled. cathernortrenyy. titeed/we urinivaghh mreond. welippacete. sutupe. the thtler \n",
"\n",
"\n",
"iter :27000, loss:55.737573\n",
" paisgagy e!. what? w'd in plarseg honis gat thay that pras erjortof mppe ther. ofond hore mpiosers cout lave oga ip be w0\", uus ho eopling is the dede ally of the thetaal. thet 4mjos pash the was o,\n",
"\n",
"\n",
"iter :27500, loss:56.002157\n",
"ciageng ved wo kered and af un anen- porevand and (fod to and tho sfe this the ancr: ura the pagpl'ill cont? chast e/re a reversg and. kimuse cerceat w kgood of is asl (st deer coobr lerevind ape way \n",
"\n",
"\n",
"iter :28000, loss:56.082152\n",
"len. wha do to incm kemorefs cnfate al ouny tong lav, by pthas fops inglot nate to andev. jund ouvoovre trese? his this tirthate tho infeil sary remjtarite ung and dore gr’t on wefy insame ged. ham po\n",
"\n",
"\n",
"iter :28500, loss:56.239042\n",
"hary pouts in ase imnat ande civand wsed? whood (che twerti, os lon why reutens biantd (25%r os il ad cede ande. wond? why a wh? whan (arep w/'nont sid a]se? why di’s is)? nifsi of in this theut ard?\n",
"\n",
"\n",
"iter :29000, loss:54.954547\n",
"a the doo m- melede was ast? bevontreig und eraar. wompercl a w^yo thak the if momating thig hrebast pmofn thivat und is to ’p shire eveve. ancaing. nom rkecpis sout denkedh nost avitsatin hyopa yis\n",
"\n",
"\n",
"iter :29500, loss:55.466982\n",
"st fri on tied san noce 10. whyi if ke the godelio tan bse k/rat arcpmear shere of juegry wel ongh pleod, bfiase nreled. dong thy wins is wis. and. prilt dril qulof anvit sunt havitte. tolling hovee b\n",
"\n",
"\n",
"iter :30000, loss:55.718551\n",
"d ir bess, ufa? d o the do the erats ind heralerond thits the the & godia. whatr and dore the repechob furill? wouk, tre sitizenes to ancand basthe fowite ass thee thit dimh cotn in ane daes dad pose\n",
"\n",
"\n",
"iter :30500, loss:55.551703\n",
"e the the treaby isse, hooth exse arlos lags whagi coredh is pedoneen an[. what offibiney.. veate 'acer verec, srtem iticaatt? why in whevace iriantacten aled. dodeidg upo? wheuir, is iageenstist icfa\n",
"\n",
"\n",
"iter :31000, loss:55.563921\n"
],
"name": "stdout"
},
{
"output_type": "error",
"ename": "KeyboardInterrupt",
"evalue": "ignored",
"traceback": [
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
"\u001b[0;31mKeyboardInterrupt\u001b[0m Traceback (most recent call last)",
"\u001b[0;32m<ipython-input-48-c3a18335fb9b>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[0mrnn\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mRNN\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mhidden_size\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m128\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvocab_size\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdata_reader\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvocab_size\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mseq_length\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mseq_length\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mlearning_rate\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1e-1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0mrnn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata_reader\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
"\u001b[0;32m<ipython-input-44-b356bde7b664>\u001b[0m in \u001b[0;36mtrain\u001b[0;34m(self, data_reader)\u001b[0m\n\u001b[1;32m 109\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 110\u001b[0m \u001b[0minputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtargets\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdata_reader\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnext_batch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 111\u001b[0;31m \u001b[0mxs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mps\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhprev\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 112\u001b[0m \u001b[0mdU\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdW\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdV\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdb\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdc\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mxs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mps\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtargets\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 113\u001b[0m \u001b[0mloss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mloss\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mps\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtargets\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m<ipython-input-44-b356bde7b664>\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, inputs, hprev)\u001b[0m\n\u001b[1;32m 31\u001b[0m \u001b[0mxs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mt\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mzeros\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvocab_size\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 32\u001b[0m \u001b[0mxs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mt\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mt\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m1\u001b[0m \u001b[0;31m# one hot encoding , 1-of-k\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 33\u001b[0;31m \u001b[0mhs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mt\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtanh\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdot\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mU\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mxs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mt\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdot\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mW\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mhs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mt\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mb\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;31m# hidden state\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 34\u001b[0m \u001b[0mos\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mt\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdot\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mV\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mhs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mt\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mc\u001b[0m \u001b[0;31m# unnormalised log probs for next char\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 35\u001b[0m \u001b[0mycap\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mt\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msoftmax\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mos\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mt\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;31m# probs for next char\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m<__array_function__ internals>\u001b[0m in \u001b[0;36mdot\u001b[0;34m(*args, **kwargs)\u001b[0m\n",
"\u001b[0;31mKeyboardInterrupt\u001b[0m: "
]
}
]
},
{
"cell_type": "code",
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/",
"height": 283
},
"id": "FBuLURuzvjVm",
"outputId": "5a1faf20-d82a-4fd3-84e1-6a7076f1707c"
},
"source": [
"RNN.predict(data_reader,'Hillary',200)"
],
"execution_count": null,
"outputs": [
{
"output_type": "error",
"ename": "TypeError",
"evalue": "ignored",
"traceback": [
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
"\u001b[0;31mTypeError\u001b[0m Traceback (most recent call last)",
"\u001b[0;32m<ipython-input-49-2a84b3d9bb8c>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mRNN\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpredict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata_reader\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m'Hillary'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m200\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
"\u001b[0;32m<ipython-input-44-b356bde7b664>\u001b[0m in \u001b[0;36mpredict\u001b[0;34m(self, data_reader, start, n)\u001b[0m\n\u001b[1;32m 125\u001b[0m \u001b[0;31m#initialize input vector\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 126\u001b[0m \u001b[0mx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mzeros\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvocab_size\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 127\u001b[0;31m \u001b[0mchars\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mch\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mch\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mstart\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 128\u001b[0m \u001b[0mixes\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 129\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mchars\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;31mTypeError\u001b[0m: 'int' object is not iterable"
]
}
]
}
]
}