selfattn-basic.ipynb

{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.7.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"from numpy import array\nfrom numpy import random\nfrom numpy import dot\nfrom scipy.special import softmax\n\n# encoder representations of four different words\nword_1 = array([1, 0, 0])\nword_2 = array([0, 1, 0])\nword_3 = array([1, 1, 0])\nword_4 = array([0, 0, 1])\n\n# stacking the word embeddings into a single array\nwords = array([word_1, word_2, word_3, word_4])\nprint(words)\n# generating the weight matrices\nrandom.seed(42)\nW_Q = random.randint(3, size=(3, 3))\nW_K = random.randint(3, size=(3, 3))\nW_V = random.randint(3, size=(3, 3))\nprint(\"W_Q,W_K,W_V::\",W_Q,W_K,W_V)\n# generating the queries, keys and values\nQ = words @ W_Q\nK = words @ W_K\nV = words @ W_V\nprint(\"Q,K,V \\n\")\nprint(Q,K,V)\n# scoring the query vectors against all key vectors\nscores = Q @ K.transpose()\nprint(\"Scores\",scores)\n# computing the weights by a softmax operation\nweights = softmax(scores / K.shape[1] ** 0.5, axis=1)\nprint(\"Weights \\n\",weights)\n# computing the attention by a weighted sum of the value vectors\nattention = weights @ V\n\nprint(attention)","metadata":{"execution":{"iopub.status.busy":"2023-03-14T07:45:33.006501Z","iopub.execute_input":"2023-03-14T07:45:33.006956Z","iopub.status.idle":"2023-03-14T07:45:33.022839Z","shell.execute_reply.started":"2023-03-14T07:45:33.006917Z","shell.execute_reply":"2023-03-14T07:45:33.021433Z"},"trusted":true},"execution_count":19,"outputs":[{"name":"stdout","text":"[[1 0 0]\n [0 1 0]\n [1 1 0]\n [0 0 1]]\nW_Q,W_K,W_V:: [[2 0 2]\n [2 0 0]\n [2 1 2]] [[2 2 2]\n [0 2 1]\n [0 1 1]] [[1 1 0]\n [0 1 1]\n [0 0 0]]\nQ,K,V \n\n[[2 0 2]\n [2 0 0]\n [4 0 2]\n [2 1 2]] [[2 2 2]\n [0 2 1]\n [2 4 3]\n [0 1 1]] [[1 1 0]\n [0 1 1]\n [1 2 1]\n [0 0 0]]\nScores [[ 8  2 10  2]\n [ 4  0  4  0]\n [12  2 14  2]\n [10  4 14  3]]\nWeights \n [[2.36089863e-01 7.38987555e-03 7.49130386e-01 7.38987555e-03]\n [4.54826323e-01 4.51736775e-02 4.54826323e-01 4.51736775e-02]\n [2.39275049e-01 7.43870015e-04 7.59237211e-01 7.43870015e-04]\n [8.99501754e-02 2.81554063e-03 9.05653685e-01 1.58059922e-03]]\n[[0.98522025 1.74174051 0.75652026]\n [0.90965265 1.40965265 0.5       ]\n [0.99851226 1.75849334 0.75998108]\n [0.99560386 1.90407309 0.90846923]]\n","output_type":"stream"}]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}