demo.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "t1wc1ZujyMNh"
      },
      "source": [
        "# CDGP demo"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "## Testing data\n",
        "\n",
        "* q1 ~ q10: From CLOTH dataset\n",
        "* q11 ~ q20: From DGen dataset\n",
        "* q21 ~ q23: From website"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {},
      "outputs": [],
      "source": [
        "questions = {\n",
        "    \"q1\": {\n",
        "        \"sentence\": \"To make Jane live a [MASK] life, Mother was very careful about spending money. [SEP] happy\",\n",
        "        \"answer\": \"happy\",\n",
        "        \"distractors\": [\"poor\", \"busy\", \"sad\"]\n",
        "    },\n",
        "    \"q2\": {\n",
        "        \"sentence\": \"[MASK] , Jane didn't understand her. [SEP] However\",\n",
        "        \"answer\": \"However\",\n",
        "        \"distractors\": [\"Though\", \"Although\", \"Or\"]\n",
        "    },\n",
        "    \"q3\": {\n",
        "        \"sentence\": \"Every day Mother was busy with her [MASK] while Jane was studying at school, so they had little time to enjoy themselves. [SEP] work\",\n",
        "        \"answer\": \"work\",\n",
        "        \"distractors\": [\"writing\", \"housework\", \"research\"]\n",
        "    },\n",
        "    \"q4\": {\n",
        "        \"sentence\": \"One day, Mother realized Jane was unhappy and even [MASK] to her. [SEP] unfriendly\",\n",
        "        \"answer\": \"unfriendly\",\n",
        "        \"distractors\": [\"loyal\", \"kind\", \"cruel\"]\n",
        "    },\n",
        "    \"q5\": {\n",
        "        \"sentence\": \"The old man was waiting for a ride across the [MASK] . [SEP] river\",\n",
        "        \"answer\": \"river\",\n",
        "        \"distractors\": [\"town\", \"country\", \"island\"]\n",
        "    },\n",
        "    \"q6\": {\n",
        "        \"sentence\": \"I felt uncomfortable and out of place as the professor carefully [MASK] what she expected us to learn. [SEP] explained\",\n",
        "        \"answer\": \"explained\",\n",
        "        \"distractors\": [\"showed\", \"designed\", \"offered\"]\n",
        "    },\n",
        "    \"q7\": {\n",
        "        \"sentence\": \"As I listened, I couldn't help but [MASK] of my own oldest daughter. [SEP] think\",\n",
        "        \"answer\": \"think\",\n",
        "        \"distractors\": [\"speak\", \"talk\", \"hear\"]\n",
        "    },\n",
        "    \"q8\": {\n",
        "        \"sentence\": \"As we were [MASK] on the third floor for old people with Alzheimer, most of them stared off at the walls or floor. [SEP] singing\",\n",
        "        \"answer\": \"singing\",\n",
        "        \"distractors\": [\"meeting\", \"gathering\", \"dancing\"]\n",
        "    },\n",
        "    \"q9\": {\n",
        "        \"sentence\": \"As we got [MASK] with each song, she did as well. [SEP] louder\",\n",
        "        \"answer\": \"louder\",\n",
        "        \"distractors\": [\"higher\", \"nearer\", \"faster\"]\n",
        "    },\n",
        "    \"q10\": {\n",
        "        \"sentence\": \"Mr. Petri,  _   injured in the fire, was rushed to hospital. [SEP] seriously\",\n",
        "        \"answer\": \"seriously\",\n",
        "        \"distractors\": [\"blindly\", \"hardly\", \"slightly\"]\n",
        "    },\n",
        "    \"q11\": {\n",
        "        \"sentence\": \"If an object is attracted to a magnet, the object is most likely made of [MASK]. [SEP] metal\",\n",
        "        \"answer\": \"metal\",\n",
        "        \"distractors\": [\"wood\", \"plastic\", \"cardboard\"]\n",
        "    },\n",
        "    \"q12\": {\n",
        "        \"sentence\": \"the main organs of the respiratory system are [MASK]. [SEP] lungs\",\n",
        "        \"answer\": \"lungs\",\n",
        "        \"distractors\": [\"ovaries\", \"intestines\", \"kidneys\"]\n",
        "    },\n",
        "    \"q13\": {\n",
        "        \"sentence\": \"The products of photosynthesis are glucose and [MASK] else. [SEP] oxygen\",\n",
        "        \"answer\": \"oxygen\",\n",
        "        \"distractors\": [\"carbon\", \"hydrogen\", \"nitrogen\"]\n",
        "    },\n",
        "    \"q14\": {\n",
        "        \"sentence\": \"frogs have [MASK] eyelid membranes. [SEP] three\",\n",
        "        \"answer\": \"three\",\n",
        "        \"distractors\": [\"two\", \"four\", \"one\"]\n",
        "    },\n",
        "    \"q15\": {\n",
        "        \"sentence\": \"the only known planet with large amounts of water is [MASK]. [SEP] earth\",\n",
        "        \"answer\": \"earth\",\n",
        "        \"distractors\": [\"saturn\", \"jupiter\", \"mars\"]\n",
        "    },\n",
        "    \"q16\": {\n",
        "        \"sentence\": \"[MASK] is responsible for erosion by flowing water and glaciers. [SEP] gravity\",\n",
        "        \"answer\": \"gravity\",\n",
        "        \"distractors\": [\"kinetic\", \"electromagnetic\", \"weight\"],\n",
        "    },\n",
        "    \"q17\": {\n",
        "        \"sentence\": \"Common among mammals and insects , pheromones are often related to [MASK] type of behavior. [SEP] reproductive\",\n",
        "        \"answer\": \"reproductive\",\n",
        "        \"distractors\": [\"aggressive\", \"immune\", \"cardiac\"]\n",
        "    },\n",
        "    \"q18\": {\n",
        "        \"sentence\": \"[MASK] can reproduce by infecting the cell of a living host. [SEP] virus\",\n",
        "        \"answer\": \"virus\",\n",
        "        \"distractors\": [\"bacteria\", \"mucus\", \"carcinogens\"]\n",
        "    },\n",
        "    \"q19\": {\n",
        "        \"sentence\": \"proteins are encoded by [MASK]. [SEP] genes\",\n",
        "        \"answer\": \"genes\",\n",
        "        \"distractors\": [\"DNA\", \"RNA\", \"codons\"]\n",
        "    },\n",
        "    \"q20\": {\n",
        "        \"sentence\": \"Producers at the base of ecological food webs are also known as [MASK]. [SEP] autotrophic\",\n",
        "        \"answer\": \"autotrophic\",\n",
        "        \"distractors\": [\"endoscopic\", \"symbiotic\", \"mutualistic\"],\n",
        "    },\n",
        "    \"q21\": {\n",
        "        \"sentence\": \"Today morning, I saw a [MASK] sitting on the wall. [SEP] cat\",\n",
        "        \"answer\": \"cat\",\n",
        "        \"distractors\": [],\n",
        "    },\n",
        "    \"q22\": {\n",
        "        \"sentence\": \"Ukrainian presidential adviser says situation is ' [MASK] control' in suburbs and outskirts of Kyiv. [SEP] under\",\n",
        "        \"answer\": \"under\",\n",
        "        \"distractors\": [],\n",
        "    },\n",
        "    \"q23\": {\n",
        "        \"sentence\": \"I don't think that after what is [MASK] now, Ukraine has weak positions. [SEP] happening\",\n",
        "        \"answer\": \"happening\",\n",
        "        \"distractors\": [],\n",
        "    },\n",
        "}"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "8aEwIB94yuVn"
      },
      "source": [
        "## Candidate Set Generator (CSG)\n",
        "\n",
        "Input: stem, answer  \n",
        "Output: candidate set of distractors"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "cKtTacFszaFj"
      },
      "source": [
        "### Import packages"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "id": "ry86IFrPzbn4"
      },
      "outputs": [],
      "source": [
        "from tqdm.notebook import tqdm\n",
        "import os"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "GolSA9-MVXox"
      },
      "source": [
        "### Load CSG model"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000,
          "referenced_widgets": [
            "13a6bbca8fe44a96a8265390874d0cc3",
            "760af93c66484f75b5698af259c96d6c",
            "5dfdb6ed6ea04b50b489432798d70555",
            "75c0744f5df244e29dff4a90e3e2460f",
            "394dae1a42dc4de888d898855309eeac",
            "9df003cfce844e629fcb9a149d735731",
            "09d6a08bc4ff4e1c9db722171c092247",
            "265b43d0e7134abbacb8693a2c6387da",
            "d1cbd3beb71a4903b30573e9dd8c3554",
            "765099c25ee64eafaab26ae21534417d",
            "e44caa4f83594150a69110c5c9612706",
            "604d98e8d1994ed7a7ab31f15e572cce",
            "d54d9712c56f4fbf9b08fa9222bb1ea9",
            "723dbc96fab04b9a94d6b8ba4cc96f74",
            "b88dea4099cc4d43a8b8315fe9bc8eb3",
            "4a0dd0b2bb074f23a6d2b634a134c233",
            "d3422d6801414024a0f8f7a65548d2b9",
            "94c5ecd282a74afdbf0539d8f717ea5c",
            "68edf999a09e4bffae1260c31bd7a9bc",
            "45088d1a68024956869c026c57f7595d",
            "869b00af41184c5f842f2630ea28cf2a",
            "b43c1f52f2dd497c8e8de5d46e1b2e99",
            "d17c9387fcbd40408a8da94e14dc15e6",
            "b7410d9320be4d0cab120d1cbcf53d2f",
            "971ebe8183e74dd5afe9e669436d8827",
            "18185801a4184d2fa0f0a3a408ce52f4",
            "862da95861ce4610a5e6299be33cddce",
            "4cc0a6159ee4424abdc2d7470bfa2bee",
            "bd1bd2cd6d8542f9bb724bbe4decc662",
            "c2fc82e1756b4d06bb62b0403e7ccfee",
            "b200f61879354d4780955f0c70415fd6",
            "57419a3899a24cad8bb50ff3abd53eea",
            "317ed13a365d432f9024d60a0db890c5"
          ]
        },
        "id": "9rIV2AAly0U_",
        "outputId": "37932414-d4cf-43d2-814d-06abf4ac4165"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "BertForMaskedLM(\n",
              "  (bert): BertModel(\n",
              "    (embeddings): BertEmbeddings(\n",
              "      (word_embeddings): Embedding(30522, 768, padding_idx=0)\n",
              "      (position_embeddings): Embedding(512, 768)\n",
              "      (token_type_embeddings): Embedding(2, 768)\n",
              "      (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "      (dropout): Dropout(p=0.1, inplace=False)\n",
              "    )\n",
              "    (encoder): BertEncoder(\n",
              "      (layer): ModuleList(\n",
              "        (0): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "            (intermediate_act_fn): GELUActivation()\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (1): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "            (intermediate_act_fn): GELUActivation()\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (2): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "            (intermediate_act_fn): GELUActivation()\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (3): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "            (intermediate_act_fn): GELUActivation()\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (4): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "            (intermediate_act_fn): GELUActivation()\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (5): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "            (intermediate_act_fn): GELUActivation()\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (6): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "            (intermediate_act_fn): GELUActivation()\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (7): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "            (intermediate_act_fn): GELUActivation()\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (8): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "            (intermediate_act_fn): GELUActivation()\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (9): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "            (intermediate_act_fn): GELUActivation()\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (10): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "            (intermediate_act_fn): GELUActivation()\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (11): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "            (intermediate_act_fn): GELUActivation()\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "    )\n",
              "  )\n",
              "  (cls): BertOnlyMLMHead(\n",
              "    (predictions): BertLMPredictionHead(\n",
              "      (transform): BertPredictionHeadTransform(\n",
              "        (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "        (transform_act_fn): GELUActivation()\n",
              "        (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "      )\n",
              "      (decoder): Linear(in_features=768, out_features=30522, bias=True)\n",
              "    )\n",
              "  )\n",
              ")"
            ]
          },
          "execution_count": 3,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "from transformers import BertTokenizer, BertConfig, BertForMaskedLM, pipeline\n",
        "\n",
        "CSG_MODEL = \"AndyChiang/cdgp-csg-bert-cloth\"\n",
        "\n",
        "tokenizer = BertTokenizer.from_pretrained(CSG_MODEL)\n",
        "csg_model = BertForMaskedLM.from_pretrained(CSG_MODEL)\n",
        "csg_model.eval()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 4,
      "metadata": {
        "id": "SOVuvEz-z9OB"
      },
      "outputs": [],
      "source": [
        "my_unmasker = pipeline('fill-mask', tokenizer=tokenizer, model=csg_model, top_k=10)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "metadata": {},
      "outputs": [
        {
          "data": {
            "text/plain": [
              "[{'score': 0.17808528244495392,\n",
              "  'token': 6517,\n",
              "  'token_str': 's a d',\n",
              "  'sequence': 'to make jane live a sad life, mother was very careful about spending money. happy'},\n",
              " {'score': 0.05460711568593979,\n",
              "  'token': 11771,\n",
              "  'token_str': 'b o r i n g',\n",
              "  'sequence': 'to make jane live a boring life, mother was very careful about spending money. happy'},\n",
              " {'score': 0.04597169905900955,\n",
              "  'token': 5697,\n",
              "  'token_str': 'b u s y',\n",
              "  'sequence': 'to make jane live a busy life, mother was very careful about spending money. happy'},\n",
              " {'score': 0.02937108650803566,\n",
              "  'token': 3407,\n",
              "  'token_str': 'h a p p y',\n",
              "  'sequence': 'to make jane live a happy life, mother was very careful about spending money. happy'},\n",
              " {'score': 0.019477127119898796,\n",
              "  'token': 2919,\n",
              "  'token_str': 'b a d',\n",
              "  'sequence': 'to make jane live a bad life, mother was very careful about spending money. happy'},\n",
              " {'score': 0.01932520419359207,\n",
              "  'token': 4138,\n",
              "  'token_str': 'r i c h',\n",
              "  'sequence': 'to make jane live a rich life, mother was very careful about spending money. happy'},\n",
              " {'score': 0.017796039581298828,\n",
              "  'token': 13971,\n",
              "  'token_str': 'l a z y',\n",
              "  'sequence': 'to make jane live a lazy life, mother was very careful about spending money. happy'},\n",
              " {'score': 0.01709771901369095,\n",
              "  'token': 3671,\n",
              "  'token_str': 'n o r m a l',\n",
              "  'sequence': 'to make jane live a normal life, mother was very careful about spending money. happy'},\n",
              " {'score': 0.01621469482779503,\n",
              "  'token': 3697,\n",
              "  'token_str': 'd i f f i c u l t',\n",
              "  'sequence': 'to make jane live a difficult life, mother was very careful about spending money. happy'},\n",
              " {'score': 0.01619175635278225,\n",
              "  'token': 4326,\n",
              "  'token_str': 's t r a n g e',\n",
              "  'sequence': 'to make jane live a strange life, mother was very careful about spending money. happy'}]"
            ]
          },
          "execution_count": 5,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "my_unmasker(questions[\"q1\"][\"sentence\"])"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "zQB5RuFI1rB6"
      },
      "source": [
        "## Candidate Set (CS)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 6,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ZVCsDqzsp9iI",
        "outputId": "1f98ea16-aeaf-4aba-a0a9-7da22ec1dc65"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "I realized that [MASK] I had accomplished in the past had prepared me for this moment.\n"
          ]
        }
      ],
      "source": [
        "origin_sent = \"I realized that [MASK] I had accomplished in the past had prepared me for this moment.\"\n",
        "print(origin_sent)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 7,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "odhAp6Pp4lio",
        "outputId": "b115e053-88cd-4137-f3f1-fad7bbd5c3f5"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "everything\n"
          ]
        }
      ],
      "source": [
        "answer = \"everything\"\n",
        "print(answer)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 8,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "RomjEX6nofJI",
        "outputId": "8f7e109b-efb9-457d-e9dc-20eaa03b1b02"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "I realized that [MASK] I had accomplished in the past had prepared me for this moment. [SEP] everything\n"
          ]
        }
      ],
      "source": [
        "target_sent = origin_sent + \" [SEP] \" + answer\n",
        "print(target_sent)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 9,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "8p1ww9uD4k2m",
        "outputId": "c016a758-5cf1-444b-8b6a-705b2ae58b71"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "[{'word': 'something', 's0': 0.4702465534210205, 's1': 0.0, 's2': 0.0, 's3': 0.0}, {'word': 'nothing', 's0': 0.21203435957431793, 's1': 0.0, 's2': 0.0, 's3': 0.0}, {'word': 'anything', 's0': 0.20486889779567719, 's1': 0.0, 's2': 0.0, 's3': 0.0}, {'word': 'everything', 's0': 0.05527903512120247, 's1': 0.0, 's2': 0.0, 's3': 0.0}, {'word': 'somewhere', 's0': 0.005942480638623238, 's1': 0.0, 's2': 0.0, 's3': 0.0}, {'word': 'none', 's0': 0.004807805176824331, 's1': 0.0, 's2': 0.0, 's3': 0.0}, {'word': 'anybody', 's0': 0.003521741833537817, 's1': 0.0, 's2': 0.0, 's3': 0.0}, {'word': 'any', 's0': 0.0026764380745589733, 's1': 0.0, 's2': 0.0, 's3': 0.0}, {'word': 'nobody', 's0': 0.002662935294210911, 's1': 0.0, 's2': 0.0, 's3': 0.0}, {'word': 'somebody', 's0': 0.00245393393561244, 's1': 0.0, 's2': 0.0, 's3': 0.0}]\n"
          ]
        }
      ],
      "source": [
        "cs = list()\n",
        "\n",
        "for cand in my_unmasker(target_sent):\n",
        "    # if cand[\"token_str\"].replace(\" \", \"\") != answer:\n",
        "    cs.append({\"word\": cand[\"token_str\"].replace(\" \", \"\"), \"s0\": cand[\"score\"], \"s1\": 0.0, \"s2\": 0.0, \"s3\": 0.0})\n",
        "\n",
        "print(cs)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5fmS9vqDbUFG"
      },
      "source": [
        "## Distractor Selector (DS)\n",
        "\n",
        "Input: stem, answer, candidate set of distractors  \n",
        "Output: Top 3 distractors"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "X8N86qMGqJiE"
      },
      "source": [
        "### Load DS model"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 10,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Ui8Oo1Eoqc5M",
        "outputId": "3ad1e228-8b0b-4d44-fae4-b4f8c83d89dc"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Warning : `load_model` does not return WordVectorModel or SupervisedModel any more, but a `FastText` object which is very similar.\n"
          ]
        }
      ],
      "source": [
        "import fasttext\n",
        "import warnings\n",
        "\n",
        "model_path = r\"./models/DS/cdgp-ds-fasttext.bin\"\n",
        "\n",
        "ds_model = fasttext.load_model(model_path)\n",
        "warnings.filterwarnings(\"ignore\", category=FutureWarning)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "WOZTGiISqiCO"
      },
      "source": [
        "### Cosine similarity"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 11,
      "metadata": {
        "id": "7Fi7Y7bpq01A"
      },
      "outputs": [],
      "source": [
        "import numpy as np\n",
        "\n",
        "def similarity(v1, v2):\n",
        "    n1 = np.linalg.norm(v1)\n",
        "    n2 = np.linalg.norm(v2)\n",
        "    return np.dot(v1, v2) / n1 / n2"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "uOprpwJPf5eq"
      },
      "source": [
        "### Normalization"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "99Mxtt4hgBr2"
      },
      "source": [
        "#### Min-Max normalization"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 12,
      "metadata": {
        "id": "eDQCpfX0-xYM"
      },
      "outputs": [],
      "source": [
        "def min_max_y(raw_data):\n",
        "  min_max_data = []\n",
        "  \n",
        "  for d in raw_data:\n",
        "    min_max_data.append((d - min(raw_data)) / (max(raw_data) - min(raw_data)))\n",
        "             \n",
        "  return min_max_data"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "LzmYNKL_gEjc"
      },
      "source": [
        "#### Z-Score normalization"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 13,
      "metadata": {
        "id": "K4Z3qZWk-AKR"
      },
      "outputs": [],
      "source": [
        "import math\n",
        "\n",
        "def z_score_y(raw_data):\n",
        "  print(raw_data)\n",
        "  total = 0\n",
        "  for d in raw_data:\n",
        "    total += d\n",
        "  aver = total/len(raw_data)\n",
        "  \n",
        "  data_s = 0\n",
        "  for s in raw_data:\n",
        "    data_s += (s - aver)**2\n",
        "  \n",
        "  std = math.sqrt(data_s/len(raw_data))\n",
        "  \n",
        "  z_score_data = []\n",
        "  for z in raw_data:\n",
        "    z_score_data.append((z - aver)/std)\n",
        "  \n",
        "  return z_score_data"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "dSDl5ZQrgN28"
      },
      "source": [
        "### Confidence Score s0"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 14,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "AxqvodkNgMeV",
        "outputId": "f5aa33ec-22ed-436d-d163-36490b711b70"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "something 1.0\n",
            "nothing 0.44801994924428895\n",
            "anything 0.43270234592997614\n",
            "everything 0.11292418688370907\n",
            "somewhere 0.007457464178995275\n",
            "none 0.0050318691299603025\n",
            "anybody 0.002282652298148721\n",
            "any 0.00047564696337299485\n",
            "nobody 0.000446782077982293\n",
            "somebody 0.0\n"
          ]
        }
      ],
      "source": [
        "s0s = [c[\"s0\"] for c in cs]\n",
        "new_s0s = min_max_y(s0s)\n",
        "\n",
        "for i, c in enumerate(cs):\n",
        "    print(c[\"word\"], new_s0s[i])\n",
        "    c[\"s0\"] = new_s0s[i]"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "uxPzPnsHquLx"
      },
      "source": [
        "### Word Embedding Similarity s1"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 15,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "c_Rybw54q-8d",
        "outputId": "51493cd1-2ce4-4774-a172-f3b77f925e71"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "everything\n",
            "----------------------------------------------------------------------------------------------------\n",
            "something 0.05007749795913696 0.07134193181991577\n",
            "nothing 0.056958675384521484 0.08114504814147949\n",
            "anything 0.05596303939819336 0.07972663640975952\n",
            "everything -1.1920928955078125e-07 0.0\n",
            "somewhere 0.2000046968460083 0.28493237495422363\n",
            "none 0.7019376754760742 1.0\n",
            "anybody 0.16913676261901855 0.2409570813179016\n",
            "any 0.35927659273147583 0.5118355453014374\n",
            "nobody 0.22937774658203125 0.3267780542373657\n",
            "somebody 0.18457138538360596 0.26294565200805664\n"
          ]
        }
      ],
      "source": [
        "print(answer)\n",
        "print(\"-\"*100)\n",
        "answer_vector = ds_model.get_word_vector(answer)\n",
        "\n",
        "word_similarities = list()\n",
        "for c in cs:\n",
        "  c_vector = ds_model.get_word_vector(c[\"word\"])\n",
        "  word_similarity = similarity(answer_vector, c_vector)\n",
        "  word_similarities.append(word_similarity)\n",
        "\n",
        "new_similarities = min_max_y(word_similarities)\n",
        "\n",
        "for i, c in enumerate(cs):\n",
        "    print(c[\"word\"], 1-word_similarities[i], 1-new_similarities[i])\n",
        "    c[\"s1\"] = 1-new_similarities[i]"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "VngDSn9krED-"
      },
      "source": [
        "### Contextual-Sentence Embedding Similarity s2"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 16,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "5BMQePMprGsL",
        "outputId": "a52c2611-6373-4bef-e7c0-9167fb0e43b9"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "I realized that everything I had accomplished in the past had prepared me for this moment.\n",
            "----------------------------------------------------------------------------------------------------\n",
            "I realized that something I had accomplished in the past had prepared me for this moment. 0.000433504581451416 0.07980388402938843\n",
            "I realized that nothing I had accomplished in the past had prepared me for this moment. 0.0004928112030029297 0.09071862697601318\n",
            "I realized that anything I had accomplished in the past had prepared me for this moment. 0.00048416852951049805 0.08912801742553711\n",
            "I realized that everything I had accomplished in the past had prepared me for this moment. -1.1920928955078125e-07 0.0\n",
            "I realized that somewhere I had accomplished in the past had prepared me for this moment. 0.0017292499542236328 0.31827205419540405\n",
            "I realized that none I had accomplished in the past had prepared me for this moment. 0.005433499813079834 1.0\n",
            "I realized that anybody I had accomplished in the past had prepared me for this moment. 0.0013561844825744629 0.24961334466934204\n",
            "I realized that any I had accomplished in the past had prepared me for this moment. 0.003071725368499756 0.5653404295444489\n",
            "I realized that nobody I had accomplished in the past had prepared me for this moment. 0.0016868114471435547 0.31046169996261597\n",
            "I realized that somebody I had accomplished in the past had prepared me for this moment. 0.001408994197845459 0.25933241844177246\n"
          ]
        }
      ],
      "source": [
        "#依據訓練過後的BERT所生成選項放入句子做比較\n",
        "correct_sent = origin_sent.replace('[MASK]', answer)\n",
        "print(correct_sent)\n",
        "print(\"-\"*100)\n",
        "correct_sent_vector = ds_model.get_sentence_vector(correct_sent)\n",
        "\n",
        "cand_sents = list()\n",
        "for c in cs:\n",
        "  cand_sents.append(origin_sent.replace('[MASK]', c[\"word\"]))\n",
        "\n",
        "sent_similarities = list()\n",
        "#兩句子距離\n",
        "for cand_sent in cand_sents:\n",
        "  cand_sent_vector = ds_model.get_sentence_vector(cand_sent)\n",
        "  sent_similarity = similarity(correct_sent_vector, cand_sent_vector)\n",
        "  sent_similarities.append(sent_similarity)\n",
        "\n",
        "new_similarities = min_max_y(sent_similarities)\n",
        "for i, c in enumerate(cs):\n",
        "    print(cand_sents[i], 1-sent_similarities[i], 1-new_similarities[i])\n",
        "    c[\"s2\"] = 1-new_similarities[i]"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-hWeywrsseyQ"
      },
      "source": [
        "### POS match score s3"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 17,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "hM5ESPedBhS-",
        "outputId": "b59d2014-8470-49c3-bab8-44c1a0e3a0c6"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "[nltk_data] Downloading package punkt to\n",
            "[nltk_data]     C:\\Users\\andy1\\AppData\\Roaming\\nltk_data...\n",
            "[nltk_data]   Unzipping tokenizers\\punkt.zip.\n",
            "[nltk_data] Downloading package averaged_perceptron_tagger to\n",
            "[nltk_data]     C:\\Users\\andy1\\AppData\\Roaming\\nltk_data...\n",
            "[nltk_data]   Package averaged_perceptron_tagger is already up-to-\n",
            "[nltk_data]       date!\n"
          ]
        },
        {
          "data": {
            "text/plain": [
              "True"
            ]
          },
          "execution_count": 17,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "import nltk\n",
        "from nltk.tokenize import word_tokenize\n",
        "\n",
        "nltk.download('punkt')\n",
        "nltk.download('averaged_perceptron_tagger')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 18,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "YXT91qfgB8Sh",
        "outputId": "445936c8-d4cf-4cd1-f6ad-427f4f5a125f"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "('everything', 'NN')\n",
            "----------------------------------------------------------------------------------------------------\n",
            "('something', 'NN') 1.0\n",
            "('nothing', 'NN') 1.0\n",
            "('anything', 'NN') 1.0\n",
            "('everything', 'NN') 1.0\n",
            "('somewhere', 'RB') 0.0\n",
            "('none', 'NN') 1.0\n",
            "('anybody', 'NN') 1.0\n",
            "('any', 'DT') 0.0\n",
            "('nobody', 'NN') 1.0\n",
            "('somebody', 'NN') 1.0\n"
          ]
        }
      ],
      "source": [
        "origin_token = word_tokenize(origin_sent)\n",
        "origin_token.remove(\"[\")\n",
        "origin_token.remove(\"]\")\n",
        "\n",
        "mask_index = origin_token.index(\"MASK\")\n",
        "\n",
        "correct_token = word_tokenize(correct_sent)\n",
        "correct_pos = nltk.pos_tag(correct_token)\n",
        "answer_pos = correct_pos[mask_index]\n",
        "print(answer_pos)\n",
        "print(\"-\"*100)\n",
        "\n",
        "for i, c in enumerate(cs):\n",
        "    cand_sent_token = word_tokenize(cand_sents[i])\n",
        "    cand_sent_pos = nltk.pos_tag(cand_sent_token)\n",
        "    cand_pos = cand_sent_pos[mask_index]\n",
        "\n",
        "    if cand_pos[1] == answer_pos[1]:\n",
        "        c[\"s3\"] = 1.0\n",
        "    else:\n",
        "        c[\"s3\"] = 0.0\n",
        "    \n",
        "    print(cand_pos, c[\"s3\"])"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "xlGFWHIOBA_4"
      },
      "source": [
        "## Ranker"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 19,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "IV1V6Y3Y_8BT",
        "outputId": "04184147-bbd4-4334-bae6-2d711f3ac921"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "[{'word': 'something',\n",
              "  's0': 1.0,\n",
              "  's1': 0.07134193181991577,\n",
              "  's2': 0.07980388402938843,\n",
              "  's3': 1.0},\n",
              " {'word': 'nothing',\n",
              "  's0': 0.44801994924428895,\n",
              "  's1': 0.08114504814147949,\n",
              "  's2': 0.09071862697601318,\n",
              "  's3': 1.0},\n",
              " {'word': 'anything',\n",
              "  's0': 0.43270234592997614,\n",
              "  's1': 0.07972663640975952,\n",
              "  's2': 0.08912801742553711,\n",
              "  's3': 1.0},\n",
              " {'word': 'everything',\n",
              "  's0': 0.11292418688370907,\n",
              "  's1': 0.0,\n",
              "  's2': 0.0,\n",
              "  's3': 1.0},\n",
              " {'word': 'somewhere',\n",
              "  's0': 0.007457464178995275,\n",
              "  's1': 0.28493237495422363,\n",
              "  's2': 0.31827205419540405,\n",
              "  's3': 0.0},\n",
              " {'word': 'none',\n",
              "  's0': 0.0050318691299603025,\n",
              "  's1': 1.0,\n",
              "  's2': 1.0,\n",
              "  's3': 1.0},\n",
              " {'word': 'anybody',\n",
              "  's0': 0.002282652298148721,\n",
              "  's1': 0.2409570813179016,\n",
              "  's2': 0.24961334466934204,\n",
              "  's3': 1.0},\n",
              " {'word': 'any',\n",
              "  's0': 0.00047564696337299485,\n",
              "  's1': 0.5118355453014374,\n",
              "  's2': 0.5653404295444489,\n",
              "  's3': 0.0},\n",
              " {'word': 'nobody',\n",
              "  's0': 0.000446782077982293,\n",
              "  's1': 0.3267780542373657,\n",
              "  's2': 0.31046169996261597,\n",
              "  's3': 1.0},\n",
              " {'word': 'somebody',\n",
              "  's0': 0.0,\n",
              "  's1': 0.26294565200805664,\n",
              "  's2': 0.25933241844177246,\n",
              "  's3': 1.0}]"
            ]
          },
          "execution_count": 19,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "cs"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2oSdNgW0lYZ6"
      },
      "source": [
        "### Weight final score"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 300,
      "metadata": {},
      "outputs": [],
      "source": [
        "WEIGHT = {\"s0\": 0.25, \"s1\": 0.25, \"s2\": 0.25, \"s3\": 0.25}"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 20,
      "metadata": {},
      "outputs": [],
      "source": [
        "WEIGHT = {\"s0\": 0.6, \"s1\": 0.15, \"s2\": 0.15, \"s3\": 0.1}"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 306,
      "metadata": {},
      "outputs": [],
      "source": [
        "WEIGHT = {\"s0\": 0.8, \"s1\": 0.05, \"s2\": 0.05, \"s3\": 0.1}"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 21,
      "metadata": {
        "id": "9e6onGt5AGho"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "[('something', 0.7226718723773956),\n",
              " ('nothing', 0.39459152081419735),\n",
              " ('anything', 0.3849496056332802),\n",
              " ('everything', 0.16775451213022546),\n",
              " ('somewhere', 0.0949551428798413),\n",
              " ('none', 0.40301912147797614),\n",
              " ('anybody', 0.17495515527697578),\n",
              " ('any', 0.16186178440490673),\n",
              " ('nobody', 0.19585403237678664),\n",
              " ('somebody', 0.1783417105674744)]"
            ]
          },
          "execution_count": 21,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "cs_fs = list()\n",
        "for c in cs:\n",
        "    fs = WEIGHT[\"s0\"]*c[\"s0\"] + WEIGHT[\"s1\"]*c[\"s1\"] + WEIGHT[\"s2\"]*c[\"s2\"] + WEIGHT[\"s3\"]*c[\"s3\"]\n",
        "    cs_fs.append((c[\"word\"], fs))\n",
        "\n",
        "cs_fs"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "jYNkGFD9lfkG"
      },
      "source": [
        "### Sort by final score"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 22,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "VkdOSg09ApWj",
        "outputId": "2ee08997-d9a1-4d2e-aea0-313b6be19547"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "[('something', 0.7226718723773956),\n",
              " ('none', 0.40301912147797614),\n",
              " ('nothing', 0.39459152081419735),\n",
              " ('anything', 0.3849496056332802),\n",
              " ('nobody', 0.19585403237678664),\n",
              " ('somebody', 0.1783417105674744),\n",
              " ('anybody', 0.17495515527697578),\n",
              " ('everything', 0.16775451213022546),\n",
              " ('any', 0.16186178440490673),\n",
              " ('somewhere', 0.0949551428798413)]"
            ]
          },
          "execution_count": 22,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "cs_fs.sort(key = lambda x: x[1], reverse=True)\n",
        "cs_fs"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "XfCp0FkUlwOx"
      },
      "source": [
        "## Distractors"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "KGGVfmSKliE8"
      },
      "source": [
        "### Top 3"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 23,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "c1Io7dMTA8IU",
        "outputId": "87521419-8040-4f30-fb7c-7aced24b913a"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "[('something', 0.7226718723773956),\n",
              " ('none', 0.40301912147797614),\n",
              " ('nothing', 0.39459152081419735)]"
            ]
          },
          "execution_count": 23,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "distractors = cs_fs[:3]\n",
        "distractors"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 24,
      "metadata": {},
      "outputs": [
        {
          "data": {
            "text/plain": [
              "['something', 'none', 'nothing']"
            ]
          },
          "execution_count": 24,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "result = [d[0] for d in distractors]\n",
        "result"
      ]
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "collapsed_sections": [
        "cKtTacFszaFj",
        "WOZTGiISqiCO",
        "99Mxtt4hgBr2",
        "LzmYNKL_gEjc",
        "0K3l_6jcXARM"
      ],
      "name": "cloze-distractors-generation.ipynb",
      "provenance": [],
      "toc_visible": true
    },
    "kernelspec": {
      "display_name": "Python 3.8.8 ('CDGP-env': venv)",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.8.8"
    },
    "vscode": {
      "interpreter": {
        "hash": "85c2d217b781005dc1c8be6b2419f9855afa8bd856110cc978d46a21df1cad86"
      }
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "09d6a08bc4ff4e1c9db722171c092247": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "13a6bbca8fe44a96a8265390874d0cc3": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_5dfdb6ed6ea04b50b489432798d70555",
              "IPY_MODEL_75c0744f5df244e29dff4a90e3e2460f",
              "IPY_MODEL_394dae1a42dc4de888d898855309eeac"
            ],
            "layout": "IPY_MODEL_760af93c66484f75b5698af259c96d6c"
          }
        },
        "18185801a4184d2fa0f0a3a408ce52f4": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_b200f61879354d4780955f0c70415fd6",
            "max": 570,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_c2fc82e1756b4d06bb62b0403e7ccfee",
            "value": 570
          }
        },
        "265b43d0e7134abbacb8693a2c6387da": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "317ed13a365d432f9024d60a0db890c5": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "394dae1a42dc4de888d898855309eeac": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_e44caa4f83594150a69110c5c9612706",
            "placeholder": "​",
            "style": "IPY_MODEL_765099c25ee64eafaab26ae21534417d",
            "value": " 226k/226k [00:00&lt;00:00, 1.00MB/s]"
          }
        },
        "45088d1a68024956869c026c57f7595d": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "4a0dd0b2bb074f23a6d2b634a134c233": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_b43c1f52f2dd497c8e8de5d46e1b2e99",
            "placeholder": "​",
            "style": "IPY_MODEL_869b00af41184c5f842f2630ea28cf2a",
            "value": " 28.0/28.0 [00:00&lt;00:00, 167B/s]"
          }
        },
        "4cc0a6159ee4424abdc2d7470bfa2bee": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "57419a3899a24cad8bb50ff3abd53eea": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "5dfdb6ed6ea04b50b489432798d70555": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_09d6a08bc4ff4e1c9db722171c092247",
            "placeholder": "​",
            "style": "IPY_MODEL_9df003cfce844e629fcb9a149d735731",
            "value": "Downloading: 100%"
          }
        },
        "604d98e8d1994ed7a7ab31f15e572cce": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_723dbc96fab04b9a94d6b8ba4cc96f74",
              "IPY_MODEL_b88dea4099cc4d43a8b8315fe9bc8eb3",
              "IPY_MODEL_4a0dd0b2bb074f23a6d2b634a134c233"
            ],
            "layout": "IPY_MODEL_d54d9712c56f4fbf9b08fa9222bb1ea9"
          }
        },
        "68edf999a09e4bffae1260c31bd7a9bc": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "723dbc96fab04b9a94d6b8ba4cc96f74": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_94c5ecd282a74afdbf0539d8f717ea5c",
            "placeholder": "​",
            "style": "IPY_MODEL_d3422d6801414024a0f8f7a65548d2b9",
            "value": "Downloading: 100%"
          }
        },
        "75c0744f5df244e29dff4a90e3e2460f": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_d1cbd3beb71a4903b30573e9dd8c3554",
            "max": 231508,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_265b43d0e7134abbacb8693a2c6387da",
            "value": 231508
          }
        },
        "760af93c66484f75b5698af259c96d6c": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "765099c25ee64eafaab26ae21534417d": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "862da95861ce4610a5e6299be33cddce": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_317ed13a365d432f9024d60a0db890c5",
            "placeholder": "​",
            "style": "IPY_MODEL_57419a3899a24cad8bb50ff3abd53eea",
            "value": " 570/570 [00:00&lt;00:00, 4.03kB/s]"
          }
        },
        "869b00af41184c5f842f2630ea28cf2a": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "94c5ecd282a74afdbf0539d8f717ea5c": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "971ebe8183e74dd5afe9e669436d8827": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_bd1bd2cd6d8542f9bb724bbe4decc662",
            "placeholder": "​",
            "style": "IPY_MODEL_4cc0a6159ee4424abdc2d7470bfa2bee",
            "value": "Downloading: 100%"
          }
        },
        "9df003cfce844e629fcb9a149d735731": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "b200f61879354d4780955f0c70415fd6": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "b43c1f52f2dd497c8e8de5d46e1b2e99": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "b7410d9320be4d0cab120d1cbcf53d2f": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "b88dea4099cc4d43a8b8315fe9bc8eb3": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_45088d1a68024956869c026c57f7595d",
            "max": 28,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_68edf999a09e4bffae1260c31bd7a9bc",
            "value": 28
          }
        },
        "bd1bd2cd6d8542f9bb724bbe4decc662": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c2fc82e1756b4d06bb62b0403e7ccfee": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "d17c9387fcbd40408a8da94e14dc15e6": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_971ebe8183e74dd5afe9e669436d8827",
              "IPY_MODEL_18185801a4184d2fa0f0a3a408ce52f4",
              "IPY_MODEL_862da95861ce4610a5e6299be33cddce"
            ],
            "layout": "IPY_MODEL_b7410d9320be4d0cab120d1cbcf53d2f"
          }
        },
        "d1cbd3beb71a4903b30573e9dd8c3554": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "d3422d6801414024a0f8f7a65548d2b9": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "d54d9712c56f4fbf9b08fa9222bb1ea9": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e44caa4f83594150a69110c5c9612706": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        }
      }
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}