inference.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "_cell_guid": "79c7e3d0-c299-4dcb-8224-4455121ee9b0",
    "_uuid": "d629ff2d2480ee46fbb7e2d37f6b5fab8052498a"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n",
      "/opt/conda/lib/python3.6/importlib/_bootstrap.py:219: ImportWarning: can't resolve package from __spec__ or __package__, falling back on __name__ and __path__\n",
      "  return f(*args, **kwds)\n",
      "/opt/conda/lib/python3.6/site-packages/tensorflow/python/keras/backend.py:5506: ResourceWarning: unclosed file <_io.TextIOWrapper name='/tmp/.keras/keras.json' mode='r' encoding='UTF-8'>\n",
      "  _config = json.load(open(_config_path))\n",
      "/opt/conda/lib/python3.6/importlib/_bootstrap.py:219: RuntimeWarning: numpy.ufunc size changed, may indicate binary incompatibility. Expected 192 from C header, got 216 from PyObject\n",
      "  return f(*args, **kwds)\n",
      "/opt/conda/lib/python3.6/importlib/_bootstrap.py:219: RuntimeWarning: numpy.ufunc size changed, may indicate binary incompatibility. Expected 192 from C header, got 216 from PyObject\n",
      "  return f(*args, **kwds)\n",
      "/opt/conda/lib/python3.6/importlib/_bootstrap.py:219: RuntimeWarning: numpy.ufunc size changed, may indicate binary incompatibility. Expected 192 from C header, got 216 from PyObject\n",
      "  return f(*args, **kwds)\n"
     ]
    }
   ],
   "source": [
    "from __future__ import absolute_import #?\n",
    "from __future__ import division #?\n",
    "from __future__ import print_function #?\n",
    "\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import os\n",
    "\n",
    "import time\n",
    "from tqdm._tqdm_notebook import tqdm_notebook as tqdm\n",
    "tqdm.pandas()\n",
    "import pickle\n",
    "import sys\n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings(action='once')\n",
    "\n",
    "import torch #?\n",
    "import torch.utils.data\n",
    "\n",
    "# BERT\n",
    "#package_dir = \"../input/ppbert/pytorch-pretrained-bert/pytorch-pretrained-BERT\" # doesnt contain GPT2ClassificationHeadModel apparently\n",
    "#sys.path.append(package_dir)\n",
    "sys.path.insert(0, \"../input/gpt2-pytorch/pytorch-pretrained-bert-master/pytorch-pretrained-BERT-master\")\n",
    "from pytorch_pretrained_bert import BertTokenizer, BertForSequenceClassification, BertAdam\n",
    "from pytorch_pretrained_bert import BertConfig\n",
    "\n",
    "# LSTM\n",
    "from keras.preprocessing import sequence, text #\n",
    "from keras.models import load_model\n",
    "\n",
    "# FASTAI\n",
    "from torch import nn # FASTAI\n",
    "from torch.nn import functional as F # FASTAI\n",
    "\n",
    "# GPT2\n",
    "#sys.path.insert(0, \"../input/gpt2-pytorch/pytorch-pretrained-bert-master/pytorch-pretrained-BERT-master\")\n",
    "from pytorch_pretrained_bert import GPT2Tokenizer, GPT2ClassificationHeadModel\n",
    "from pytorch_pretrained_bert import OpenAIAdam"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['d-fastai-n5-e4-s4',\n",
       " 'd-lstm-m-n2-e6-s456-2',\n",
       " 'd-bert-pytorch-15-soft-targets-w-t15e-10',\n",
       " 'pickled-crawl300d2m-for-kernel-competitions',\n",
       " 'd-gpt2-pytorch-m-1515-st-w-1',\n",
       " 'pickled-glove840b300d-for-10sec-loading',\n",
       " 'd-bert-pytorch-15-soft-targets-w-t15e-1',\n",
       " 'd-gpt2-pytorch-m2f-1515-1',\n",
       " 'd-bert-pytorch-15-soft-targets-w-t15e-4',\n",
       " 'd-gpt2-pytorch-m2-1515-taw3-lrscos-1',\n",
       " 'd-bert-pytorch-15-soft-targets-w-t15e-2',\n",
       " 'd-bert-pytorch-15-soft-targets-w-t15e-3']"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "INPUT_PATH = '../input'\n",
    "#JIGSAW_PATH = '../input/jigsaw-unintended-bias-in-toxicity-classification'\n",
    "\n",
    "UTILS = ['jigsaw-unintended-bias-in-toxicity-classification',\n",
    "         'ppbert',\n",
    "         'bert-pretrained-models',\n",
    "         'gpt2-pytorch',\n",
    "         'gpt2-models',         \n",
    "        ]\n",
    "\n",
    "MODEL_PATHS = [p for p in os.listdir('../input') if p not in UTILS]\n",
    "MODEL_PATHS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['../input/d-bert-pytorch-15-soft-targets-w-t15e-10',\n",
       " '../input/d-bert-pytorch-15-soft-targets-w-t15e-1',\n",
       " '../input/d-bert-pytorch-15-soft-targets-w-t15e-4',\n",
       " '../input/d-bert-pytorch-15-soft-targets-w-t15e-2',\n",
       " '../input/d-bert-pytorch-15-soft-targets-w-t15e-3']"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "MODEL_PATHS_BERT = [os.path.join(INPUT_PATH, p) for p in os.listdir(INPUT_PATH) if p not in UTILS and p.split('-')[1] == 'bert']\n",
    "MODEL_PATHS_BERT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['../input/d-lstm-m-n2-e6-s456-2']"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "MODEL_PATHS_LSTM = [os.path.join(INPUT_PATH, p) for p in os.listdir(INPUT_PATH) if p not in UTILS and p.split('-')[1] == 'lstm']\n",
    "MODEL_PATHS_LSTM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['../input/d-fastai-n5-e4-s4']"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "MODEL_PATHS_FASTAI = [os.path.join(INPUT_PATH, p) for p in os.listdir(INPUT_PATH) if p not in UTILS and p.split('-')[1] == 'fastai']\n",
    "MODEL_PATHS_FASTAI"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['../input/d-gpt2-pytorch-m-1515-st-w-1',\n",
       " '../input/d-gpt2-pytorch-m2f-1515-1',\n",
       " '../input/d-gpt2-pytorch-m2-1515-taw3-lrscos-1']"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "MODEL_PATHS_GPT2 = [os.path.join(INPUT_PATH, p) for p in os.listdir(INPUT_PATH) if p not in UTILS and p.split('-')[1] == 'gpt2']\n",
    "MODEL_PATHS_GPT2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## TEST DATAFRAME"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>comment_text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>7097320</td>\n",
       "      <td>[ Integrity means that you pay your debts.]\\n\\...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>7097321</td>\n",
       "      <td>This is malfeasance by the Administrator and t...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>7097322</td>\n",
       "      <td>@Rmiller101 - Spoken like a true elitist. But ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>7097323</td>\n",
       "      <td>Paul: Thank you for your kind words.  I do, in...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>7097324</td>\n",
       "      <td>Sorry you missed high school. Eisenhower sent ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        id                                       comment_text\n",
       "0  7097320  [ Integrity means that you pay your debts.]\\n\\...\n",
       "1  7097321  This is malfeasance by the Administrator and t...\n",
       "2  7097322  @Rmiller101 - Spoken like a true elitist. But ...\n",
       "3  7097323  Paul: Thank you for your kind words.  I do, in...\n",
       "4  7097324  Sorry you missed high school. Eisenhower sent ..."
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_df = pd.read_csv(\"../input/jigsaw-unintended-bias-in-toxicity-classification/test.csv\")\n",
    "test_df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## FUNCTIONS"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### BERT FUNCTIONS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "BERT_PRETRAINED_MODEL_PATH = '../input/bert-pretrained-models/uncased_l-12_h-768_a-12/uncased_L-12_H-768_A-12/' # contains tokenizer\n",
    "BERT_PYTORCH_PATH = '../input/d-bert-pytorch-15-soft-targets-w-t15e-3' # contains pretrained model converted to pytorch (better to save/load only one)\n",
    "\n",
    "MAX_SEQUENCE_LENGTH_BERT = 220\n",
    "NUM_LABELS_BERT = 6\n",
    "BATCH_SIZE_BERT = 32\n",
    "#BATCH_SIZE_BERT = 768 # a bit faster but riskier because of RAM usage\n",
    "\n",
    "DEVICE = torch.device('cuda')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def preprocess_bert():\n",
    "    start = time.time()\n",
    "    \n",
    "    ## grab tokenizer from pretrained BERT\n",
    "    tokenizer = BertTokenizer.from_pretrained(BERT_PRETRAINED_MODEL_PATH, cache_dir=None, do_lower_case=True)\n",
    "    \n",
    "    # Converting the lines to BERT format\n",
    "    # Thanks to https://www.kaggle.com/httpwwwfszyc/bert-in-keras-taming\n",
    "    def convert_lines_bert(example, max_seq_length, tokenizer):\n",
    "        max_seq_length -=2\n",
    "        all_tokens = []\n",
    "        longer = 0\n",
    "        for text in tqdm(example):\n",
    "            tokens_a = tokenizer.tokenize(text)\n",
    "            if len(tokens_a)>max_seq_length:\n",
    "                tokens_a = tokens_a[:max_seq_length]\n",
    "                longer += 1\n",
    "            one_token = tokenizer.convert_tokens_to_ids([\"[CLS]\"]+tokens_a+[\"[SEP]\"])+[0] * (max_seq_length - len(tokens_a))\n",
    "            all_tokens.append(one_token)\n",
    "        #print(longer)\n",
    "        return np.array(all_tokens)\n",
    "    \n",
    "    ## test read & conversion to BERT format\n",
    "    #test_df['comment_text'] = test_df['comment_text'].astype(str) # we want test_df intact\n",
    "    X_test_bert = convert_lines_bert(test_df[\"comment_text\"].astype(str).fillna(\"DUMMY_VALUE\"), MAX_SEQUENCE_LENGTH_BERT, tokenizer)\n",
    "\n",
    "    print(f'{time.time() - start:.2f} seconds')\n",
    "    return X_test_bert    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def bert_predict(model_path, X_test, batch_size=BATCH_SIZE_BERT):\n",
    "    start = time.time()\n",
    "    \n",
    "    # loading pytorch model (pytorch_model.bin) and config (bert_config.json)\n",
    "    model = BertForSequenceClassification.from_pretrained(BERT_PYTORCH_PATH, cache_dir=None, num_labels=NUM_LABELS_BERT)\n",
    "\n",
    "    # loading model state \n",
    "    model.load_state_dict(torch.load(os.path.join(model_path, 'bert_pytorch.bin'))['state_dict'])\n",
    "\n",
    "    # setting model in eval mode\n",
    "    model.to(DEVICE)\n",
    "    for param in model.parameters():\n",
    "        param.requires_grad=False\n",
    "    model.eval()\n",
    "    \n",
    "    # predicting\n",
    "    test_preds = np.zeros((len(X_test)))\n",
    "    test = torch.utils.data.TensorDataset(torch.tensor(X_test, dtype=torch.long))\n",
    "    test_loader = torch.utils.data.DataLoader(test, batch_size=batch_size, shuffle=False)\n",
    "    for i,(x_batch,) in enumerate(tqdm(test_loader)):\n",
    "        pred = model(x_batch.to(DEVICE), attention_mask=(x_batch>0).to(DEVICE), labels=None)\n",
    "        test_preds[i*batch_size:(i+1)*batch_size]=pred[:,0].detach().cpu().squeeze().numpy()\n",
    "    test_pred = torch.sigmoid(torch.tensor(test_preds)).numpy().ravel()\n",
    "    \n",
    "    print(f'{time.time() - start:.2f} seconds')\n",
    "    return test_pred    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### LSTM FUNCTIONS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "CHARS_TO_ISOLATE_LSTM = '!\"#$%&()*+,-./:;<=>?@[\\\\]^_`{|}~\\t\\n“”’\\'∞θ÷α•à−β∅³π‘₹´°£€\\×™√²—'\n",
    "CHARS_TO_REMOVE_LSTM = '_`\\t\\n₹' # applied by tokenizer\n",
    "\n",
    "MAX_LEN_LSTM = 220\n",
    "BATCH_SIZE_LSTM = 2048"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def clean_text_lstm(x, puncts):\n",
    "    x = str(x)\n",
    "    for punct in puncts:\n",
    "        if punct in x: \n",
    "            x = x.replace(punct, f' {punct} ')\n",
    "    return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "EMBEDDING_FILES = [\n",
    "    '../input/pickled-crawl300d2m-for-kernel-competitions/crawl-300d-2M.pkl',\n",
    "    '../input/pickled-glove840b300d-for-10sec-loading/glove.840B.300d.pkl'\n",
    "]\n",
    "N_EMBS_LSTM = 328444 # we need to build an emb matrix with the same shape as the old one"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_embeddings_new(path):\n",
    "    with open(path, 'rb') as f:\n",
    "        emb_arr = pickle.load(f)\n",
    "        #return dict(get_coefs(*line.strip().split(' ')) for line in f)\n",
    "    return emb_arr\n",
    "\n",
    "\n",
    "def build_matrix_new(word_index, path):\n",
    "    embedding_index = load_embeddings_new(path)\n",
    "    #embedding_matrix = np.zeros((len(word_index) + 1, 300)) # we need to build a matrix with the same shape so we can swap the weights\n",
    "    # otherwise 'ValueError: Layer weight shape (328444, 600) not compatible with provided weight shape (75216, 600)'\n",
    "    embedding_matrix = np.zeros((N_EMBS_LSTM, 300))\n",
    "    for word, i in word_index.items():\n",
    "        try:\n",
    "            embedding_matrix[i] = embedding_index[word]\n",
    "        except KeyError:\n",
    "            pass\n",
    "    return embedding_matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "def preprocess_lstm():\n",
    "    start = time.time()\n",
    "\n",
    "    X_test_lstm = test_df['comment_text'].astype(str)\n",
    "    X_test_lstm = X_test_lstm.progress_apply(lambda x: clean_text_lstm(x, CHARS_TO_ISOLATE_LSTM))\n",
    "\n",
    "    # loading tokenizer\n",
    "    with open(os.path.join(MODEL_PATHS_LSTM[0],'tokenizer.pickle'), 'rb') as handle:\n",
    "        tokenizer = pickle.load(handle)\n",
    "\n",
    "    X_test_lstm = tokenizer.texts_to_sequences(X_test_lstm)\n",
    "    X_test_lstm = sequence.pad_sequences(X_test_lstm, maxlen=MAX_LEN_LSTM)\n",
    "\n",
    "    print(f'{time.time() - start:.2f} seconds')    \n",
    "    return X_test_lstm    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "def preprocess_lstm_emb():\n",
    "    start = time.time()\n",
    "\n",
    "    X_test_lstm = test_df['comment_text'].astype(str)\n",
    "    X_test_lstm = X_test_lstm.progress_apply(lambda x: clean_text_lstm(x, CHARS_TO_ISOLATE_LSTM))\n",
    "\n",
    "    # creating tokenizer\n",
    "    tokenizer = text.Tokenizer(filters=CHARS_TO_REMOVE_LSTM)\n",
    "    tokenizer.fit_on_texts(list(X_test_lstm))    \n",
    "\n",
    "    X_test_lstm = tokenizer.texts_to_sequences(X_test_lstm)\n",
    "    X_test_lstm = sequence.pad_sequences(X_test_lstm, maxlen=MAX_LEN_LSTM)\n",
    "    \n",
    "    emb_matrix_new = np.concatenate(\n",
    "    [build_matrix_new(tokenizer.word_index, f) for f in EMBEDDING_FILES], axis=-1)\n",
    "\n",
    "    print(f'{time.time() - start:.2f} seconds')    \n",
    "    return X_test_lstm, emb_matrix_new"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "def lstm_predict_emb(model_path, X_test, embedding_matrix_new, batch_size=BATCH_SIZE_LSTM, checkpoints=[], weights=[]):\n",
    "    start = time.time()\n",
    "    \n",
    "    all_test_pred = []\n",
    "    for checkpoint in checkpoints:\n",
    "    \n",
    "        # LOADING MODEL\n",
    "        model = load_model(os.path.join(model_path, checkpoint))\n",
    "        model.layers[1].set_weights([embedding_matrix_new]) #\n",
    "\n",
    "        # PREDICTING\n",
    "        test_pred = model.predict(X_test, batch_size=BATCH_SIZE_LSTM)[0].flatten()\n",
    "        all_test_pred.append(test_pred)\n",
    "        \n",
    "    test_pred_lstm = np.average(all_test_pred, weights=weights, axis=0)\n",
    "    \n",
    "    print(f'{time.time() - start:.2f} seconds')\n",
    "    return test_pred_lstm   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "def lstm_predict(model_path, X_test, batch_size=BATCH_SIZE_LSTM, checkpoints=[], weights=[]):\n",
    "    start = time.time()\n",
    "    \n",
    "    all_test_pred = []\n",
    "    for checkpoint in checkpoints:\n",
    "    \n",
    "        # LOADING MODEL\n",
    "        model = load_model(os.path.join(model_path, checkpoint))\n",
    "\n",
    "        # PREDICTING\n",
    "        test_pred = model.predict(X_test, batch_size=BATCH_SIZE_LSTM)[0].flatten()\n",
    "        all_test_pred.append(test_pred)\n",
    "        \n",
    "    test_pred_lstm = np.average(all_test_pred, weights=weights, axis=0)\n",
    "    \n",
    "    print(f'{time.time() - start:.2f} seconds')\n",
    "    return test_pred_lstm    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### FASTAI FUNCTIONS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "MAX_LEN_FASTAI = 300\n",
    "MAX_FEATURES_FASTAI = 410047 # it wont be used if we create new emb matrix\n",
    "LSTM_UNITS = 128\n",
    "DENSE_HIDDEN_UNITS = 4 * LSTM_UNITS\n",
    "NUM_AUX_TARGETS_FASTAI = 6 #['target', 'severe_toxicity', 'obscene', 'identity_attack', 'insult', 'threat']\n",
    "BATCH_SIZE_FASTAI = 512"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "class NeuralNet(nn.Module):\n",
    "    def __init__(self, embedding_matrix, num_aux_targets):\n",
    "        super(NeuralNet, self).__init__()\n",
    "        embed_size = embedding_matrix.shape[1]\n",
    "        \n",
    "        #self.embedding = nn.Embedding(MAX_FEATURES_FASTAI, embed_size) # better with embedding_matrix directly\n",
    "        self.embedding = nn.Embedding(embedding_matrix.shape[0]-1, embed_size) #\n",
    "        self.embedding.weight = nn.Parameter(torch.tensor(embedding_matrix, dtype=torch.float32))\n",
    "        self.embedding.weight.requires_grad = False\n",
    "        self.embedding_dropout = SpatialDropout(0.3)\n",
    "        \n",
    "        self.lstm1 = nn.LSTM(embed_size, LSTM_UNITS, bidirectional=True, batch_first=True)\n",
    "        self.lstm2 = nn.LSTM(LSTM_UNITS * 2, LSTM_UNITS, bidirectional=True, batch_first=True)\n",
    "    \n",
    "        self.linear1 = nn.Linear(DENSE_HIDDEN_UNITS, DENSE_HIDDEN_UNITS)\n",
    "        self.linear2 = nn.Linear(DENSE_HIDDEN_UNITS, DENSE_HIDDEN_UNITS)\n",
    "        \n",
    "        self.linear_out = nn.Linear(DENSE_HIDDEN_UNITS, 1)\n",
    "        self.linear_aux_out = nn.Linear(DENSE_HIDDEN_UNITS, num_aux_targets)\n",
    "        \n",
    "    def forward(self, x, lengths=None):\n",
    "        h_embedding = self.embedding(x.long())\n",
    "        h_embedding = self.embedding_dropout(h_embedding)\n",
    "        \n",
    "        h_lstm1, _ = self.lstm1(h_embedding)\n",
    "        h_lstm2, _ = self.lstm2(h_lstm1)\n",
    "        \n",
    "        # global average pooling\n",
    "        avg_pool = torch.mean(h_lstm2, 1)\n",
    "        # global max pooling\n",
    "        max_pool, _ = torch.max(h_lstm2, 1)\n",
    "        \n",
    "        h_conc = torch.cat((max_pool, avg_pool), 1)\n",
    "        h_conc_linear1  = F.relu(self.linear1(h_conc))\n",
    "        h_conc_linear2  = F.relu(self.linear2(h_conc))\n",
    "        \n",
    "        hidden = h_conc + h_conc_linear1 + h_conc_linear2\n",
    "        \n",
    "        result = self.linear_out(hidden)\n",
    "        aux_result = self.linear_aux_out(hidden)\n",
    "        out = torch.cat([result, aux_result], 1)\n",
    "        \n",
    "        return out\n",
    "    \n",
    "class SpatialDropout(nn.Dropout2d):\n",
    "    def forward(self, x):\n",
    "        x = x.unsqueeze(2)    # (N, T, 1, K)\n",
    "        x = x.permute(0, 3, 2, 1)  # (N, K, 1, T)\n",
    "        x = super(SpatialDropout, self).forward(x)  # (N, K, 1, T), some features are masked\n",
    "        x = x.permute(0, 3, 2, 1)  # (N, T, 1, K)\n",
    "        x = x.squeeze(2)  # (N, T, K)\n",
    "        return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "CRAWL_EMBEDDING_PATH = '../input/pickled-crawl300d2m-for-kernel-competitions/crawl-300d-2M.pkl'\n",
    "GLOVE_EMBEDDING_PATH = '../input/pickled-glove840b300d-for-10sec-loading/glove.840B.300d.pkl'\n",
    "\n",
    "def get_coefs(word, *arr):\n",
    "    return word, np.asarray(arr, dtype='float32')\n",
    "\n",
    "\n",
    "def load_embeddings(path):\n",
    "    with open(path,'rb') as f:\n",
    "        emb_arr = pickle.load(f)\n",
    "    return emb_arr\n",
    "\n",
    "\n",
    "def build_matrix(word_index, path, max_features):\n",
    "    embedding_index = load_embeddings(path)\n",
    "    embedding_matrix = np.zeros((max_features + 1, 300))\n",
    "    unknown_words = []\n",
    "    \n",
    "    for word, i in word_index.items():\n",
    "        if i <= max_features:\n",
    "            try:\n",
    "                embedding_matrix[i] = embedding_index[word]\n",
    "            except KeyError:\n",
    "                try:\n",
    "                    embedding_matrix[i] = embedding_index[word.lower()]\n",
    "                except KeyError:\n",
    "                    try:\n",
    "                        embedding_matrix[i] = embedding_index[word.title()]\n",
    "                    except KeyError:\n",
    "                        unknown_words.append(word)\n",
    "    return embedding_matrix, unknown_words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "SYMBOLS_TO_ISOLATE_FASTAI = '.,?!-;*\"…:—()%#$&_/@＼・ω+=”“[]^–>\\\\°<~•≠™ˈʊɒ∞§{}·τα❤☺ɡ|¢→̶`❥━┣┫┗Ｏ►★©―ɪ✔®\\x96\\x92●£♥➤´¹☕≈÷♡◐║▬′ɔː€۩۞†μ✒➥═☆ˌ◄½ʻπδηλσερνʃ✬ＳＵＰＥＲＩＴ☻±♍µº¾✓◾؟．⬅℅»Вав❣⋅¿¬♫ＣＭβ█▓▒░⇒⭐›¡₂₃❧▰▔◞▀▂▃▄▅▆▇↙γ̄″☹➡«φ⅓„✋：¥̲̅́∙‛◇✏▷❓❗¶˚˙）сиʿ✨。ɑ\\x80◕！％¯−ﬂﬁ₁²ʌ¼⁴⁄₄⌠♭✘╪▶☭✭♪☔☠♂☃☎✈✌✰❆☙○‣⚓年∎ℒ▪▙☏⅛ｃａｓǀ℮¸ｗ‚∼‖ℳ❄←☼⋆ʒ⊂、⅔¨͡๏⚾⚽Φ×θ￦？（℃⏩☮⚠月✊❌⭕▸■⇌☐☑⚡☄ǫ╭∩╮，例＞ʕɐ̣Δ₀✞┈╱╲▏▕┃╰▊▋╯┳┊≥☒↑☝ɹ✅☛♩☞ＡＪＢ◔◡↓♀⬆̱ℏ\\x91⠀ˤ╚↺⇤∏✾◦♬³の｜／∵∴√Ω¤☜▲↳▫‿⬇✧ｏｖｍ－２０８＇‰≤∕ˆ⚜☁'\n",
    "SYMBOLS_TO_DELETE_FASTAI = '\\n🍕\\r🐵😑\\xa0\\ue014\\t\\uf818\\uf04a\\xad😢🐶️\\uf0e0😜😎👊\\u200b\\u200e😁عدويهصقأناخلىبمغر😍💖💵Е👎😀😂\\u202a\\u202c🔥😄🏻💥ᴍʏʀᴇɴᴅᴏᴀᴋʜᴜʟᴛᴄᴘʙғᴊᴡɢ😋👏שלוםבי😱‼\\x81エンジ故障\\u2009🚌ᴵ͞🌟😊😳😧🙀😐😕\\u200f👍😮😃😘אעכח💩💯⛽🚄🏼ஜ😖ᴠ🚲‐😟😈💪🙏🎯🌹😇💔😡\\x7f👌ἐὶήιὲκἀίῃἴξ🙄Ｈ😠\\ufeff\\u2028😉😤⛺🙂\\u3000تحكسة👮💙فزط😏🍾🎉😞\\u2008🏾😅😭👻😥😔😓🏽🎆🍻🍽🎶🌺🤔😪\\x08‑🐰🐇🐱🙆😨🙃💕𝘊𝘦𝘳𝘢𝘵𝘰𝘤𝘺𝘴𝘪𝘧𝘮𝘣💗💚地獄谷улкнПоАН🐾🐕😆ה🔗🚽歌舞伎🙈😴🏿🤗🇺🇸мυтѕ⤵🏆🎃😩\\u200a🌠🐟💫💰💎эпрд\\x95🖐🙅⛲🍰🤐👆🙌\\u2002💛🙁👀🙊🙉\\u2004ˢᵒʳʸᴼᴷᴺʷᵗʰᵉᵘ\\x13🚬🤓\\ue602😵άοόςέὸתמדףנרךצט😒͝🆕👅👥👄🔄🔤👉👤👶👲🔛🎓\\uf0b7\\uf04c\\x9f\\x10成都😣⏺😌🤑🌏😯ех😲Ἰᾶὁ💞🚓🔔📚🏀👐\\u202d💤🍇\\ue613小土豆🏡❔⁉\\u202f👠》कर्मा🇹🇼🌸蔡英文🌞🎲レクサス😛外国人关系Сб💋💀🎄💜🤢َِьыгя不是\\x9c\\x9d🗑\\u2005💃📣👿༼つ༽😰ḷЗз▱ц￼🤣卖温哥华议会下降你失去所有的钱加拿大坏税骗子🐝ツ🎅\\x85🍺آإشء🎵🌎͟ἔ油别克🤡🤥😬🤧й\\u2003🚀🤴ʲшчИОРФДЯМюж😝🖑ὐύύ特殊作戦群щ💨圆明园קℐ🏈😺🌍⏏ệ🍔🐮🍁🍆🍑🌮🌯🤦\\u200d𝓒𝓲𝓿𝓵안영하세요ЖљКћ🍀😫🤤ῦ我出生在了可以说普通话汉语好极🎼🕺🍸🥂🗽🎇🎊🆘🤠👩🖒🚪天一家⚲\\u2006⚭⚆⬭⬯⏖新✀╌🇫🇷🇩🇪🇮🇬🇧😷🇨🇦ХШ🌐\\x1f杀鸡给猴看ʁ𝗪𝗵𝗲𝗻𝘆𝗼𝘂𝗿𝗮𝗹𝗶𝘇𝗯𝘁𝗰𝘀𝘅𝗽𝘄𝗱📺ϖ\\u2000үսᴦᎥһͺ\\u2007հ\\u2001ɩｙｅ൦ｌƽｈ𝐓𝐡𝐞𝐫𝐮𝐝𝐚𝐃𝐜𝐩𝐭𝐢𝐨𝐧Ƅᴨןᑯ໐ΤᏧ௦Іᴑ܁𝐬𝐰𝐲𝐛𝐦𝐯𝐑𝐙𝐣𝐇𝐂𝐘𝟎ԜТᗞ౦〔Ꭻ𝐳𝐔𝐱𝟔𝟓𝐅🐋ﬃ💘💓ё𝘥𝘯𝘶💐🌋🌄🌅𝙬𝙖𝙨𝙤𝙣𝙡𝙮𝙘𝙠𝙚𝙙𝙜𝙧𝙥𝙩𝙪𝙗𝙞𝙝𝙛👺🐷ℋ𝐀𝐥𝐪🚶𝙢Ἱ🤘ͦ💸ج패티Ｗ𝙇ᵻ👂👃ɜ🎫\\uf0a7БУі🚢🚂ગુજરાતીῆ🏃𝓬𝓻𝓴𝓮𝓽𝓼☘﴾̯﴿₽\\ue807𝑻𝒆𝒍𝒕𝒉𝒓𝒖𝒂𝒏𝒅𝒔𝒎𝒗𝒊👽😙\\u200cЛ‒🎾👹⎌🏒⛸公寓养宠物吗🏄🐀🚑🤷操美𝒑𝒚𝒐𝑴🤙🐒欢迎来到阿拉斯ספ𝙫🐈𝒌𝙊𝙭𝙆𝙋𝙍𝘼𝙅ﷻ🦄巨收赢得白鬼愤怒要买额ẽ🚗🐳𝟏𝐟𝟖𝟑𝟕𝒄𝟗𝐠𝙄𝙃👇锟斤拷𝗢𝟳𝟱𝟬⦁マルハニチロ株式社⛷한국어ㄸㅓ니͜ʖ𝘿𝙔₵𝒩ℯ𝒾𝓁𝒶𝓉𝓇𝓊𝓃𝓈𝓅ℴ𝒻𝒽𝓀𝓌𝒸𝓎𝙏ζ𝙟𝘃𝗺𝟮𝟭𝟯𝟲👋🦊多伦🐽🎻🎹⛓🏹🍷🦆为和中友谊祝贺与其想象对法如直接问用自己猜本传教士没积唯认识基督徒曾经让相信耶稣复活死怪他但当们聊些政治题时候战胜因圣把全堂结婚孩恐惧且栗谓这样还♾🎸🤕🤒⛑🎁批判检讨🏝🦁🙋😶쥐스탱트뤼도석유가격인상이경제황을렵게만들지않록잘관리해야합다캐나에서대마초와화약금의품런성분갈때는반드시허된사용🔫👁凸ὰ💲🗯𝙈Ἄ𝒇𝒈𝒘𝒃𝑬𝑶𝕾𝖙𝖗𝖆𝖎𝖌𝖍𝖕𝖊𝖔𝖑𝖉𝖓𝖐𝖜𝖞𝖚𝖇𝕿𝖘𝖄𝖛𝖒𝖋𝖂𝕴𝖟𝖈𝕸👑🚿💡知彼百\\uf005𝙀𝒛𝑲𝑳𝑾𝒋𝟒😦𝙒𝘾𝘽🏐𝘩𝘨ὼṑ𝑱𝑹𝑫𝑵𝑪🇰🇵👾ᓇᒧᔭᐃᐧᐦᑳᐨᓃᓂᑲᐸᑭᑎᓀᐣ🐄🎈🔨🐎🤞🐸💟🎰🌝🛳点击查版🍭𝑥𝑦𝑧ＮＧ👣\\uf020っ🏉ф💭🎥Ξ🐴👨🤳🦍\\x0b🍩𝑯𝒒😗𝟐🏂👳🍗🕉🐲چی𝑮𝗕𝗴🍒ꜥⲣⲏ🐑⏰鉄リ事件ї💊「」\\uf203\\uf09a\\uf222\\ue608\\uf202\\uf099\\uf469\\ue607\\uf410\\ue600燻製シ虚偽屁理屈Г𝑩𝑰𝒀𝑺🌤𝗳𝗜𝗙𝗦𝗧🍊ὺἈἡχῖΛ⤏🇳𝒙ψՁմեռայինրւդձ冬至ὀ𝒁🔹🤚🍎𝑷🐂💅𝘬𝘱𝘸𝘷𝘐𝘭𝘓𝘖𝘹𝘲𝘫کΒώ💢ΜΟΝΑΕ🇱♲𝝈↴💒⊘Ȼ🚴🖕🖤🥘📍👈➕🚫🎨🌑🐻𝐎𝐍𝐊𝑭🤖🎎😼🕷ｇｒｎｔｉｄｕｆｂｋ𝟰🇴🇭🇻🇲𝗞𝗭𝗘𝗤👼📉🍟🍦🌈🔭《🐊🐍\\uf10aლڡ🐦\\U0001f92f\\U0001f92a🐡💳ἱ🙇𝗸𝗟𝗠𝗷🥜さようなら🔼'\n",
    "\n",
    "isolate_dict = {ord(c):f' {c} ' for c in SYMBOLS_TO_ISOLATE_FASTAI}\n",
    "remove_dict = {ord(c):f'' for c in SYMBOLS_TO_DELETE_FASTAI}\n",
    "\n",
    "\n",
    "def handle_punctuation(x):\n",
    "    x = x.translate(remove_dict)\n",
    "    x = x.translate(isolate_dict)\n",
    "    return x\n",
    "\n",
    "def handle_contractions(x):\n",
    "    from nltk.tokenize.treebank import TreebankWordTokenizer\n",
    "    tokenizer = TreebankWordTokenizer()\n",
    "    x = tokenizer.tokenize(x)\n",
    "    return x\n",
    "\n",
    "def fix_quote(x):\n",
    "    x = [x_[1:] if x_.startswith(\"'\") else x_ for x_ in x]\n",
    "    x = ' '.join(x)\n",
    "    return x\n",
    "\n",
    "def preprocess(x):\n",
    "    x = handle_punctuation(x)\n",
    "    x = handle_contractions(x)\n",
    "    x = fix_quote(x)\n",
    "    return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "def preprocess_fastai():\n",
    "    start = time.time()\n",
    "    \n",
    "    with open(os.path.join(MODEL_PATHS_FASTAI[0], 'tokenizer.pickle'), 'rb') as handle:\n",
    "        tokenizer = pickle.load(handle)\n",
    "\n",
    "    X_test_fastai = test_df['comment_text'].progress_apply(lambda x: preprocess(x))\n",
    "    X_test_fastai = tokenizer.texts_to_sequences(X_test_fastai)\n",
    "    X_test_fastai = sequence.pad_sequences(X_test_fastai, maxlen=MAX_LEN_FASTAI)\n",
    "\n",
    "    print(f'{time.time() - start:.2f} seconds')    \n",
    "\n",
    "    return X_test_fastai"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "def preprocess_fastai_emb():\n",
    "    start = time.time()\n",
    "\n",
    "    X_test_fastai = test_df['comment_text'].progress_apply(lambda x: preprocess(x))\n",
    "    \n",
    "    # creating new tokenizer\n",
    "    tokenizer = text.Tokenizer(num_words = MAX_FEATURES_FASTAI, filters='', lower=False)\n",
    "    tokenizer.fit_on_texts(list(X_test_fastai))\n",
    "\n",
    "    X_test_fastai = tokenizer.texts_to_sequences(X_test_fastai)\n",
    "    X_test_fastai = sequence.pad_sequences(X_test_fastai, maxlen=MAX_LEN_FASTAI)\n",
    "    \n",
    "    max_features_fastai = min(MAX_FEATURES_FASTAI, len(tokenizer.word_index)) # its gonna be len(tokenizer.word_index) if we only fit on test    \n",
    "    crawl_matrix, unknown_words_crawl = build_matrix(tokenizer.word_index, CRAWL_EMBEDDING_PATH, max_features_fastai)\n",
    "    glove_matrix, unknown_words_glove = build_matrix(tokenizer.word_index, GLOVE_EMBEDDING_PATH, max_features_fastai)\n",
    "    emb_matrix_new = np.concatenate([crawl_matrix, glove_matrix], axis=-1)\n",
    "\n",
    "    print(f'{time.time() - start:.2f} seconds')    \n",
    "\n",
    "    return X_test_fastai, emb_matrix_new"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "def fastai_predict_emb(model_path, X_test, embedding_matrix_new, batch_size=BATCH_SIZE_FASTAI, checkpoints=[], weights=[]):\n",
    "    start = time.time()\n",
    "    \n",
    "    all_test_pred = []\n",
    "    for checkpoint in checkpoints:\n",
    "    \n",
    "        # LOADING MODEL        \n",
    "        model = NeuralNet(embedding_matrix_new, NUM_AUX_TARGETS_FASTAI)\n",
    "        state = torch.load(os.path.join(model_path, checkpoint)) # state_dict saved\n",
    "        state['embedding.weight'] = torch.tensor(embedding_matrix_new)\n",
    "        model.load_state_dict(state)\n",
    "\n",
    "        # PREDICTING\n",
    "        model.cuda()\n",
    "        model.eval()\n",
    "\n",
    "        test_preds = np.zeros((len(X_test)))\n",
    "        test_tensor = torch.utils.data.TensorDataset(torch.tensor(X_test, dtype=torch.long))\n",
    "        test_loader = torch.utils.data.DataLoader(test_tensor, batch_size=batch_size, shuffle=False)\n",
    "        for i, (x_batch,) in enumerate(tqdm(test_loader)):\n",
    "            pred = model(x_batch.cuda())    \n",
    "            test_preds[i * batch_size:(i + 1) * batch_size] = pred[:, 0].detach().cpu().squeeze().numpy()\n",
    "        test_pred = torch.sigmoid(torch.tensor(test_preds)).numpy()        \n",
    "        all_test_pred.append(test_pred)\n",
    "        \n",
    "    test_pred_fastai = np.average(all_test_pred, weights=weights, axis=0)\n",
    "    \n",
    "    print(f'{time.time() - start:.2f} seconds')\n",
    "    return test_pred_fastai    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "def fastai_predict(model_path, X_test, batch_size=BATCH_SIZE_FASTAI, checkpoints=[], weights=[]):\n",
    "    start = time.time()\n",
    "    \n",
    "    all_test_pred = []\n",
    "    for checkpoint in checkpoints:\n",
    "    \n",
    "        # LOADING MODEL\n",
    "        state = torch.load(os.path.join(model_path, checkpoint)) # state_dict saved\n",
    "        embedding_matrix = state['embedding.weight']\n",
    "        model = NeuralNet(embedding_matrix, NUM_AUX_TARGETS_FASTAI)\n",
    "        model.load_state_dict(state)\n",
    "\n",
    "        # PREDICTING\n",
    "        model.cuda()\n",
    "        model.eval()\n",
    "\n",
    "        test_preds = np.zeros((len(X_test)))\n",
    "        test_tensor = torch.utils.data.TensorDataset(torch.tensor(X_test, dtype=torch.long))\n",
    "        test_loader = torch.utils.data.DataLoader(test_tensor, batch_size=batch_size, shuffle=False)\n",
    "        for i, (x_batch,) in enumerate(tqdm(test_loader)):\n",
    "            pred = model(x_batch.cuda())    \n",
    "            test_preds[i * batch_size:(i + 1) * batch_size] = pred[:, 0].detach().cpu().squeeze().numpy()\n",
    "        test_pred = torch.sigmoid(torch.tensor(test_preds)).numpy()        \n",
    "        all_test_pred.append(test_pred)\n",
    "        \n",
    "    test_pred_fastai = np.average(all_test_pred, weights=weights, axis=0)\n",
    "    \n",
    "    print(f'{time.time() - start:.2f} seconds')\n",
    "    return test_pred_fastai    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### GPT2 FUNCTIONS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "GPT2_PRETRAINED_MODEL_PATH = '../input/gpt2-models' # contains tokenizer AND pytorch converted model\n",
    "\n",
    "MAX_SEQUENCE_LENGTH_GPT2 = 220\n",
    "NUM_LABELS_GPT2 = 6\n",
    "BATCH_SIZE_GPT2 = 32\n",
    "\n",
    "DEVICE = torch.device('cuda')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "def preprocess_gpt2():\n",
    "    start = time.time()\n",
    "    \n",
    "    # grab tokenizer from pretrained\n",
    "    tokenizer = GPT2Tokenizer.from_pretrained(GPT2_PRETRAINED_MODEL_PATH)\n",
    "    \n",
    "    # converting the lines to GPT2 format\n",
    "    def convert_lines_gpt2(example, max_seq_length, tokenizer):\n",
    "        all_tokens = []\n",
    "        for text in tqdm(example):\n",
    "            tokens_a = tokenizer.tokenize(text)    \n",
    "            if len(tokens_a) > MAX_SEQUENCE_LENGTH_GPT2:\n",
    "                tokens_a = tokens_a[-MAX_SEQUENCE_LENGTH_GPT2:]\n",
    "            one_token = tokenizer.convert_tokens_to_ids(tokens_a) + [0]*(MAX_SEQUENCE_LENGTH_GPT2 - len(tokens_a))\n",
    "            all_tokens.append(one_token)\n",
    "        return np.array(all_tokens)\n",
    "    \n",
    "    # test read & conversion to GPT2 format\n",
    "    #test_df['comment_text'] = test_df['comment_text'].astype(str) \n",
    "    X_test_gpt2 = convert_lines_gpt2(test_df[\"comment_text\"].astype(str).fillna(\"DUMMY_VALUE\"), MAX_SEQUENCE_LENGTH_GPT2, tokenizer)\n",
    "\n",
    "    print(f'{time.time() - start:.2f} seconds')\n",
    "\n",
    "    return X_test_gpt2 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "def gpt2_predict(model_path, X_test, batch_size=BATCH_SIZE_GPT2):\n",
    "    start = time.time()  \n",
    "    \n",
    "    # loading gpt2 pytorch model (looks for pytorch_model.bin AND config.json)\n",
    "    #model = GPT2ClassificationHeadModel.from_pretrained(model_path, clf_dropout=0.4, n_class=NUM_LABEL_GPT2) # WORKS even when pytorch_model.bin only contains state_dict\n",
    "    model = GPT2ClassificationHeadModel.from_pretrained(GPT2_PRETRAINED_MODEL_PATH, clf_dropout=0.4, n_class=NUM_LABELS_GPT2) # ? SHOULD WORK\n",
    "\n",
    "    # loading model state \n",
    "    model.load_state_dict(torch.load(os.path.join(model_path, 'pytorch_model.bin'))['state_dict'])\n",
    "\n",
    "    # setting model in eval mode\n",
    "    model.to(DEVICE)\n",
    "    for param in model.parameters():\n",
    "        param.requires_grad=False\n",
    "    model.eval()\n",
    "    \n",
    "    # predicting\n",
    "    test_preds = np.zeros((len(X_test)))\n",
    "    test = torch.utils.data.TensorDataset(torch.tensor(X_test, dtype=torch.long))\n",
    "    test_loader = torch.utils.data.DataLoader(test, batch_size=batch_size, shuffle=False)\n",
    "    for i,(x_batch,) in enumerate(tqdm(test_loader)):\n",
    "        pred = model(x_batch.to(DEVICE))\n",
    "        test_preds[i*batch_size:(i+1)*batch_size]=pred[:,0].detach().cpu().squeeze().numpy()\n",
    "    test_pred = torch.sigmoid(torch.tensor(test_preds)).numpy().ravel()\n",
    "    \n",
    "    print(f'{time.time() - start:.2f} seconds')\n",
    "                          \n",
    "    return test_pred    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## ENSEMBLE"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### LSTM"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### ~ 10 s"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8294426abea84d7d86349b7e4d12af96",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=97320), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "32.32 seconds\n"
     ]
    }
   ],
   "source": [
    "#X_test_lstm = preprocess_lstm()\n",
    "X_test_lstm, emb_matrix_new = preprocess_lstm_emb()\n",
    "\n",
    "#6.50 seconds\n",
    "#29.99 seconds"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### ~ 2 m 10 s *(~25s per ckpt)*"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['../input/d-lstm-m-n2-e6-s456-2']"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "MODEL_PATHS_LSTM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.6/importlib/_bootstrap.py:219: RuntimeWarning: numpy.ufunc size changed, may indicate binary incompatibility. Expected 192 from C header, got 216 from PyObject\n",
      "  return f(*args, **kwds)\n",
      "/opt/conda/lib/python3.6/importlib/_bootstrap.py:219: RuntimeWarning: numpy.ufunc size changed, may indicate binary incompatibility. Expected 192 from C header, got 216 from PyObject\n",
      "  return f(*args, **kwds)\n",
      "/opt/conda/lib/python3.6/importlib/_bootstrap.py:219: RuntimeWarning: numpy.ufunc size changed, may indicate binary incompatibility. Expected 192 from C header, got 216 from PyObject\n",
      "  return f(*args, **kwds)\n",
      "/opt/conda/lib/python3.6/importlib/_bootstrap.py:219: ImportWarning: can't resolve package from __spec__ or __package__, falling back on __name__ and __path__\n",
      "  return f(*args, **kwds)\n",
      "/opt/conda/lib/python3.6/site-packages/keras/engine/saving.py:251: DeprecationWarning: The truth value of an empty array is ambiguous. Returning False, but in future this will result in an error. Use `array.size > 0` to check that an array is not empty.\n",
      "  if weight_names:\n",
      "/opt/conda/lib/python3.6/site-packages/keras/engine/saving.py:251: DeprecationWarning: The truth value of an empty array is ambiguous. Returning False, but in future this will result in an error. Use `array.size > 0` to check that an array is not empty.\n",
      "  if weight_names:\n",
      "/opt/conda/lib/python3.6/site-packages/keras/engine/saving.py:251: DeprecationWarning: The truth value of an empty array is ambiguous. Returning False, but in future this will result in an error. Use `array.size > 0` to check that an array is not empty.\n",
      "  if weight_names:\n",
      "/opt/conda/lib/python3.6/site-packages/keras/engine/saving.py:251: DeprecationWarning: The truth value of an empty array is ambiguous. Returning False, but in future this will result in an error. Use `array.size > 0` to check that an array is not empty.\n",
      "  if weight_names:\n",
      "/opt/conda/lib/python3.6/site-packages/keras/engine/saving.py:251: DeprecationWarning: The truth value of an empty array is ambiguous. Returning False, but in future this will result in an error. Use `array.size > 0` to check that an array is not empty.\n",
      "  if weight_names:\n",
      "/opt/conda/lib/python3.6/site-packages/keras/engine/saving.py:251: DeprecationWarning: The truth value of an empty array is ambiguous. Returning False, but in future this will result in an error. Use `array.size > 0` to check that an array is not empty.\n",
      "  if weight_names:\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "150.80 seconds\n"
     ]
    }
   ],
   "source": [
    "predictions_lstm = []\n",
    "predictions_lstm += [\n",
    "                        #lstm_predict('../input/d-lstm-m-n2-e6-s456-2',\n",
    "                        lstm_predict_emb('../input/d-lstm-m-n2-e6-s456-2', #                                                                                                                                                                                    \n",
    "                                     X_test_lstm,\n",
    "                                     emb_matrix_new, #\n",
    "                                     checkpoints=['LSTM_1_4.h5',\n",
    "                                                  'LSTM_1_5.h5',\n",
    "                                                  'LSTM_1_6.h5',\n",
    "                                                  'LSTM_2_4.h5',\n",
    "                                                  'LSTM_2_5.h5',\n",
    "                                                  'LSTM_2_6.h5',\n",
    "                                                 ],\n",
    "                                     weights=[1,1,1,1,1,1],\n",
    "                                     )\n",
    "                     ]\n",
    "prediction_lstm = np.average(predictions_lstm, weights=None, axis=0)\n",
    "\n",
    "#129.14 seconds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "from numba import cuda\n",
    "cuda.select_device(0)\n",
    "cuda.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### FASTAI"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### ~ 35 s"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c8f68a767f0240dbbca7fb2c41e70d13",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=97320), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.6/importlib/_bootstrap.py:219: RuntimeWarning: numpy.ufunc size changed, may indicate binary incompatibility. Expected 192 from C header, got 216 from PyObject\n",
      "  return f(*args, **kwds)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "60.53 seconds\n"
     ]
    }
   ],
   "source": [
    "#X_test_fastai = preprocess_fastai()\n",
    "X_test_fastai, emb_matrix_new = preprocess_fastai_emb() #\n",
    "\n",
    "#34.10 seconds"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### ~ 1 m 30 s *(~20s per ckpt)*"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['../input/d-fastai-n5-e4-s4']"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "MODEL_PATHS_FASTAI"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "dafcbd098cc240078e2d4318566d8481",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=191), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3f0ff935de2c427b8a143b32eeaf9ab1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=191), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "08c872d8316d42f58974dd40e5ab02d3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=191), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ccc9c2260ed64be4a03109cf7fbf528a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=191), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7c3aa92507974b18a984c4e47d5b6bf0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=191), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "92.48 seconds\n"
     ]
    }
   ],
   "source": [
    "predictions_fastai = []\n",
    "predictions_fastai += [\n",
    "                       #fastai_predict('../input/d-fastai-n5-e4-s4',\n",
    "                       fastai_predict_emb('../input/d-fastai-n5-e4-s4', #                                                                        \n",
    "                                     X_test_fastai,\n",
    "                                     emb_matrix_new, #\n",
    "                                     checkpoints=['state_1_4.bin',\n",
    "                                                  'state_2_4.bin',\n",
    "                                                  'state_3_4.bin',\n",
    "                                                  'state_4_4.bin',\n",
    "                                                  'state_5_4.bin',\n",
    "                                                 ],\n",
    "                                     weights=[1,1,1,1,1],\n",
    "                                     )\n",
    "                     ]\n",
    "prediction_fastai = np.average(predictions_fastai, weights=None, axis=0)\n",
    "\n",
    "#85.71 seconds\n",
    "#107.72 seconds"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### GPT2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### ~ 45 s"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "../input/gpt2-pytorch/pytorch-pretrained-bert-master/pytorch-pretrained-BERT-master/pytorch_pretrained_bert/tokenization_gpt2.py:146: ResourceWarning: unclosed file <_io.TextIOWrapper name='../input/gpt2-models/vocab.json' mode='r' encoding='UTF-8'>\n",
      "  self.encoder = json.load(open(vocab_file))\n",
      "../input/gpt2-pytorch/pytorch-pretrained-bert-master/pytorch-pretrained-BERT-master/pytorch_pretrained_bert/tokenization_gpt2.py:151: ResourceWarning: unclosed file <_io.TextIOWrapper name='../input/gpt2-models/merges.txt' mode='r' encoding='utf-8'>\n",
      "  bpe_data = open(merges_file, encoding='utf-8').read().split('\\n')[1:-1]\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "68e8b4fbada6439eb09bf542ba87203d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=97320), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "40.44 seconds\n"
     ]
    }
   ],
   "source": [
    "X_test_gpt2 = preprocess_gpt2()\n",
    "\n",
    "#42.42 seconds"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### ~ 13 m 30 s *(~14m per model)*"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5ee23937d3344ee2bc53932ab77c3a81",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=3042), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "791.91 seconds\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "30067b3820914011aaae02eb11fd2540",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=3042), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "predictions_gpt2 = [gpt2_predict(path, X_test_gpt2) for path in MODEL_PATHS_GPT2]\n",
    "prediction_gpt2 = np.average(predictions_gpt2, weights=None, axis=0)\n",
    "\n",
    "#789.48 seconds\n",
    "#793.64 seconds #cuda 'closed'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### BERT"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### ~ 2 m"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "056107b81ff54a96a63123ac8ac996b0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=97320), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "119.45 seconds\n"
     ]
    }
   ],
   "source": [
    "X_test_bert = preprocess_bert()\n",
    "\n",
    "#116.52 seconds"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### ~ 12 m *(~12m per model)*"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "990b3a4cbc2046da8771ff906ab7e6c6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=3042), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "722.42 seconds\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "04d0b271d46d4f55b4040b913d80dc19",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=3042), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "721.90 seconds\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "33189894dc1b4f4f9a6bd9f4f826670a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=3042), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "predictions_bert = [bert_predict(path, X_test_bert) for path in MODEL_PATHS_BERT]\n",
    "prediction_bert = np.average(predictions_bert, weights=None, axis=0)\n",
    "\n",
    "#719.34 seconds"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## ENSEMBLE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "predictions = []\n",
    "predictions.append(prediction_bert)\n",
    "predictions.append(prediction_lstm)\n",
    "predictions.append(prediction_fastai)\n",
    "predictions.append(prediction_gpt2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[4, 0.75, 0.75, 2]"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "WEIGHT_BERT = 4   # 4 models\n",
    "WEIGHT_LSTM = .75   # 1 model \n",
    "WEIGHT_FASTAI = .75   # 1 model\n",
    "WEIGHT_GPT2 = 2   # 1 model\n",
    "\n",
    "weights = [WEIGHT_BERT, WEIGHT_LSTM, WEIGHT_FASTAI, WEIGHT_GPT2]\n",
    "weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "prediction = np.average(predictions,\n",
    "                        weights=weights,\n",
    "                        axis=0,\n",
    "                       )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>prediction</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>7097320</td>\n",
       "      <td>0.008471</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>7097321</td>\n",
       "      <td>0.065478</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>7097322</td>\n",
       "      <td>0.263842</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>7097323</td>\n",
       "      <td>0.058740</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>7097324</td>\n",
       "      <td>0.010455</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>7097325</td>\n",
       "      <td>0.008007</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>7097326</td>\n",
       "      <td>0.801430</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>7097327</td>\n",
       "      <td>0.487713</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>7097328</td>\n",
       "      <td>0.005724</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>7097329</td>\n",
       "      <td>0.047942</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        id  prediction\n",
       "0  7097320    0.008471\n",
       "1  7097321    0.065478\n",
       "2  7097322    0.263842\n",
       "3  7097323    0.058740\n",
       "4  7097324    0.010455\n",
       "5  7097325    0.008007\n",
       "6  7097326    0.801430\n",
       "7  7097327    0.487713\n",
       "8  7097328    0.005724\n",
       "9  7097329    0.047942"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "submission = pd.DataFrame.from_dict({\n",
    "    'id': test_df['id'],\n",
    "    'prediction': prediction,\n",
    "})\n",
    "\n",
    "submission.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "submission.to_csv('submission.csv', index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "state": {
     "0355ef841b2b4f27b38c00d11ae02a7a": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.1.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.1.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.1.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "04d0b271d46d4f55b4040b913d80dc19": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.4.0",
      "model_name": "HBoxModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.4.0",
       "_model_name": "HBoxModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.4.0",
       "_view_name": "HBoxView",
       "box_style": "",
       "children": [
        "IPY_MODEL_32c11a55321f4043b446f7c13ee143a8",
        "IPY_MODEL_e2204e1a8ba640c990e7d3e6ae8712df"
       ],
       "layout": "IPY_MODEL_cdd6bfd4fd0744ef9382aa7735240c67"
      }
     },
     "056107b81ff54a96a63123ac8ac996b0": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.4.0",
      "model_name": "HBoxModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.4.0",
       "_model_name": "HBoxModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.4.0",
       "_view_name": "HBoxView",
       "box_style": "",
       "children": [
        "IPY_MODEL_5237cd298e0646f9ab31a3c62faaf55d",
        "IPY_MODEL_4d935464d2de4631a542fb1dd5aaaab5"
       ],
       "layout": "IPY_MODEL_997b0334229443c5b908d6682c55fb83"
      }
     },
     "0761a8a30eec4af5bec9d8967a3664e6": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.1.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.1.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.1.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "08c872d8316d42f58974dd40e5ab02d3": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.4.0",
      "model_name": "HBoxModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.4.0",
       "_model_name": "HBoxModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.4.0",
       "_view_name": "HBoxView",
       "box_style": "",
       "children": [
        "IPY_MODEL_12418bd2679345da8ca3aae3c2d9a0b1",
        "IPY_MODEL_30c587e3b66843bd974bb201cad08474"
       ],
       "layout": "IPY_MODEL_c24ae4ee94f64d0baced954f2d9efc7d"
      }
     },
     "0a767be83d004c54a5888bedd3824e93": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.4.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.4.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.1.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "12418bd2679345da8ca3aae3c2d9a0b1": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.4.0",
      "model_name": "IntProgressModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.4.0",
       "_model_name": "IntProgressModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.4.0",
       "_view_name": "ProgressView",
       "bar_style": "success",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_995c3814a9bc4fc38c3aeb63a78988c5",
       "max": 191,
       "min": 0,
       "orientation": "horizontal",
       "style": "IPY_MODEL_d4b3b329b7c64fa8b7f5ca30a156fc7b",
       "value": 191
      }
     },
     "13e351659fb841a68e0e757ea19ffaa4": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.1.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.1.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.1.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "166164f4095f43d482ba3095acc1cb58": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.4.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.4.0",
       "_model_name": "ProgressStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.1.0",
       "_view_name": "StyleView",
       "bar_color": null,
       "description_width": ""
      }
     },
     "176dfbfb599c45a186238e5d4dda40f1": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.4.0",
      "model_name": "IntProgressModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.4.0",
       "_model_name": "IntProgressModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.4.0",
       "_view_name": "ProgressView",
       "bar_style": "success",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_59e0964e94ee48beaff3e6179d90d144",
       "max": 191,
       "min": 0,
       "orientation": "horizontal",
       "style": "IPY_MODEL_46df9ac942ea4068b7331ea17b01d858",
       "value": 191
      }
     },
     "1cceed0d47dc4706868a6f3c31cd41ef": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.4.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.4.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.4.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_b02667e651314e77a12bc76a5c5715c1",
       "placeholder": "​",
       "style": "IPY_MODEL_cc83ecbdc8e94eeb920f87c900ed29d5",
       "value": "100% 191/191 [00:15&lt;00:00, 14.01it/s]"
      }
     },
     "1d60eb822b2d44ba83c7c5a2097319c3": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.1.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.1.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.1.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "1db89b547f5d49148bc8f93ee3c8e84d": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.4.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.4.0",
       "_model_name": "ProgressStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.1.0",
       "_view_name": "StyleView",
       "bar_color": null,
       "description_width": ""
      }
     },
     "1e7d2c2af12d4176a7113f8b300466e3": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.1.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.1.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.1.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "233fe49c7eac412fb126b82b4520e30e": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.1.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.1.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.1.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "236e3315a64d43da8505e5b75c54bc19": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.4.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.4.0",
       "_model_name": "ProgressStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.1.0",
       "_view_name": "StyleView",
       "bar_color": null,
       "description_width": ""
      }
     },
     "23a0c75f34fd43a2a2305cfabf3d3fb7": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.1.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.1.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.1.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "27c9c94914fd4ab9b8913783cdfce55e": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.4.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.4.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.1.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "29423601698749a4b1634a3806297656": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.1.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.1.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.1.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "2e39284260144cd0a78c7ec5b7b73f6a": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.4.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.4.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.4.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_ea9b8095fb794f9f9953f2e75d53902d",
       "placeholder": "​",
       "style": "IPY_MODEL_5359aae797ba4876be673cca638ad3b4",
       "value": "100% 191/191 [00:15&lt;00:00, 12.57it/s]"
      }
     },
     "30067b3820914011aaae02eb11fd2540": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.4.0",
      "model_name": "HBoxModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.4.0",
       "_model_name": "HBoxModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.4.0",
       "_view_name": "HBoxView",
       "box_style": "",
       "children": [
        "IPY_MODEL_5dd3eed7164d4a7a9c1835f043f1fb85",
        "IPY_MODEL_ba2d66d3909d49a6a2ceb905145f3332"
       ],
       "layout": "IPY_MODEL_c5f6d6f93e6940488aaf98e33970615c"
      }
     },
     "30c587e3b66843bd974bb201cad08474": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.4.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.4.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.4.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_951b8b9ff9174c90ac49a066610d4d6a",
       "placeholder": "​",
       "style": "IPY_MODEL_5e653eb65b094f70b1a29a861aba7d5c",
       "value": "100% 191/191 [00:15&lt;00:00, 12.23it/s]"
      }
     },
     "32c11a55321f4043b446f7c13ee143a8": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.4.0",
      "model_name": "IntProgressModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.4.0",
       "_model_name": "IntProgressModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.4.0",
       "_view_name": "ProgressView",
       "bar_style": "success",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_dfa3259f28004d3b9f77666e4c3c5dbc",
       "max": 3042,
       "min": 0,
       "orientation": "horizontal",
       "style": "IPY_MODEL_6eb4641dc1f7489682ce11857e14159d",
       "value": 3042
      }
     },
     "33189894dc1b4f4f9a6bd9f4f826670a": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.4.0",
      "model_name": "HBoxModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.4.0",
       "_model_name": "HBoxModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.4.0",
       "_view_name": "HBoxView",
       "box_style": "",
       "children": [
        "IPY_MODEL_dd44dc23ce554e7d86f30ddc51092fdb",
        "IPY_MODEL_749f7cd452954d8287f79c232ba09645"
       ],
       "layout": "IPY_MODEL_bc0a5b41821042119d495951f4b7f260"
      }
     },
     "3aeb13c3428c4866b5b52f0a04714e84": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.4.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.4.0",
       "_model_name": "ProgressStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.1.0",
       "_view_name": "StyleView",
       "bar_color": null,
       "description_width": ""
      }
     },
     "3eb3b58d9d3947a8b49c6cfce7ccd397": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.1.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.1.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.1.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "3f0ff935de2c427b8a143b32eeaf9ab1": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.4.0",
      "model_name": "HBoxModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.4.0",
       "_model_name": "HBoxModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.4.0",
       "_view_name": "HBoxView",
       "box_style": "",
       "children": [
        "IPY_MODEL_9cf7a492486e4dea8b4a4d1047879d87",
        "IPY_MODEL_2e39284260144cd0a78c7ec5b7b73f6a"
       ],
       "layout": "IPY_MODEL_bb2b10b0dea746f488033d08271907d8"
      }
     },
     "43670f9ef0bc4e19b389e77621dd711c": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.4.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.4.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.1.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "46df9ac942ea4068b7331ea17b01d858": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.4.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.4.0",
       "_model_name": "ProgressStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.1.0",
       "_view_name": "StyleView",
       "bar_color": null,
       "description_width": ""
      }
     },
     "4d935464d2de4631a542fb1dd5aaaab5": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.4.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.4.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.4.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_29423601698749a4b1634a3806297656",
       "placeholder": "​",
       "style": "IPY_MODEL_43670f9ef0bc4e19b389e77621dd711c",
       "value": "100% 97320/97320 [01:57&lt;00:00, 828.32it/s]"
      }
     },
     "5237cd298e0646f9ab31a3c62faaf55d": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.4.0",
      "model_name": "IntProgressModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.4.0",
       "_model_name": "IntProgressModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.4.0",
       "_view_name": "ProgressView",
       "bar_style": "success",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_bfacd5e004be4510ba65fec5d666b826",
       "max": 97320,
       "min": 0,
       "orientation": "horizontal",
       "style": "IPY_MODEL_bf78f23baaf04cb3888ad208d5df8f4c",
       "value": 97320
      }
     },
     "52ff598372604bb68a9f840b1b192782": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.1.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.1.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.1.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "5359aae797ba4876be673cca638ad3b4": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.4.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.4.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.1.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "54f58e6ced384e778d8e6aea71a5efc0": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.4.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.4.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.4.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_23a0c75f34fd43a2a2305cfabf3d3fb7",
       "placeholder": "​",
       "style": "IPY_MODEL_5c9ffb82ce0f417398144bddc49cb335",
       "value": "100% 3042/3042 [13:07&lt;00:00,  3.86it/s]"
      }
     },
     "57cad37f007449c38fa318f6a6fb8002": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.1.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.1.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.1.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "585781ef4a474abeaad44514e6f86301": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.1.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.1.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.1.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "5973648532ce406eb60d64587e5a4c08": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.4.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.4.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.4.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_3eb3b58d9d3947a8b49c6cfce7ccd397",
       "placeholder": "​",
       "style": "IPY_MODEL_dcd12b9e66e444c4af91f33b036287c1",
       "value": "100% 3042/3042 [11:57&lt;00:00,  4.24it/s]"
      }
     },
     "59e0964e94ee48beaff3e6179d90d144": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.1.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.1.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.1.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "5c9ffb82ce0f417398144bddc49cb335": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.4.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.4.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.1.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "5dd3eed7164d4a7a9c1835f043f1fb85": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.4.0",
      "model_name": "IntProgressModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.4.0",
       "_model_name": "IntProgressModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.4.0",
       "_view_name": "ProgressView",
       "bar_style": "",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_8501cf441cc048eb8f92de654a502c83",
       "max": 3042,
       "min": 0,
       "orientation": "horizontal",
       "style": "IPY_MODEL_aea3aa45b33446988bcc70b778a43708",
       "value": 2411
      }
     },
     "5e653eb65b094f70b1a29a861aba7d5c": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.4.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.4.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.1.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "5ee23937d3344ee2bc53932ab77c3a81": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.4.0",
      "model_name": "HBoxModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.4.0",
       "_model_name": "HBoxModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.4.0",
       "_view_name": "HBoxView",
       "box_style": "",
       "children": [
        "IPY_MODEL_824b969dbb794ebd916b280bd2e348f2",
        "IPY_MODEL_54f58e6ced384e778d8e6aea71a5efc0"
       ],
       "layout": "IPY_MODEL_d342f84f7b134d759d5bdad427a66635"
      }
     },
     "60206fd0636f400a883975decc79a1d6": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.4.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.4.0",
       "_model_name": "ProgressStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.1.0",
       "_view_name": "StyleView",
       "bar_color": null,
       "description_width": ""
      }
     },
     "60f7f9e47cfc4e5db49c3c07b7c1c5ca": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.1.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.1.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.1.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "652496a1b59946e6a595fabcb1fd4398": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.4.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.4.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.1.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "68e8b4fbada6439eb09bf542ba87203d": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.4.0",
      "model_name": "HBoxModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.4.0",
       "_model_name": "HBoxModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.4.0",
       "_view_name": "HBoxView",
       "box_style": "",
       "children": [
        "IPY_MODEL_87e27df3d3704904b18ccbe56d75da64",
        "IPY_MODEL_ddaa139661e24d64a9749d2a3cc1a699"
       ],
       "layout": "IPY_MODEL_e551aa6c29b84c1eb52a2a1d8f344ea7"
      }
     },
     "6d4a7039a9104fae954b6b11b8c5af95": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.4.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.4.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.1.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "6eb4641dc1f7489682ce11857e14159d": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.4.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.4.0",
       "_model_name": "ProgressStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.1.0",
       "_view_name": "StyleView",
       "bar_color": null,
       "description_width": ""
      }
     },
     "749f7cd452954d8287f79c232ba09645": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.4.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.4.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.4.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_ebdb8f8e01f346f08bee5d4fc05db636",
       "placeholder": "​",
       "style": "IPY_MODEL_652496a1b59946e6a595fabcb1fd4398",
       "value": " 17% 525/3042 [02:03&lt;09:53,  4.24it/s]"
      }
     },
     "7c3aa92507974b18a984c4e47d5b6bf0": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.4.0",
      "model_name": "HBoxModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.4.0",
       "_model_name": "HBoxModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.4.0",
       "_view_name": "HBoxView",
       "box_style": "",
       "children": [
        "IPY_MODEL_d49d3a34d3c249c28281d613e1edad2a",
        "IPY_MODEL_1cceed0d47dc4706868a6f3c31cd41ef"
       ],
       "layout": "IPY_MODEL_0355ef841b2b4f27b38c00d11ae02a7a"
      }
     },
     "824b969dbb794ebd916b280bd2e348f2": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.4.0",
      "model_name": "IntProgressModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.4.0",
       "_model_name": "IntProgressModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.4.0",
       "_view_name": "ProgressView",
       "bar_style": "success",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_f7a4ca30621040499631afdb884c0fd7",
       "max": 3042,
       "min": 0,
       "orientation": "horizontal",
       "style": "IPY_MODEL_236e3315a64d43da8505e5b75c54bc19",
       "value": 3042
      }
     },
     "8294426abea84d7d86349b7e4d12af96": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.4.0",
      "model_name": "HBoxModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.4.0",
       "_model_name": "HBoxModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.4.0",
       "_view_name": "HBoxView",
       "box_style": "",
       "children": [
        "IPY_MODEL_8d6e04ba41e14e9c9c175018b5772bd1",
        "IPY_MODEL_eaeb404915444953a1543120e434db46"
       ],
       "layout": "IPY_MODEL_c6c2b41c9f7547eca20373fd72eeab4f"
      }
     },
     "8501cf441cc048eb8f92de654a502c83": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.1.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.1.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.1.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "87e27df3d3704904b18ccbe56d75da64": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.4.0",
      "model_name": "IntProgressModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.4.0",
       "_model_name": "IntProgressModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.4.0",
       "_view_name": "ProgressView",
       "bar_style": "success",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_fddb9e973516450a86631442175548bb",
       "max": 97320,
       "min": 0,
       "orientation": "horizontal",
       "style": "IPY_MODEL_905900f6141d4e9ca7c75246796c913a",
       "value": 97320
      }
     },
     "88b156a033bd47a69431846cf994598c": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.4.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.4.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.4.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_585781ef4a474abeaad44514e6f86301",
       "placeholder": "​",
       "style": "IPY_MODEL_0a767be83d004c54a5888bedd3824e93",
       "value": "100% 191/191 [00:15&lt;00:00, 12.55it/s]"
      }
     },
     "890d5ec243774ad398ca4edeb7b98ab9": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.1.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.1.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.1.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "8bb73cfe8b5146da9bcd3e177e435068": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.4.0",
      "model_name": "IntProgressModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.4.0",
       "_model_name": "IntProgressModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.4.0",
       "_view_name": "ProgressView",
       "bar_style": "success",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_60f7f9e47cfc4e5db49c3c07b7c1c5ca",
       "max": 191,
       "min": 0,
       "orientation": "horizontal",
       "style": "IPY_MODEL_1db89b547f5d49148bc8f93ee3c8e84d",
       "value": 191
      }
     },
     "8d6e04ba41e14e9c9c175018b5772bd1": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.4.0",
      "model_name": "IntProgressModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.4.0",
       "_model_name": "IntProgressModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.4.0",
       "_view_name": "ProgressView",
       "bar_style": "success",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_13e351659fb841a68e0e757ea19ffaa4",
       "max": 97320,
       "min": 0,
       "orientation": "horizontal",
       "style": "IPY_MODEL_ac1f6a37bc41444bad72bec273e9599d",
       "value": 97320
      }
     },
     "905900f6141d4e9ca7c75246796c913a": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.4.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.4.0",
       "_model_name": "ProgressStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.1.0",
       "_view_name": "StyleView",
       "bar_color": null,
       "description_width": ""
      }
     },
     "951b8b9ff9174c90ac49a066610d4d6a": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.1.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.1.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.1.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "990b3a4cbc2046da8771ff906ab7e6c6": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.4.0",
      "model_name": "HBoxModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.4.0",
       "_model_name": "HBoxModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.4.0",
       "_view_name": "HBoxView",
       "box_style": "",
       "children": [
        "IPY_MODEL_e30074543f4b4d3f8657e4a86f69fcfa",
        "IPY_MODEL_5973648532ce406eb60d64587e5a4c08"
       ],
       "layout": "IPY_MODEL_233fe49c7eac412fb126b82b4520e30e"
      }
     },
     "995c3814a9bc4fc38c3aeb63a78988c5": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.1.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.1.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.1.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "997b0334229443c5b908d6682c55fb83": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.1.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.1.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.1.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "9c408135ccc74c3daa9ae82ed0df3e7f": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.4.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.4.0",
       "_model_name": "ProgressStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.1.0",
       "_view_name": "StyleView",
       "bar_color": null,
       "description_width": ""
      }
     },
     "9cf7a492486e4dea8b4a4d1047879d87": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.4.0",
      "model_name": "IntProgressModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.4.0",
       "_model_name": "IntProgressModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.4.0",
       "_view_name": "ProgressView",
       "bar_style": "success",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_57cad37f007449c38fa318f6a6fb8002",
       "max": 191,
       "min": 0,
       "orientation": "horizontal",
       "style": "IPY_MODEL_3aeb13c3428c4866b5b52f0a04714e84",
       "value": 191
      }
     },
     "ac1f6a37bc41444bad72bec273e9599d": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.4.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.4.0",
       "_model_name": "ProgressStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.1.0",
       "_view_name": "StyleView",
       "bar_color": null,
       "description_width": ""
      }
     },
     "aea3aa45b33446988bcc70b778a43708": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.4.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.4.0",
       "_model_name": "ProgressStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.1.0",
       "_view_name": "StyleView",
       "bar_color": null,
       "description_width": ""
      }
     },
     "b02667e651314e77a12bc76a5c5715c1": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.1.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.1.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.1.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "b33efbd3f0a0444da222486bae37d506": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.1.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.1.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.1.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "ba2d66d3909d49a6a2ceb905145f3332": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.4.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.4.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.4.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_b33efbd3f0a0444da222486bae37d506",
       "placeholder": "​",
       "style": "IPY_MODEL_27c9c94914fd4ab9b8913783cdfce55e",
       "value": " 79% 2411/3042 [10:24&lt;02:43,  3.87it/s]"
      }
     },
     "bb2b10b0dea746f488033d08271907d8": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.1.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.1.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.1.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "bc0a5b41821042119d495951f4b7f260": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.1.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.1.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.1.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "bc6472394eeb4468ab4aa3e17b08b167": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.1.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.1.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.1.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "be51789205874dac9d85e1b34955f374": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.1.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.1.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.1.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "bf78f23baaf04cb3888ad208d5df8f4c": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.4.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.4.0",
       "_model_name": "ProgressStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.1.0",
       "_view_name": "StyleView",
       "bar_color": null,
       "description_width": ""
      }
     },
     "bfacd5e004be4510ba65fec5d666b826": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.1.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.1.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.1.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "c24ae4ee94f64d0baced954f2d9efc7d": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.1.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.1.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.1.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "c464dbc95ca0480086967e3b920b7801": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.4.0",
      "model_name": "IntProgressModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.4.0",
       "_model_name": "IntProgressModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.4.0",
       "_view_name": "ProgressView",
       "bar_style": "success",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_c7266f9687a84f048598aef624713324",
       "max": 97320,
       "min": 0,
       "orientation": "horizontal",
       "style": "IPY_MODEL_c48fb7e9b1ae46bb868e0a9543232136",
       "value": 97320
      }
     },
     "c48fb7e9b1ae46bb868e0a9543232136": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.4.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.4.0",
       "_model_name": "ProgressStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.1.0",
       "_view_name": "StyleView",
       "bar_color": null,
       "description_width": ""
      }
     },
     "c5f6d6f93e6940488aaf98e33970615c": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.1.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.1.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.1.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "c6c2b41c9f7547eca20373fd72eeab4f": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.1.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.1.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.1.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "c6fdbea5e8764cf99093d2690807451e": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.1.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.1.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.1.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "c7266f9687a84f048598aef624713324": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.1.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.1.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.1.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "c8f68a767f0240dbbca7fb2c41e70d13": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.4.0",
      "model_name": "HBoxModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.4.0",
       "_model_name": "HBoxModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.4.0",
       "_view_name": "HBoxView",
       "box_style": "",
       "children": [
        "IPY_MODEL_c464dbc95ca0480086967e3b920b7801",
        "IPY_MODEL_f40aa09d4d4b4cfb91a8ccdc13595995"
       ],
       "layout": "IPY_MODEL_e06370a5bc5f4c319373724567a2b3b6"
      }
     },
     "c97a9da014304bb1aaa1a4d9326d68ec": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.1.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.1.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.1.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "cc83ecbdc8e94eeb920f87c900ed29d5": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.4.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.4.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.1.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "ccc9c2260ed64be4a03109cf7fbf528a": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.4.0",
      "model_name": "HBoxModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.4.0",
       "_model_name": "HBoxModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.4.0",
       "_view_name": "HBoxView",
       "box_style": "",
       "children": [
        "IPY_MODEL_176dfbfb599c45a186238e5d4dda40f1",
        "IPY_MODEL_88b156a033bd47a69431846cf994598c"
       ],
       "layout": "IPY_MODEL_0761a8a30eec4af5bec9d8967a3664e6"
      }
     },
     "cdd6bfd4fd0744ef9382aa7735240c67": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.1.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.1.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.1.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "d342f84f7b134d759d5bdad427a66635": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.1.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.1.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.1.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "d49d3a34d3c249c28281d613e1edad2a": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.4.0",
      "model_name": "IntProgressModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.4.0",
       "_model_name": "IntProgressModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.4.0",
       "_view_name": "ProgressView",
       "bar_style": "success",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_bc6472394eeb4468ab4aa3e17b08b167",
       "max": 191,
       "min": 0,
       "orientation": "horizontal",
       "style": "IPY_MODEL_166164f4095f43d482ba3095acc1cb58",
       "value": 191
      }
     },
     "d4b3b329b7c64fa8b7f5ca30a156fc7b": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.4.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.4.0",
       "_model_name": "ProgressStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.1.0",
       "_view_name": "StyleView",
       "bar_color": null,
       "description_width": ""
      }
     },
     "d5251789f7a340489288631dc3ae66e1": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.4.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.4.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.1.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "dafcbd098cc240078e2d4318566d8481": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.4.0",
      "model_name": "HBoxModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.4.0",
       "_model_name": "HBoxModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.4.0",
       "_view_name": "HBoxView",
       "box_style": "",
       "children": [
        "IPY_MODEL_8bb73cfe8b5146da9bcd3e177e435068",
        "IPY_MODEL_f399d94dcb5040c88e0fced549730da5"
       ],
       "layout": "IPY_MODEL_1d60eb822b2d44ba83c7c5a2097319c3"
      }
     },
     "dbc1e37c3bd44919a43861fe1cd4dd7d": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.1.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.1.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.1.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "dcd12b9e66e444c4af91f33b036287c1": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.4.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.4.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.1.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "dd44dc23ce554e7d86f30ddc51092fdb": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.4.0",
      "model_name": "IntProgressModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.4.0",
       "_model_name": "IntProgressModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.4.0",
       "_view_name": "ProgressView",
       "bar_style": "",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_52ff598372604bb68a9f840b1b192782",
       "max": 3042,
       "min": 0,
       "orientation": "horizontal",
       "style": "IPY_MODEL_60206fd0636f400a883975decc79a1d6",
       "value": 526
      }
     },
     "dda99ec70a8b4988867f037ee2952787": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.4.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.4.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.1.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "ddaa139661e24d64a9749d2a3cc1a699": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.4.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.4.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.4.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_dbc1e37c3bd44919a43861fe1cd4dd7d",
       "placeholder": "​",
       "style": "IPY_MODEL_e87766b2c5db4bcbaeeaf71e9fbe6d3e",
       "value": "100% 97320/97320 [00:38&lt;00:00, 2524.72it/s]"
      }
     },
     "dfa3259f28004d3b9f77666e4c3c5dbc": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.1.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.1.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.1.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "e06370a5bc5f4c319373724567a2b3b6": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.1.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.1.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.1.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "e2204e1a8ba640c990e7d3e6ae8712df": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.4.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.4.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.4.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_c97a9da014304bb1aaa1a4d9326d68ec",
       "placeholder": "​",
       "style": "IPY_MODEL_d5251789f7a340489288631dc3ae66e1",
       "value": "100% 3042/3042 [11:57&lt;00:00,  4.24it/s]"
      }
     },
     "e30074543f4b4d3f8657e4a86f69fcfa": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.4.0",
      "model_name": "IntProgressModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.4.0",
       "_model_name": "IntProgressModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.4.0",
       "_view_name": "ProgressView",
       "bar_style": "success",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_be51789205874dac9d85e1b34955f374",
       "max": 3042,
       "min": 0,
       "orientation": "horizontal",
       "style": "IPY_MODEL_9c408135ccc74c3daa9ae82ed0df3e7f",
       "value": 3042
      }
     },
     "e551aa6c29b84c1eb52a2a1d8f344ea7": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.1.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.1.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.1.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "e87766b2c5db4bcbaeeaf71e9fbe6d3e": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.4.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.4.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.1.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "ea9b8095fb794f9f9953f2e75d53902d": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.1.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.1.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.1.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "eaeb404915444953a1543120e434db46": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.4.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.4.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.4.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_890d5ec243774ad398ca4edeb7b98ab9",
       "placeholder": "​",
       "style": "IPY_MODEL_6d4a7039a9104fae954b6b11b8c5af95",
       "value": "100% 97320/97320 [00:00&lt;00:00, 98017.67it/s]"
      }
     },
     "ebdb8f8e01f346f08bee5d4fc05db636": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.1.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.1.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.1.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "f399d94dcb5040c88e0fced549730da5": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.4.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.4.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.4.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_1e7d2c2af12d4176a7113f8b300466e3",
       "placeholder": "​",
       "style": "IPY_MODEL_fed97d798616411fa704192c8369b9f7",
       "value": "100% 191/191 [00:15&lt;00:00, 12.58it/s]"
      }
     },
     "f40aa09d4d4b4cfb91a8ccdc13595995": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.4.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.4.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.4.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_c6fdbea5e8764cf99093d2690807451e",
       "placeholder": "​",
       "style": "IPY_MODEL_dda99ec70a8b4988867f037ee2952787",
       "value": "100% 97320/97320 [00:31&lt;00:00, 3099.71it/s]"
      }
     },
     "f7a4ca30621040499631afdb884c0fd7": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.1.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.1.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.1.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "fddb9e973516450a86631442175548bb": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.1.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.1.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.1.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "fed97d798616411fa704192c8369b9f7": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.4.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.4.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.1.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     }
    },
    "version_major": 2,
    "version_minor": 0
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}