From 342bf3640d9e3acb9794f65d15d42a3d6bec6bd4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miguel=20Gonz=C3=A1lez-Fierro?= <3491412+miguelgfierro@users.noreply.github.com> Date: Wed, 20 Nov 2019 21:12:39 +0000 Subject: [PATCH 01/18] update mlflow version to match the other azureml versions --- tools/generate_conda_file.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/generate_conda_file.py b/tools/generate_conda_file.py index 06d6291cf..68a5b39d6 100644 --- a/tools/generate_conda_file.py +++ b/tools/generate_conda_file.py @@ -63,14 +63,14 @@ "azureml-train-automl": "azureml-train-automl==1.0.57", "azureml-dataprep": "azureml-dataprep==1.1.8", "azureml-widgets": "azureml-widgets==1.0.57", - "azureml-mlflow": "azureml-mlflow>=1.0.43.1", + "azureml-mlflow": "azureml-mlflow==1.0.57", "black": "black>=18.6b4", "cached-property": "cached-property==1.5.1", "jsonlines": "jsonlines>=1.2.0", "nteract-scrapbook": "nteract-scrapbook>=0.2.1", "pydocumentdb": "pydocumentdb>=2.3.3", "pytorch-pretrained-bert": "pytorch-pretrained-bert>=0.6", - "tqdm": "tqdm==4.31.1", + "tqdm": "tqdm==4.32.2", "pyemd": "pyemd==0.5.1", "ipywebrtc": "ipywebrtc==0.4.3", "pre-commit": "pre-commit>=1.14.4", From e91b9efa4828a20a7ad96b78ab45487ef662869e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miguel=20Gonz=C3=A1lez-Fierro?= <3491412+miguelgfierro@users.noreply.github.com> Date: Thu, 21 Nov 2019 18:01:48 +0000 Subject: [PATCH 02/18] Update generate_conda_file.py --- tools/generate_conda_file.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/generate_conda_file.py b/tools/generate_conda_file.py index 68a5b39d6..afb82199a 100644 --- a/tools/generate_conda_file.py +++ b/tools/generate_conda_file.py @@ -29,6 +29,7 @@ --display-name "Python ({conda_env})" """ + CHANNELS = ["defaults", "conda-forge", "pytorch"] CONDA_BASE = { From 00d9ca0b17bdb5f19200fae9aa233fb2268a6c16 Mon Sep 17 00:00:00 2001 From: miguelgfierro Date: Thu, 21 Nov 2019 19:15:49 +0000 Subject: [PATCH 03/18] added temporary --- tests/unit/test_notebooks_cpu.py | 8 ++------ tests/unit/test_notebooks_gpu.py | 8 ++------ 2 files changed, 4 insertions(+), 12 deletions(-) diff --git a/tests/unit/test_notebooks_cpu.py b/tests/unit/test_notebooks_cpu.py index b5514894a..ab47ef87a 100644 --- a/tests/unit/test_notebooks_cpu.py +++ b/tests/unit/test_notebooks_cpu.py @@ -9,17 +9,13 @@ @pytest.mark.notebooks -def test_bert_encoder(notebooks): +def test_bert_encoder(notebooks, tmp): notebook_path = notebooks["bert_encoder"] pm.execute_notebook( notebook_path, OUTPUT_NOTEBOOK, kernel_name=KERNEL_NAME, parameters=dict( - NUM_GPUS=0, - LANGUAGE=Language.ENGLISH, - TO_LOWER=True, - MAX_SEQ_LENGTH=128, - CACHE_DIR="./temp", + NUM_GPUS=0, LANGUAGE=Language.ENGLISH, TO_LOWER=True, MAX_SEQ_LENGTH=128, CACHE_DIR=tmp ), ) diff --git a/tests/unit/test_notebooks_gpu.py b/tests/unit/test_notebooks_gpu.py index fe7149b8e..e066cbf77 100644 --- a/tests/unit/test_notebooks_gpu.py +++ b/tests/unit/test_notebooks_gpu.py @@ -10,17 +10,13 @@ @pytest.mark.notebooks @pytest.mark.gpu -def test_bert_encoder(notebooks): +def test_bert_encoder(notebooks, tmp): notebook_path = notebooks["bert_encoder"] pm.execute_notebook( notebook_path, OUTPUT_NOTEBOOK, kernel_name=KERNEL_NAME, parameters=dict( - NUM_GPUS=1, - LANGUAGE=Language.ENGLISH, - TO_LOWER=True, - MAX_SEQ_LENGTH=128, - CACHE_DIR="./temp", + NUM_GPUS=1, LANGUAGE=Language.ENGLISH, TO_LOWER=True, MAX_SEQ_LENGTH=128, CACHE_DIR=tmp ), ) From 2f9bfad49335073420c7e117dc960fe8b61ff807 Mon Sep 17 00:00:00 2001 From: Emmanuel Awa Date: Mon, 25 Nov 2019 19:03:32 +0000 Subject: [PATCH 04/18] doc: update github url references --- SETUP.md | 8 ++++---- docs/source/index.rst | 4 ++-- .../question_answering_system_bidaf_quickstart.ipynb | 2 +- setup.py | 6 +++--- utils_nlp/README.md | 2 +- 5 files changed, 11 insertions(+), 11 deletions(-) diff --git a/SETUP.md b/SETUP.md index d53870d74..6ba5e6642 100755 --- a/SETUP.md +++ b/SETUP.md @@ -49,7 +49,7 @@ which you can use to create the target environment using the Python version 3.6 Assuming the repo is cloned as `nlp` in the system, to install **a default (Python CPU) environment**: - cd nlp + cd nlp-recipes python tools/generate_conda_file.py conda env create -f nlp_cpu.yaml @@ -62,7 +62,7 @@ Click on the following menus to see how to install the Python GPU environment: Assuming that you have a GPU machine, to install the Python GPU environment, which by default installs the CPU environment: - cd nlp + cd nlp-recipes python tools/generate_conda_file.py --gpu conda env create -n nlp_gpu -f nlp_gpu.yaml @@ -79,7 +79,7 @@ Assuming that you have an Azure GPU DSVM machine, here are the steps to setup th 2. Install the GPU environment. - cd nlp + cd nlp-recipes python tools/generate_conda_file.py --gpu conda env create -n nlp_gpu -f nlp_gpu.yaml @@ -110,7 +110,7 @@ Running the command tells pip to install the `utils_nlp` package from source in > It is also possible to install directly from Github, which is the best way to utilize the `utils_nlp` package in external projects (while still reflecting updates to the source as it's installed as an editable `'-e'` package). -> `pip install -e git+git@github.com:microsoft/nlp.git@master#egg=utils_nlp` +> `pip install -e git+git@github.com:microsoft/nlp-recipes.git@master#egg=utils_nlp` Either command, from above, makes `utils_nlp` available in your conda virtual environment. You can verify it was properly installed by running: diff --git a/docs/source/index.rst b/docs/source/index.rst index 067478672..836b501cb 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -2,9 +2,9 @@ NLP Utilities =================================================== -The `NLP repository `_ provides examples and best practices for building NLP systems, provided as Jupyter notebooks. +The `NLP repository `_ provides examples and best practices for building NLP systems, provided as Jupyter notebooks. -The module `utils_nlp `_ contains functions to simplify common tasks used when developing and +The module `utils_nlp `_ contains functions to simplify common tasks used when developing and evaluating NLP systems. .. toctree:: diff --git a/examples/question_answering/question_answering_system_bidaf_quickstart.ipynb b/examples/question_answering/question_answering_system_bidaf_quickstart.ipynb index 68f4894d8..d41391ad9 100644 --- a/examples/question_answering/question_answering_system_bidaf_quickstart.ipynb +++ b/examples/question_answering/question_answering_system_bidaf_quickstart.ipynb @@ -175,7 +175,7 @@ "metadata": {}, "source": [ "This step downloads the pre-trained [AllenNLP](https://allennlp.org/models) pretrained model and registers the model in our Workspace. The pre-trained AllenNLP model we use is called Bidirectional Attention Flow for Machine Comprehension ([BiDAF](https://www.semanticscholar.org/paper/Bidirectional-Attention-Flow-for-Machine-Seo-Kembhavi/007ab5528b3bd310a80d553cccad4b78dc496b02\n", - ")) It achieved state-of-the-art performance on the [SQuAD](https://rajpurkar.github.io/SQuAD-explorer/) dataset in 2017 and is a well-respected, performant baseline for QA. AllenNLP's pre-trained BIDAF model is trained on the SQuAD training set and achieves an EM score of 68.3 on the SQuAD development set. See the [BIDAF deep dive notebook](https://github.com/microsoft/nlp/examples/question_answering/bidaf_deep_dive.ipynb\n", + ")) It achieved state-of-the-art performance on the [SQuAD](https://rajpurkar.github.io/SQuAD-explorer/) dataset in 2017 and is a well-respected, performant baseline for QA. AllenNLP's pre-trained BIDAF model is trained on the SQuAD training set and achieves an EM score of 68.3 on the SQuAD development set. See the [BIDAF deep dive notebook](https://github.com/microsoft/nlp-recipes/examples/question_answering/bidaf_deep_dive.ipynb\n", ") for more information on this algorithm and AllenNLP implementation." ] }, diff --git a/setup.py b/setup.py index 38c240720..f87901585 100644 --- a/setup.py +++ b/setup.py @@ -29,7 +29,7 @@ def read(*names, **kwargs): ), author=AUTHOR, author_email="teamsharat@microsoft.com", - url="https://github.com/microsoft/nlp", + url="https://github.com/microsoft/nlp-recipes", packages=["utils_nlp"], include_package_data=True, zip_safe=True, @@ -56,8 +56,8 @@ def read(*names, **kwargs): "Intended Audience :: Telecommunications Industry", ], project_urls={ - "Documentation": "https://github.com/microsoft/nlp/", - "Issue Tracker": "https://github.com/microsoft/nlp/issues", + "Documentation": "https://github.com/microsoft/nlp-recipes/", + "Issue Tracker": "https://github.com/microsoft/nlp-recipes/issues", }, keywords=["Microsoft NLP", "Natural Language Processing", "Text Processing", "Word Embedding"], python_requires=">=3.6", diff --git a/utils_nlp/README.md b/utils_nlp/README.md index 14727ef70..b21ad2169 100755 --- a/utils_nlp/README.md +++ b/utils_nlp/README.md @@ -26,7 +26,7 @@ ws = get_or_create_workspace( This submodule contains high-level utilities that are commonly used in multiple algorithms as well as helper functions for managing frameworks like pytorch. ### [Dataset](dataset) -This submodule includes helper functions for interacting with well-known datasets, utility functions to process datasets for different NLP tasks, as well as utilities for splitting data for training/testing. For example, the [snli module](snli.py) will allow you to load a dataframe in pandas from the Stanford Natural Language Inference (SNLI) Corpus dataset, with the option to set the number of rows to load in order to test algorithms and evaluate performance benchmarks. Information on the datasets used in the repo can be found [here](https://github.com/microsoft/nlp/tree/staging/utils_nlp/dataset#datasets). +This submodule includes helper functions for interacting with well-known datasets, utility functions to process datasets for different NLP tasks, as well as utilities for splitting data for training/testing. For example, the [snli module](snli.py) will allow you to load a dataframe in pandas from the Stanford Natural Language Inference (SNLI) Corpus dataset, with the option to set the number of rows to load in order to test algorithms and evaluate performance benchmarks. Information on the datasets used in the repo can be found [here](https://github.com/microsoft/nlp-recipes/tree/staging/utils_nlp/dataset#datasets). Most datasets may be split into `train`, `dev`, and `test`. From c8abcbebbf0a0fb3f3ba857e4df53bd7d9614661 Mon Sep 17 00:00:00 2001 From: Emmanuel Awa Date: Mon, 25 Nov 2019 19:12:31 +0000 Subject: [PATCH 05/18] docs: update nlp recipes references --- SETUP.md | 2 +- docs/source/conf.py | 2 +- setup.py | 8 +++++++- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/SETUP.md b/SETUP.md index 6ba5e6642..7b337182f 100755 --- a/SETUP.md +++ b/SETUP.md @@ -47,7 +47,7 @@ You can learn how to create a Notebook VM [here](https://docs.microsoft.com/en-u We provide a script, [generate_conda_file.py](tools/generate_conda_file.py), to generate a conda-environment yaml file which you can use to create the target environment using the Python version 3.6 with all the correct dependencies. -Assuming the repo is cloned as `nlp` in the system, to install **a default (Python CPU) environment**: +Assuming the repo is cloned as `nlp-recipes` in the system, to install **a default (Python CPU) environment**: cd nlp-recipes python tools/generate_conda_file.py diff --git a/docs/source/conf.py b/docs/source/conf.py index 812c8d28d..14d77536b 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -34,7 +34,7 @@ # The full version, including alpha/beta/rc tags release = VERSION -prefix = "NLP" +prefix = "NLPRecipes" # -- General configuration --------------------------------------------------- diff --git a/setup.py b/setup.py index f87901585..70fe2aeab 100644 --- a/setup.py +++ b/setup.py @@ -59,7 +59,13 @@ def read(*names, **kwargs): "Documentation": "https://github.com/microsoft/nlp-recipes/", "Issue Tracker": "https://github.com/microsoft/nlp-recipes/issues", }, - keywords=["Microsoft NLP", "Natural Language Processing", "Text Processing", "Word Embedding"], + keywords=[ + "Microsoft NLP", + "NLP Recipes", + "Natural Language Processing", + "Text Processing", + "Word Embedding", + ], python_requires=">=3.6", install_requires=[], dependency_links=[], From 99d00d4008188699bca27fcb4c8bfba9ae8f0a4a Mon Sep 17 00:00:00 2001 From: Ke Huang Date: Mon, 25 Nov 2019 15:32:35 -0500 Subject: [PATCH 06/18] Minor bug fix for text classification of multi languages notebook --- .../text_classification/tc_multi_languages_transformers.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/text_classification/tc_multi_languages_transformers.ipynb b/examples/text_classification/tc_multi_languages_transformers.ipynb index 789986bfc..437c95cfb 100644 --- a/examples/text_classification/tc_multi_languages_transformers.ipynb +++ b/examples/text_classification/tc_multi_languages_transformers.ipynb @@ -440,7 +440,7 @@ " test_labels, \n", " preds,\n", " digits=2,\n", - " labels=test_labels.unique(),\n", + " labels=np.unique(test_labels),\n", " target_names=label_encoder.classes_\n", ")\n", "\n", From d71de4a1d53fac115c97357887e3ea102c63c7cb Mon Sep 17 00:00:00 2001 From: saidbleik Date: Mon, 25 Nov 2019 20:43:14 +0000 Subject: [PATCH 07/18] remove bert and xlnet notebooks --- .../text_classification/tc_bbc_bert_hi.ipynb | 1198 ----------------- .../text_classification/tc_dac_bert_ar.ipynb | 821 ----------- .../text_classification/tc_mnli_xlnet.ipynb | 974 -------------- tests/conftest.py | 6 - tests/unit/test_xlnet_common.py | 27 - .../test_xlnet_sequence_classification.py | 44 - 6 files changed, 3070 deletions(-) delete mode 100644 examples/text_classification/tc_bbc_bert_hi.ipynb delete mode 100644 examples/text_classification/tc_dac_bert_ar.ipynb delete mode 100644 examples/text_classification/tc_mnli_xlnet.ipynb delete mode 100644 tests/unit/test_xlnet_common.py delete mode 100644 tests/unit/test_xlnet_sequence_classification.py diff --git a/examples/text_classification/tc_bbc_bert_hi.ipynb b/examples/text_classification/tc_bbc_bert_hi.ipynb deleted file mode 100644 index 93ef0f24b..000000000 --- a/examples/text_classification/tc_bbc_bert_hi.ipynb +++ /dev/null @@ -1,1198 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "*Copyright (c) Microsoft Corporation. All rights reserved.*\n", - "\n", - "*Licensed under the MIT License.*\n", - "\n", - "# Classification of Hindi BBC News Data using BERT" - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [], - "source": [ - "import os\n", - "import sys\n", - "\n", - "import json\n", - "import numpy as np\n", - "import pandas as pd\n", - "import torch\n", - "import torch.nn as nn\n", - "import scrapbook as sb\n", - "from sklearn.metrics import accuracy_score, classification_report\n", - "from sklearn.model_selection import train_test_split\n", - "from sklearn.preprocessing import LabelEncoder\n", - "\n", - "sys.path.append(\"../../\")\n", - "from utils_nlp.common.timer import Timer\n", - "from utils_nlp.dataset.multinli import load_pandas_df\n", - "from utils_nlp.models.bert.common import Language, Tokenizer\n", - "from utils_nlp.models.bert.sequence_classification import BERTSequenceClassifier" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Introduction\n", - "In this notebook, we fine-tune and evaluate a pretrained [BERT](https://arxiv.org/abs/1810.04805) model on a subset of the [BBC Hindi News](https://github.com/NirantK/hindi2vec/releases/tag/bbc-hindi-v0.1) dataset.\n", - "\n", - "We use a [sequence classifier](../../utils_nlp/bert/sequence_classification.py) that wraps [Hugging Face's PyTorch implementation](https://github.com/huggingface/pytorch-pretrained-BERT) of Google's [BERT](https://github.com/google-research/bert)." - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": { - "tags": [ - "parameters" - ] - }, - "outputs": [], - "source": [ - "DATA_FOLDER = \"./temp\"\n", - "BERT_CACHE_DIR = \"./temp\"\n", - "LANGUAGE = Language.MULTILINGUAL\n", - "TO_LOWER = False\n", - "MAX_LEN = 128\n", - "BATCH_SIZE = 8\n", - "WARMUP_PROPORTION = 0.1\n", - "NUM_GPUS = 2\n", - "NUM_EPOCHS = 2\n", - "LABEL_COL = \"news_category\"\n", - "TEXT_COL = \"news_content\"" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Read Dataset\n", - "We start by downloading the dataset by using the following command.\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": { - "scrolled": true - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "--2019-09-12 16:01:58-- https://github.com/NirantK/hindi2vec/releases/download/bbc-hindi-v0.1/bbc-hindiv01.tar.gz\n", - "Resolving github.com (github.com)... 140.82.113.3\n", - "Connecting to github.com (github.com)|140.82.113.3|:443... connected.\n", - "HTTP request sent, awaiting response... 302 Found\n", - "Location: https://github-production-release-asset-2e65be.s3.amazonaws.com/123591003/701307f8-3cb5-11e8-9472-df990c204ce8?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIAIWNJYAX4CSVEH53A%2F20190912%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20190912T160158Z&X-Amz-Expires=300&X-Amz-Signature=f1da6919e49dba6ebcc3f040ff6a9ffa2c7235a60b9797ba37b86a798214def9&X-Amz-SignedHeaders=host&actor_id=0&response-content-disposition=attachment%3B%20filename%3Dbbc-hindiv01.tar.gz&response-content-type=application%2Foctet-stream [following]\n", - "--2019-09-12 16:01:58-- https://github-production-release-asset-2e65be.s3.amazonaws.com/123591003/701307f8-3cb5-11e8-9472-df990c204ce8?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIAIWNJYAX4CSVEH53A%2F20190912%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20190912T160158Z&X-Amz-Expires=300&X-Amz-Signature=f1da6919e49dba6ebcc3f040ff6a9ffa2c7235a60b9797ba37b86a798214def9&X-Amz-SignedHeaders=host&actor_id=0&response-content-disposition=attachment%3B%20filename%3Dbbc-hindiv01.tar.gz&response-content-type=application%2Foctet-stream\n", - "Resolving github-production-release-asset-2e65be.s3.amazonaws.com (github-production-release-asset-2e65be.s3.amazonaws.com)... 52.216.233.131\n", - "Connecting to github-production-release-asset-2e65be.s3.amazonaws.com (github-production-release-asset-2e65be.s3.amazonaws.com)|52.216.233.131|:443... connected.\n", - "HTTP request sent, awaiting response... 200 OK\n", - "Length: 11265715 (11M) [application/octet-stream]\n", - "Saving to: ‘bbc-hindiv01.tar.gz’\n", - "\n", - "bbc-hindiv01.tar.gz 100%[===================>] 10.74M --.-KB/s in 0.1s \n", - "\n", - "2019-09-12 16:01:58 (74.3 MB/s) - ‘bbc-hindiv01.tar.gz’ saved [11265715/11265715]\n", - "\n", - "bbc-hindi-news.json\n", - "hindi-test.csv\n", - "hindi-train.csv\n" - ] - } - ], - "source": [ - "!wget https://github.com/NirantK/hindi2vec/releases/download/bbc-hindi-v0.1/bbc-hindiv01.tar.gz &&\\\n", - " mkdir -p bbc-hindiv01 &&\\\n", - " mv bbc-hindiv01.tar.gz ./bbc-hindiv01 && cd ./bbc-hindiv01 &&\\\n", - " tar -xvf bbc-hindiv01.tar.gz " - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Once dataset is downloaded, we can just use pandas to load the training and testing data into dataframes and also inspect the dataframes. \n", - "\n", - "For our classification task, we are limited by the memory of the machine we use. We need to set appropriate maximum sequence MAX_LEN and bath size BATCH_SIZE to fit the training data into memory. This notebook has ran on a machine with two Tesla K80 GPUS. If you experience any out of memory issue, you should consider descrease the MAX_LEN and/or BATCH_SIZE but you may see difference accuracy of the model" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
01
0indiaमेट्रो की इस लाइन के चलने से दक्षिणी दिल्ली से...
1pakistanनेटिजन यानि इंटरनेट पर सक्रिय नागरिक अब ट्विटर...
2newsइसमें एक फ़्लाइट एटेनडेंट की मदद की गुहार है औ...
3indiaप्रतीक खुलेपन का, आज़ाद ख्याली का और भीड़ से अ...
4indiaख़ासकर पिछले 10 साल तक प्रधानमंत्री रहे मनमोहन...
\n", - "
" - ], - "text/plain": [ - " 0 1\n", - "0 india मेट्रो की इस लाइन के चलने से दक्षिणी दिल्ली से...\n", - "1 pakistan नेटिजन यानि इंटरनेट पर सक्रिय नागरिक अब ट्विटर...\n", - "2 news इसमें एक फ़्लाइट एटेनडेंट की मदद की गुहार है औ...\n", - "3 india प्रतीक खुलेपन का, आज़ाद ख्याली का और भीड़ से अ...\n", - "4 india ख़ासकर पिछले 10 साल तक प्रधानमंत्री रहे मनमोहन..." - ] - }, - "execution_count": 4, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "df_train = pd.read_csv('./bbc-hindiv01/hindi-train.csv', sep=\"\\t\", encoding='utf-8', header=None)\n", - "df_train.head()" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
01
0indiaबुधवार को राज्य सभा में विपक्ष के सवालों के जव...
1indiaलखनऊ स्थित पत्रकार समीरात्मज मिश्र को बुलंदशहर...
2indiaलगभग 1300 हेक्टेयर ज़मीन का अधिग्रहण किया जा च...
3internationalहालांकि उनके अंगरक्षकों को बमों को जाम करने वा...
4indiaआयोग का कहना है कि इस तरह के परीक्षण से महिलाओ...
\n", - "
" - ], - "text/plain": [ - " 0 1\n", - "0 india बुधवार को राज्य सभा में विपक्ष के सवालों के जव...\n", - "1 india लखनऊ स्थित पत्रकार समीरात्मज मिश्र को बुलंदशहर...\n", - "2 india लगभग 1300 हेक्टेयर ज़मीन का अधिग्रहण किया जा च...\n", - "3 international हालांकि उनके अंगरक्षकों को बमों को जाम करने वा...\n", - "4 india आयोग का कहना है कि इस तरह के परीक्षण से महिलाओ..." - ] - }, - "execution_count": 5, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "df_test = pd.read_csv('./bbc-hindiv01/hindi-test.csv', sep=\"\\t\", encoding='utf-8', header=None)\n", - "df_test.head()" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
01
count34683467
unique143458
topindiaहम प्रायः पशु, पक्षियों और कीड़ों-मकोड़ों के ह...
freq13902
\n", - "
" - ], - "text/plain": [ - " 0 1\n", - "count 3468 3467\n", - "unique 14 3458\n", - "top india हम प्रायः पशु, पक्षियों और कीड़ों-मकोड़ों के ह...\n", - "freq 1390 2" - ] - }, - "execution_count": 6, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "df_train.describe()" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
01
count867866
unique14865
topindiaयहां घर-घर में साड़ी बुनने के हैंडलूम लगे हैं....
freq3572
\n", - "
" - ], - "text/plain": [ - " 0 1\n", - "count 867 866\n", - "unique 14 865\n", - "top india यहां घर-घर में साड़ी बुनने के हैंडलूम लगे हैं....\n", - "freq 357 2" - ] - }, - "execution_count": 7, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "df_test.describe()" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [], - "source": [ - "df_train.columns = [LABEL_COL, TEXT_COL]\n", - "df_test.columns = [LABEL_COL, TEXT_COL]" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [], - "source": [ - "df_train = df_train.fillna(\"\")\n", - "df_test = df_test.fillna(\"\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The examples in the dataset are grouped into 14 categories:" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "india 1390\n", - "international 904\n", - "entertainment 285\n", - "sport 258\n", - "news 230\n", - "science 194\n", - "business 54\n", - "pakistan 43\n", - "southasia 42\n", - "institutional 19\n", - "social 18\n", - "china 14\n", - "multimedia 12\n", - "learningenglish 5\n", - "Name: news_category, dtype: int64" - ] - }, - "execution_count": 10, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "df_train[LABEL_COL].value_counts()" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Number of training examples: 3468\n", - "Number of testing examples: 867\n" - ] - } - ], - "source": [ - "print(\"Number of training examples: {}\".format(df_train.shape[0]))\n", - "print(\"Number of testing examples: {}\".format(df_test.shape[0]))" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Tokenize and Preprocess \n", - "Before training, we tokenize the text documents and convert them to lists of tokens. The following steps instantiate a BERT tokenizer given the language, and tokenize the text of the training and testing sets. \n", - "In addition, we perform the following preprocessing steps in the following cell:\n", - "- Convert the tokens into token indices corresponding to the BERT tokenizer's vocabulary\n", - "- Add the special tokens [CLS] and [SEP] to mark the beginning and end of a sentence\n", - "- Pad or truncate the token lists to the specified max length\n", - "- Return mask lists that indicate paddings' positions\n", - "- Return token type id lists that indicate which sentence the tokens belong to (not needed for one-sequence classification)\n", - "\n", - "*See the original [implementation](https://github.com/google-research/bert/blob/master/run_classifier.py) for more information on BERT's input format.*" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "100%|██████████| 3468/3468 [00:27<00:00, 123.97it/s]\n", - "100%|██████████| 867/867 [00:06<00:00, 125.47it/s]\n" - ] - } - ], - "source": [ - "tokenizer = Tokenizer(LANGUAGE, TO_LOWER, BERT_CACHE_DIR)\n", - "tokens_train = tokenizer.tokenize(list(df_train[TEXT_COL]))\n", - "tokens_test = tokenizer.tokenize(list(df_test[TEXT_COL]))\n", - "\n", - "label_encoder = LabelEncoder()\n", - "labels_train = label_encoder.fit_transform(df_train[LABEL_COL])\n", - "labels_test = label_encoder.transform(df_test[LABEL_COL])\n", - "num_labels = len(np.unique(labels_train))" - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "metadata": {}, - "outputs": [], - "source": [ - "tokens_train, mask_train, _ = tokenizer.preprocess_classification_tokens(\n", - " tokens_train, MAX_LEN\n", - ")\n", - "tokens_test, mask_test, _ = tokenizer.preprocess_classification_tokens(\n", - " tokens_test, MAX_LEN\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Create Model\n", - "Next, we create a sequence classifier that loads a pre-trained BERT model." - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "metadata": {}, - "outputs": [], - "source": [ - "classifier = BERTSequenceClassifier(LANGUAGE, num_labels=num_labels, cache_dir=BERT_CACHE_DIR)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Train\n", - "We train the classifier using the training examples. This involves fine-tuning the BERT Transformer and learning a linear classification layer on top of that:" - ] - }, - { - "cell_type": "code", - "execution_count": 15, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Iteration: 0%| | 1/434 [00:02<18:06, 2.51s/it]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "epoch:1/2; batch:1->44/434; average training loss:2.665879\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Iteration: 10%|█ | 45/434 [00:32<04:27, 1.46it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "epoch:1/2; batch:45->88/434; average training loss:2.100084\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Iteration: 21%|██ | 89/434 [01:02<03:57, 1.45it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "epoch:1/2; batch:89->132/434; average training loss:1.840270\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Iteration: 31%|███ | 133/434 [01:33<03:27, 1.45it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "epoch:1/2; batch:133->176/434; average training loss:1.703301\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Iteration: 41%|████ | 177/434 [02:03<02:57, 1.44it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "epoch:1/2; batch:177->220/434; average training loss:1.611534\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Iteration: 51%|█████ | 221/434 [02:34<02:27, 1.44it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "epoch:1/2; batch:221->264/434; average training loss:1.581564\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Iteration: 61%|██████ | 265/434 [03:04<01:56, 1.45it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "epoch:1/2; batch:265->308/434; average training loss:1.549611\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Iteration: 71%|███████ | 309/434 [03:35<01:30, 1.39it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "epoch:1/2; batch:309->352/434; average training loss:1.507914\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Iteration: 81%|████████▏ | 353/434 [04:07<00:59, 1.37it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "epoch:1/2; batch:353->396/434; average training loss:1.474626\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Iteration: 91%|█████████▏| 397/434 [04:39<00:26, 1.40it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "epoch:1/2; batch:397->434/434; average training loss:1.453205\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Iteration: 100%|██████████| 434/434 [05:06<00:00, 1.38it/s]\n", - "Iteration: 0%| | 1/434 [00:00<05:57, 1.21it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "epoch:2/2; batch:1->44/434; average training loss:0.690934\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Iteration: 10%|█ | 45/434 [00:34<05:07, 1.27it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "epoch:2/2; batch:45->88/434; average training loss:1.146616\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Iteration: 21%|██ | 89/434 [01:08<04:27, 1.29it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "epoch:2/2; batch:89->132/434; average training loss:1.077667\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Iteration: 31%|███ | 133/434 [01:43<03:54, 1.29it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "epoch:2/2; batch:133->176/434; average training loss:1.033159\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Iteration: 41%|████ | 177/434 [02:18<03:29, 1.23it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "epoch:2/2; batch:177->220/434; average training loss:1.023701\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Iteration: 51%|█████ | 221/434 [02:52<02:51, 1.24it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "epoch:2/2; batch:221->264/434; average training loss:1.049415\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Iteration: 61%|██████ | 265/434 [03:23<01:57, 1.44it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "epoch:2/2; batch:265->308/434; average training loss:1.049472\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Iteration: 71%|███████ | 309/434 [03:54<01:26, 1.44it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "epoch:2/2; batch:309->352/434; average training loss:1.027788\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Iteration: 81%|████████▏ | 353/434 [04:24<00:55, 1.45it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "epoch:2/2; batch:353->396/434; average training loss:1.000812\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Iteration: 91%|█████████▏| 397/434 [04:55<00:25, 1.44it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "epoch:2/2; batch:397->434/434; average training loss:0.998862\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Iteration: 100%|██████████| 434/434 [05:20<00:00, 1.49it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "[Training time: 0.175 hrs]\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\n" - ] - } - ], - "source": [ - "with Timer() as t:\n", - " classifier.fit(\n", - " token_ids=tokens_train,\n", - " input_mask=mask_train,\n", - " labels=labels_train, \n", - " num_gpus=NUM_GPUS, \n", - " num_epochs=NUM_EPOCHS,\n", - " batch_size=BATCH_SIZE,\n", - " warmup_proportion=WARMUP_PROPORTION,\n", - " verbose=True,\n", - " ) \n", - "print(\"[Training time: {:.3f} hrs]\".format(t.interval / 3600))" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Score\n", - "We score the test set using the trained classifier:" - ] - }, - { - "cell_type": "code", - "execution_count": 16, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Iteration: 100%|██████████| 109/109 [00:21<00:00, 5.31it/s]\n" - ] - } - ], - "source": [ - "preds = classifier.predict(\n", - " token_ids=tokens_test, input_mask=mask_test, num_gpus=NUM_GPUS, batch_size=BATCH_SIZE\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Evaluate Results\n", - "Finally, we compute the accuracy, precision, recall, and F1 metrics of the evaluation on the test set." - ] - }, - { - "cell_type": "code", - "execution_count": 17, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "accuracy: 0.7104959630911188\n", - "{\n", - " \"business\": {\n", - " \"f1-score\": 0.0,\n", - " \"precision\": 0.0,\n", - " \"recall\": 0.0,\n", - " \"support\": 7\n", - " },\n", - " \"china\": {\n", - " \"f1-score\": 0.0,\n", - " \"precision\": 0.0,\n", - " \"recall\": 0.0,\n", - " \"support\": 5\n", - " },\n", - " \"entertainment\": {\n", - " \"f1-score\": 0.7133757961783439,\n", - " \"precision\": 0.6511627906976745,\n", - " \"recall\": 0.7887323943661971,\n", - " \"support\": 71\n", - " },\n", - " \"india\": {\n", - " \"f1-score\": 0.8192090395480226,\n", - " \"precision\": 0.8262108262108262,\n", - " \"recall\": 0.8123249299719888,\n", - " \"support\": 357\n", - " },\n", - " \"institutional\": {\n", - " \"f1-score\": 0.0,\n", - " \"precision\": 0.0,\n", - " \"recall\": 0.0,\n", - " \"support\": 4\n", - " },\n", - " \"international\": {\n", - " \"f1-score\": 0.6787878787878788,\n", - " \"precision\": 0.5936395759717314,\n", - " \"recall\": 0.7924528301886793,\n", - " \"support\": 212\n", - " },\n", - " \"learningenglish\": {\n", - " \"f1-score\": 0.0,\n", - " \"precision\": 0.0,\n", - " \"recall\": 0.0,\n", - " \"support\": 3\n", - " },\n", - " \"macro avg\": {\n", - " \"f1-score\": 0.26085260285746015,\n", - " \"precision\": 0.2462770617515731,\n", - " \"recall\": 0.2788537537792841,\n", - " \"support\": 867\n", - " },\n", - " \"micro avg\": {\n", - " \"f1-score\": 0.7104959630911188,\n", - " \"precision\": 0.7104959630911188,\n", - " \"recall\": 0.7104959630911188,\n", - " \"support\": 867\n", - " },\n", - " \"multimedia\": {\n", - " \"f1-score\": 0.0,\n", - " \"precision\": 0.0,\n", - " \"recall\": 0.0,\n", - " \"support\": 1\n", - " },\n", - " \"news\": {\n", - " \"f1-score\": 0.0,\n", - " \"precision\": 0.0,\n", - " \"recall\": 0.0,\n", - " \"support\": 49\n", - " },\n", - " \"pakistan\": {\n", - " \"f1-score\": 0.0,\n", - " \"precision\": 0.0,\n", - " \"recall\": 0.0,\n", - " \"support\": 8\n", - " },\n", - " \"science\": {\n", - " \"f1-score\": 0.6562500000000001,\n", - " \"precision\": 0.6268656716417911,\n", - " \"recall\": 0.6885245901639344,\n", - " \"support\": 61\n", - " },\n", - " \"social\": {\n", - " \"f1-score\": 0.0,\n", - " \"precision\": 0.0,\n", - " \"recall\": 0.0,\n", - " \"support\": 6\n", - " },\n", - " \"southasia\": {\n", - " \"f1-score\": 0.0,\n", - " \"precision\": 0.0,\n", - " \"recall\": 0.0,\n", - " \"support\": 10\n", - " },\n", - " \"sport\": {\n", - " \"f1-score\": 0.7843137254901962,\n", - " \"precision\": 0.75,\n", - " \"recall\": 0.821917808219178,\n", - " \"support\": 73\n", - " },\n", - " \"weighted avg\": {\n", - " \"f1-score\": 0.6739290552608086,\n", - " \"precision\": 0.6459402758626944,\n", - " \"recall\": 0.7104959630911188,\n", - " \"support\": 867\n", - " }\n", - "}\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/media/bleik2/miniconda3/envs/nlp_gpu/lib/python3.6/site-packages/sklearn/metrics/classification.py:1143: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.\n", - " 'precision', 'predicted', average, warn_for)\n" - ] - } - ], - "source": [ - "report = classification_report(labels_test, preds, target_names=label_encoder.classes_, output_dict=True) \n", - "accuracy = accuracy_score(labels_test, preds )\n", - "print(\"accuracy: {}\".format(accuracy))\n", - "print(json.dumps(report, indent=4, sort_keys=True))" - ] - }, - { - "cell_type": "code", - "execution_count": 18, - "metadata": {}, - "outputs": [ - { - "data": { - "application/scrapbook.scrap.json+json": { - "data": 0.7104959630911188, - "encoder": "json", - "name": "accuracy", - "version": 1 - } - }, - "metadata": { - "scrapbook": { - "data": true, - "display": false, - "name": "accuracy" - } - }, - "output_type": "display_data" - }, - { - "data": { - "application/scrapbook.scrap.json+json": { - "data": 0.2462770617515731, - "encoder": "json", - "name": "precision", - "version": 1 - } - }, - "metadata": { - "scrapbook": { - "data": true, - "display": false, - "name": "precision" - } - }, - "output_type": "display_data" - }, - { - "data": { - "application/scrapbook.scrap.json+json": { - "data": 0.2788537537792841, - "encoder": "json", - "name": "recall", - "version": 1 - } - }, - "metadata": { - "scrapbook": { - "data": true, - "display": false, - "name": "recall" - } - }, - "output_type": "display_data" - }, - { - "data": { - "application/scrapbook.scrap.json+json": { - "data": 0.26085260285746015, - "encoder": "json", - "name": "f1", - "version": 1 - } - }, - "metadata": { - "scrapbook": { - "data": true, - "display": false, - "name": "f1" - } - }, - "output_type": "display_data" - } - ], - "source": [ - "# for testing\n", - "sb.glue(\"accuracy\", accuracy)\n", - "sb.glue(\"precision\", report[\"macro avg\"][\"precision\"])\n", - "sb.glue(\"recall\", report[\"macro avg\"][\"recall\"])\n", - "sb.glue(\"f1\", report[\"macro avg\"][\"f1-score\"])" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "nlp_gpu", - "language": "python", - "name": "nlp_gpu" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.6.8" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/examples/text_classification/tc_dac_bert_ar.ipynb b/examples/text_classification/tc_dac_bert_ar.ipynb deleted file mode 100644 index d4fd6d332..000000000 --- a/examples/text_classification/tc_dac_bert_ar.ipynb +++ /dev/null @@ -1,821 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "*Copyright (c) Microsoft Corporation. All rights reserved.*\n", - "\n", - "*Licensed under the MIT License.*\n", - "\n", - "# Classification of Arabic News Articles using BERT" - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [], - "source": [ - "import json\n", - "import os\n", - "import sys\n", - "\n", - "import numpy as np\n", - "import pandas as pd\n", - "import scrapbook as sb\n", - "import torch\n", - "import torch.nn as nn\n", - "from sklearn.metrics import accuracy_score, classification_report\n", - "from sklearn.model_selection import train_test_split\n", - "\n", - "sys.path.append(\"../../\")\n", - "from utils_nlp.common.timer import Timer\n", - "from utils_nlp.dataset.dac import load_pandas_df\n", - "from utils_nlp.models.bert.common import Language, Tokenizer\n", - "from utils_nlp.models.bert.sequence_classification import BERTSequenceClassifier" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Introduction\n", - "In this notebook, we fine-tune and evaluate a pretrained [BERT](https://arxiv.org/abs/1810.04805) model on an Arabic dataset of news articles. The [dataset](https://data.mendeley.com/datasets/v524p5dhpj/2) includes articles from 3 different newspapers, and the articles are categorized into 5 classes: *sports, politics, culture, economy and diverse*. The data is described in more detail in this [paper](http://article.nadiapub.com/IJGDC/vol11_no9/9.pdf).\n", - "\n", - "We use a [sequence classifier](../../utils_nlp/bert/sequence_classification.py) that wraps [Hugging Face's PyTorch implementation](https://github.com/huggingface/pytorch-pretrained-BERT) of Google's [BERT](https://github.com/google-research/bert). The classifier loads a pretrained [multilingual BERT model](https://github.com/google-research/bert/blob/master/multilingual.md) that was trained on 104 languages, including Arabic." - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": { - "tags": [ - "parameters" - ] - }, - "outputs": [], - "source": [ - "DATA_FOLDER = \"./temp\"\n", - "BERT_CACHE_DIR = \"./temp\"\n", - "LANGUAGE = Language.MULTILINGUAL\n", - "MAX_LEN = 200\n", - "BATCH_SIZE = 32\n", - "NUM_GPUS = 2\n", - "NUM_EPOCHS = 1\n", - "TRAIN_SIZE = 0.8\n", - "NUM_ROWS = 15000\n", - "RANDOM_STATE = 0" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Read Dataset\n", - "We start by loading the data. The following line also downloads the file if it doesn't exist, and extracts the csv file into the specified data folder. We retain a subset, of size *NUM_ROWS*, of the data for quicker model training." - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [], - "source": [ - "df = load_pandas_df(DATA_FOLDER).sample(NUM_ROWS, random_state=RANDOM_STATE)" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
texttarge
80414فاز فريق الدفاع الحسني الجديدي على مضيفه الكوك...4
6649أمام آلاف مشاهد من لبنان ومصر والمغرب والإمارا...0
3722أخبارنا المغربية بعد أن أصدرت المحكمة الإبتداي...0
82317الفريق طبق قانونا قبل المصادقة عليه وجدل حول ه...4
5219المطرب المصري يخوض حملة إعلامية لترويج ألبومه ...0
\n", - "
" - ], - "text/plain": [ - " text targe\n", - "80414 فاز فريق الدفاع الحسني الجديدي على مضيفه الكوك... 4\n", - "6649 أمام آلاف مشاهد من لبنان ومصر والمغرب والإمارا... 0\n", - "3722 أخبارنا المغربية بعد أن أصدرت المحكمة الإبتداي... 0\n", - "82317 الفريق طبق قانونا قبل المصادقة عليه وجدل حول ه... 4\n", - "5219 المطرب المصري يخوض حملة إعلامية لترويج ألبومه ... 0" - ] - }, - "execution_count": 4, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "df.head()" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [], - "source": [ - "# set the text and label columns\n", - "text_col = df.columns[0]\n", - "label_col = df.columns[1]" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [], - "source": [ - "# remove empty documents\n", - "df = df[df[text_col].isna() == False]" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Inspect the distribution of labels:" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "4 5844\n", - "3 2796\n", - "1 2139\n", - "0 1917\n", - "2 1900\n", - "Name: targe, dtype: int64" - ] - }, - "execution_count": 7, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "df[label_col].value_counts()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We compare the counts with those presented in the author's [paper](http://article.nadiapub.com/IJGDC/vol11_no9/9.pdf), and infer the following label mapping:\n" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
label
0culture
1diverse
2economy
3politics
4sports
\n", - "
" - ], - "text/plain": [ - " label\n", - "0 culture\n", - "1 diverse\n", - "2 economy\n", - "3 politics\n", - "4 sports" - ] - }, - "execution_count": 8, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# ordered list of labels\n", - "labels = [\"culture\", \"diverse\", \"economy\", \"politics\", \"sports\"]\n", - "num_labels = len(labels)\n", - "pd.DataFrame({\"label\": labels})" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Next, we split the data for training and testing:" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Number of training examples: 11676\n", - "Number of testing examples: 2920\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/media/bleik2/miniconda3/envs/nlp_gpu/lib/python3.6/site-packages/sklearn/model_selection/_split.py:2179: FutureWarning: From version 0.21, test_size will always complement train_size unless both are specified.\n", - " FutureWarning)\n" - ] - } - ], - "source": [ - "df_train, df_test = train_test_split(df, train_size = TRAIN_SIZE, random_state=RANDOM_STATE)\n", - "print(\"Number of training examples: {}\".format(df_train.shape[0]))\n", - "print(\"Number of testing examples: {}\".format(df_test.shape[0]))" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Tokenize and Preprocess" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Before training, we tokenize the text documents and convert them to lists of tokens. The following steps instantiate a BERT tokenizer given the language, and tokenize the text of the training and testing sets." - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "100%|██████████| 11676/11676 [00:59<00:00, 196.42it/s]\n", - "100%|██████████| 2920/2920 [00:14<00:00, 197.99it/s]\n" - ] - } - ], - "source": [ - "tokenizer = Tokenizer(LANGUAGE, cache_dir=BERT_CACHE_DIR)\n", - "tokens_train = tokenizer.tokenize(list(df_train[text_col].astype(str)))\n", - "tokens_test = tokenizer.tokenize(list(df_test[text_col].astype(str)))" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "In addition, we perform the following preprocessing steps in the cell below:\n", - "- Convert the tokens into token indices corresponding to the BERT tokenizer's vocabulary\n", - "- Add the special tokens [CLS] and [SEP] to mark the beginning and end of a sentence\n", - "- Pad or truncate the token lists to the specified max length\n", - "- Return mask lists that indicate paddings' positions\n", - "\n", - "*See the original [implementation](https://github.com/google-research/bert/blob/master/run_classifier.py) for more information on BERT's input format.*" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": {}, - "outputs": [], - "source": [ - "tokens_train, mask_train, _ = tokenizer.preprocess_classification_tokens(\n", - " tokens_train, MAX_LEN\n", - ")\n", - "tokens_test, mask_test, _ = tokenizer.preprocess_classification_tokens(\n", - " tokens_test, MAX_LEN\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Create Model\n", - "Next, we create a sequence classifier that loads a pre-trained BERT model, given the language and number of labels." - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "metadata": {}, - "outputs": [], - "source": [ - "classifier = BERTSequenceClassifier(\n", - " language=LANGUAGE, num_labels=num_labels, cache_dir=BERT_CACHE_DIR\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Train\n", - "We train the classifier using the training examples. This involves fine-tuning the BERT Transformer and learning a linear classification layer on top of that:" - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "metadata": { - "scrolled": true - }, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "t_total value of -1 results in schedule not being applied\n", - "Iteration: 0%| | 1/365 [00:03<21:12, 3.49s/it]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "epoch:1/1; batch:1->37/365; average training loss:1.591262\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Iteration: 10%|█ | 38/365 [01:02<08:45, 1.61s/it]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "epoch:1/1; batch:38->74/365; average training loss:0.745935\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Iteration: 21%|██ | 75/365 [02:02<07:52, 1.63s/it]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "epoch:1/1; batch:75->111/365; average training loss:0.593934\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Iteration: 31%|███ | 112/365 [03:03<06:56, 1.65s/it]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "epoch:1/1; batch:112->148/365; average training loss:0.530150\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Iteration: 41%|████ | 149/365 [04:03<05:54, 1.64s/it]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "epoch:1/1; batch:149->185/365; average training loss:0.481620\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Iteration: 51%|█████ | 186/365 [05:05<05:02, 1.69s/it]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "epoch:1/1; batch:186->222/365; average training loss:0.455032\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Iteration: 61%|██████ | 223/365 [06:06<03:59, 1.69s/it]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "epoch:1/1; batch:223->259/365; average training loss:0.421702\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Iteration: 71%|███████ | 260/365 [07:08<02:56, 1.68s/it]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "epoch:1/1; batch:260->296/365; average training loss:0.401165\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Iteration: 81%|████████▏ | 297/365 [08:09<01:52, 1.65s/it]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "epoch:1/1; batch:297->333/365; average training loss:0.382719\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Iteration: 92%|█████████▏| 334/365 [09:12<00:52, 1.71s/it]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "epoch:1/1; batch:334->365/365; average training loss:0.372204\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Iteration: 100%|██████████| 365/365 [10:04<00:00, 1.63s/it]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "[Training time: 0.169 hrs]\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\n" - ] - } - ], - "source": [ - "with Timer() as t:\n", - " classifier.fit(\n", - " token_ids=tokens_train,\n", - " input_mask=mask_train,\n", - " labels=list(df_train[label_col]), \n", - " num_gpus=NUM_GPUS, \n", - " num_epochs=NUM_EPOCHS,\n", - " batch_size=BATCH_SIZE, \n", - " verbose=True,\n", - " ) \n", - "print(\"[Training time: {:.3f} hrs]\".format(t.interval / 3600))" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Score\n", - "We score the test set using the trained classifier:" - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Iteration: 100%|██████████| 92/92 [00:48<00:00, 2.25it/s]\n" - ] - } - ], - "source": [ - "preds = classifier.predict(\n", - " token_ids=tokens_test, input_mask=mask_test, num_gpus=NUM_GPUS, batch_size=BATCH_SIZE\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Evaluate Results\n", - "Finally, we compute the accuracy, precision, recall, and F1 metrics of the evaluation on the test set." - ] - }, - { - "cell_type": "code", - "execution_count": 15, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "accuracy: 0.9277397260273973\n", - "{\n", - " \"culture\": {\n", - " \"f1-score\": 0.9081761006289307,\n", - " \"precision\": 0.8848039215686274,\n", - " \"recall\": 0.9328165374677002,\n", - " \"support\": 387\n", - " },\n", - " \"diverse\": {\n", - " \"f1-score\": 0.9237983587338804,\n", - " \"precision\": 0.9471153846153846,\n", - " \"recall\": 0.9016018306636155,\n", - " \"support\": 437\n", - " },\n", - " \"economy\": {\n", - " \"f1-score\": 0.8547418967587034,\n", - " \"precision\": 0.8221709006928406,\n", - " \"recall\": 0.89,\n", - " \"support\": 400\n", - " },\n", - " \"macro avg\": {\n", - " \"f1-score\": 0.9099850933798536,\n", - " \"precision\": 0.9087524907040864,\n", - " \"recall\": 0.9125256551533433,\n", - " \"support\": 2920\n", - " },\n", - " \"micro avg\": {\n", - " \"f1-score\": 0.9277397260273973,\n", - " \"precision\": 0.9277397260273973,\n", - " \"recall\": 0.9277397260273973,\n", - " \"support\": 2920\n", - " },\n", - " \"politics\": {\n", - " \"f1-score\": 0.8734177215189873,\n", - " \"precision\": 0.8994413407821229,\n", - " \"recall\": 0.8488576449912126,\n", - " \"support\": 569\n", - " },\n", - " \"sports\": {\n", - " \"f1-score\": 0.9897913892587662,\n", - " \"precision\": 0.9902309058614565,\n", - " \"recall\": 0.9893522626441881,\n", - " \"support\": 1127\n", - " },\n", - " \"weighted avg\": {\n", - " \"f1-score\": 0.9279213601549715,\n", - " \"precision\": 0.9290922105520572,\n", - " \"recall\": 0.9277397260273973,\n", - " \"support\": 2920\n", - " }\n", - "}\n" - ] - } - ], - "source": [ - "report = classification_report(df_test[label_col], preds, target_names=labels, output_dict=True) \n", - "accuracy = accuracy_score(df_test[label_col], preds )\n", - "print(\"accuracy: {}\".format(accuracy))\n", - "print(json.dumps(report, indent=4, sort_keys=True))" - ] - }, - { - "cell_type": "code", - "execution_count": 16, - "metadata": {}, - "outputs": [ - { - "data": { - "application/scrapbook.scrap.json+json": { - "data": 0.9277397260273973, - "encoder": "json", - "name": "accuracy", - "version": 1 - } - }, - "metadata": { - "scrapbook": { - "data": true, - "display": false, - "name": "accuracy" - } - }, - "output_type": "display_data" - }, - { - "data": { - "application/scrapbook.scrap.json+json": { - "data": 0.9087524907040864, - "encoder": "json", - "name": "precision", - "version": 1 - } - }, - "metadata": { - "scrapbook": { - "data": true, - "display": false, - "name": "precision" - } - }, - "output_type": "display_data" - }, - { - "data": { - "application/scrapbook.scrap.json+json": { - "data": 0.9125256551533433, - "encoder": "json", - "name": "recall", - "version": 1 - } - }, - "metadata": { - "scrapbook": { - "data": true, - "display": false, - "name": "recall" - } - }, - "output_type": "display_data" - }, - { - "data": { - "application/scrapbook.scrap.json+json": { - "data": 0.9099850933798536, - "encoder": "json", - "name": "f1", - "version": 1 - } - }, - "metadata": { - "scrapbook": { - "data": true, - "display": false, - "name": "f1" - } - }, - "output_type": "display_data" - } - ], - "source": [ - "# for testing\n", - "sb.glue(\"accuracy\", accuracy)\n", - "sb.glue(\"precision\", report[\"macro avg\"][\"precision\"])\n", - "sb.glue(\"recall\", report[\"macro avg\"][\"recall\"])\n", - "sb.glue(\"f1\", report[\"macro avg\"][\"f1-score\"])" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "nlp_gpu", - "language": "python", - "name": "nlp_gpu" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.6.8" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/examples/text_classification/tc_mnli_xlnet.ipynb b/examples/text_classification/tc_mnli_xlnet.ipynb deleted file mode 100644 index a7ce53232..000000000 --- a/examples/text_classification/tc_mnli_xlnet.ipynb +++ /dev/null @@ -1,974 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "*Copyright (c) Microsoft Corporation. All rights reserved.*\n", - "\n", - "*Licensed under the MIT License.*\n", - "\n", - "# Text Classification of MultiNLI Sentences using XLNet\n", - "**XLNet: Generalized Autoregressive Pretraining for Language Understanding** [\\[1\\]](#References)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Table of Contents\n", - "1. [Introduction](#1.-Introduction)\n", - " * 1.1. What is XLNet?\n", - " * 1.2. How to use XLNet for Text Classification?\n", - "2. [Getting Started](#2.-Getting-Started) \n", - " * 2.1. Import Modules\n", - " * 2.2. Define Variables and Hyperparameters\n", - " * 2.3. Load Dataset\n", - "3. [Preprocessing Data](#3.-Preprocessing-Data)\n", - " * 3.1 Splitting Data\n", - " * 3.2. Tokenizing and Preprocess\n", - "4. [Model Training](#4.-Model-Training)\n", - " * 4.1 Create Model\n", - " * 4.2 Train Model\n", - " * 4.3 MLflow for train-validation loss plot\n", - "5. [Evaluation](#5.-Evaluation)\n", - " * 5.1 Predict\n", - " * 5.2 Report Classification Metrics\n", - " * 5.3 Confusion Matrix\n", - "6. [References](#References)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 1. Introduction\n", - "------------------\n", - "In this notebook, we fine-tune and evaluate a pretrained [XLNet](https://arxiv.org/abs/1906.08237) model on a subset of the [MultiNLI](https://www.nyu.edu/projects/bowman/multinli/) dataset.\n", - "\n", - "We use a [sequence classifier](../../utils_nlp/xlnet/sequence_classification.py) that wraps [Hugging Face's PyTorch implementation](https://github.com/huggingface/pytorch-transformers) of CMU and Google's [XLNet](https://github.com/zihangdai/xlnet)." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### 1.1. What is XLNet?\n", - "\n", - "[XLNet](https://arxiv.org/pdf/1906.08237.pdf) is a generalized autoregressive pretraining method incorporating 3 ideas:\n", - "1. maximum expected likelihood over all permutations of the factorization order that enables learning bidirectional context \n", - "\n", - "2. autoregressive formulation that overcomes the limitations of BERT [\\[2\\]](#References) \n", - "\n", - "3. relative positional embeddings and recurrence mechanism from Transformer XL [\\[3\\]](#References)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### 1.2. How to use XLNet for Text Classification?\n", - "\n", - "Using a pre-trained XLNet model, we can fine-tune the model for text classification by training it on the MNLI dataset [\\[4\\]](#References). The Multi-Genre Natural Language Inference (MultiNLI) corpus is a crowd-sourced collection of 433k sentence pairs annotated with textual entailment information. \n", - "\n", - "This notebook contains an end-to-end walkthrough of a pipeline to run Transformer's reimplementation [\\[5\\]](#References) of the XLNet model." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 2. Getting Started\n", - "--------------\n", - "In this section, we will:\n", - "\n", - "1. Import the modules required to run XLNet and this notebook\n", - "2. Define and discuss variables and hyperparameters in the XLNet model\n", - "3. Load the MNLI dataset using Pandas" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### 2.1. Import Modules\n", - "\n", - "Some key modules we will use include:\n", - "\n", - "1. utils_nlp: contains the xlnet model from Hugging Face\n", - "2. mlflow: track, log and visualize key metrics in the machine learning process" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "%load_ext autoreload\n", - "%autoreload 2" - ] - }, - { - "cell_type": "code", - "execution_count": 19, - "metadata": {}, - "outputs": [], - "source": [ - "import sys\n", - "sys.path.append(\"../../\")\n", - "import os\n", - "import numpy as np\n", - "import pandas as pd\n", - "import random\n", - "import torch\n", - "import torch.nn as nn\n", - "from sklearn.metrics import classification_report\n", - "from sklearn.preprocessing import LabelEncoder\n", - "from sklearn.model_selection import train_test_split\n", - "\n", - "from utils_nlp.dataset.multinli import load_pandas_df\n", - "from utils_nlp.eval.classification import eval_classification, plot_confusion_matrix\n", - "from utils_nlp.common.timer import Timer\n", - "from utils_nlp.models.xlnet.common import Language, Tokenizer\n", - "from utils_nlp.models.xlnet.sequence_classification import XLNetSequenceClassifier\n", - "from utils_nlp.models.xlnet.common import log_xlnet_params\n", - "import mlflow\n", - "import datetime" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### 2.2. Define Variables and Hyperparameters\n", - "\n", - "**Global Variables:**\n", - "- DATA_FOLDER : data downloaded to this folder \n", - "- XLNET_CACHE_DIR : model caches information to this folder \n", - "- LANGUAGE : which pretrained model to use \n", - "- LABEL_COL : column of data containing label \n", - "- TEXT_COL : column of data containing sentence \n", - "\n", - "**Hyperparmeters:**\n", - "- MAX_SEQ_LENGTH : maximum sentence length to pad or truncate examples to \n", - "- WEIGHT_DECAY : regularization on model weights \n", - "- WARMUP_STEPS : number of steps to increase learning rate over at start of training (then decrease learning rate for duration of training) \n", - "\n", - "**Debug Switch:**\n", - "- DEBUG : If True, will train and evaluate model only on a small portion of data " - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [], - "source": [ - "DATA_FOLDER = \"../../../temp\"\n", - "XLNET_CACHE_DIR=\"../../../temp\"\n", - "LANGUAGE = Language.ENGLISHCASED\n", - "MAX_SEQ_LENGTH = 128\n", - "BATCH_SIZE = 16\n", - "NUM_GPUS = 1\n", - "NUM_EPOCHS = 1\n", - "TRAIN_SIZE = 0.6\n", - "VAL_SIZE = 0.1\n", - "LABEL_COL = \"genre\"\n", - "TEXT_COL = \"sentence1\"\n", - "WEIGHT_DECAY = 0.0\n", - "WARMUP_STEPS = 1000\n", - "\n", - "### Hyperparamters to tune\n", - "MAX_SEQ_LENGTH = 128\n", - "LEARNING_RATE = 5e-5\n", - "ADAM_EPSILON = 1e-8\n", - "\n", - "DEBUG = False\n", - "LOGGING_STEPS = 10\n", - "SAVE_STEPS = 100\n", - "VAL_STEPS = 100\n", - "mlflow.start_run(run_name = datetime.datetime.now())\n", - "log_xlnet_params(locals())" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### 2.3. Load Dataset\n", - "We start by loading a subset of the data. The following function also downloads and extracts the files, if they don't exist in the data folder.\n", - "\n", - "The MultiNLI dataset is mainly used for natural language inference (NLI) tasks, where the inputs are sentence pairs and the labels are entailment indicators. The sentence pairs are also classified into *genres* that allow for more coverage and better evaluation of NLI models.\n", - "\n", - "For our classification task, we use the first sentence only as the text input, and the corresponding genre as the label. We select the examples corresponding to one of the entailment labels (*neutral* in this case) to avoid duplicate rows, as the sentences are not unique, whereas the sentence pairs are." - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [], - "source": [ - "df = load_pandas_df(DATA_FOLDER, \"train\")\n", - "df = df[df[\"gold_label\"]==\"neutral\"] # get unique sentences\n", - "\n", - "if DEBUG:\n", - " inds = random.sample(range(len(df.index)), 10000)\n", - " df = df.iloc[inds]" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
annotator_labelsgenregold_labelpairIDpromptIDsentence1sentence1_binary_parsesentence1_parsesentence2sentence2_binary_parsesentence2_parse
0[neutral]governmentneutral31193n31193Conceptually cream skimming has two basic dime...( ( Conceptually ( cream skimming ) ) ( ( has ...(ROOT (S (NP (JJ Conceptually) (NN cream) (NN ...Product and geography are what make cream skim...( ( ( Product and ) geography ) ( ( are ( what...(ROOT (S (NP (NN Product) (CC and) (NN geograp...
4[neutral]telephoneneutral50563n50563yeah i tell you what though if you go price so...( yeah ( i ( ( tell you ) ( what ( ( though ( ...(ROOT (S (VP (VB yeah) (S (NP (FW i)) (VP (VB ...The tennis shoes have a range of prices.( ( The ( tennis shoes ) ) ( ( have ( ( a rang...(ROOT (S (NP (DT The) (NN tennis) (NNS shoes))...
6[neutral]travelneutral42487n42487But a few Christian mosaics survive above the ...( But ( ( a ( few ( Christian mosaics ) ) ) ( ...(ROOT (S (CC But) (NP (DT a) (JJ few) (JJ Chri...Most of the Christian mosaics were destroyed b...( ( Most ( of ( the ( Christian mosaics ) ) ) ...(ROOT (S (NP (NP (JJS Most)) (PP (IN of) (NP (...
12[neutral]slateneutral32819n32819It's not that the questions they asked weren't...( It ( ( ( ( 's not ) ( that ( ( ( the questio...(ROOT (S (NP (PRP It)) (VP (VBZ 's) (RB not) (...All of the questions were interesting accordin...( ( All ( of ( the questions ) ) ) ( ( ( were ...(ROOT (S (NP (NP (DT All)) (PP (IN of) (NP (DT...
13[neutral]travelneutral52772n52772Thebes held onto power until the 12th Dynasty,...( Thebes ( ( ( ( ( held ( onto power ) ) ( unt...(ROOT (S (NP (NNS Thebes)) (VP (VBD held) (PP ...The capital near Memphis lasted only half a ce...( ( ( The capital ) ( near Memphis ) ) ( ( ( (...(ROOT (S (NP (NP (DT The) (NN capital)) (PP (I...
\n", - "
" - ], - "text/plain": [ - " annotator_labels genre gold_label pairID promptID \\\n", - "0 [neutral] government neutral 31193n 31193 \n", - "4 [neutral] telephone neutral 50563n 50563 \n", - "6 [neutral] travel neutral 42487n 42487 \n", - "12 [neutral] slate neutral 32819n 32819 \n", - "13 [neutral] travel neutral 52772n 52772 \n", - "\n", - " sentence1 \\\n", - "0 Conceptually cream skimming has two basic dime... \n", - "4 yeah i tell you what though if you go price so... \n", - "6 But a few Christian mosaics survive above the ... \n", - "12 It's not that the questions they asked weren't... \n", - "13 Thebes held onto power until the 12th Dynasty,... \n", - "\n", - " sentence1_binary_parse \\\n", - "0 ( ( Conceptually ( cream skimming ) ) ( ( has ... \n", - "4 ( yeah ( i ( ( tell you ) ( what ( ( though ( ... \n", - "6 ( But ( ( a ( few ( Christian mosaics ) ) ) ( ... \n", - "12 ( It ( ( ( ( 's not ) ( that ( ( ( the questio... \n", - "13 ( Thebes ( ( ( ( ( held ( onto power ) ) ( unt... \n", - "\n", - " sentence1_parse \\\n", - "0 (ROOT (S (NP (JJ Conceptually) (NN cream) (NN ... \n", - "4 (ROOT (S (VP (VB yeah) (S (NP (FW i)) (VP (VB ... \n", - "6 (ROOT (S (CC But) (NP (DT a) (JJ few) (JJ Chri... \n", - "12 (ROOT (S (NP (PRP It)) (VP (VBZ 's) (RB not) (... \n", - "13 (ROOT (S (NP (NNS Thebes)) (VP (VBD held) (PP ... \n", - "\n", - " sentence2 \\\n", - "0 Product and geography are what make cream skim... \n", - "4 The tennis shoes have a range of prices. \n", - "6 Most of the Christian mosaics were destroyed b... \n", - "12 All of the questions were interesting accordin... \n", - "13 The capital near Memphis lasted only half a ce... \n", - "\n", - " sentence2_binary_parse \\\n", - "0 ( ( ( Product and ) geography ) ( ( are ( what... \n", - "4 ( ( The ( tennis shoes ) ) ( ( have ( ( a rang... \n", - "6 ( ( Most ( of ( the ( Christian mosaics ) ) ) ... \n", - "12 ( ( All ( of ( the questions ) ) ) ( ( ( were ... \n", - "13 ( ( ( The capital ) ( near Memphis ) ) ( ( ( (... \n", - "\n", - " sentence2_parse \n", - "0 (ROOT (S (NP (NN Product) (CC and) (NN geograp... \n", - "4 (ROOT (S (NP (DT The) (NN tennis) (NNS shoes))... \n", - "6 (ROOT (S (NP (NP (JJS Most)) (PP (IN of) (NP (... \n", - "12 (ROOT (S (NP (NP (DT All)) (PP (IN of) (NP (DT... \n", - "13 (ROOT (S (NP (NP (DT The) (NN capital)) (PP (I... " - ] - }, - "execution_count": 5, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "df.head()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The examples in the dataset are grouped into 5 genres:" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "telephone 27783\n", - "government 25784\n", - "travel 25783\n", - "fiction 25782\n", - "slate 25768\n", - "Name: genre, dtype: int64" - ] - }, - "execution_count": 6, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "df[LABEL_COL].value_counts()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 3. Preprocessing Data\n", - "-------------" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### 3.1. Splitting Data\n", - "We split the data into 3 parts with the following proportions:\n", - "- Train Set 60%\n", - "- Validation Set 10%\n", - "- Test Set 30%\n", - "\n", - "Then we encode the class labels from categories into integers." - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/home/amgupte/anaconda3/envs/nlp_gpu/lib/python3.6/site-packages/sklearn/model_selection/_split.py:2179: FutureWarning: From version 0.21, test_size will always complement train_size unless both are specified.\n", - " FutureWarning)\n" - ] - } - ], - "source": [ - "# split\n", - "df_trainval, df_test = train_test_split(df, train_size = TRAIN_SIZE + VAL_SIZE, random_state=0)\n", - "df_train, df_val = train_test_split(df_trainval, train_size = TRAIN_SIZE / (TRAIN_SIZE + VAL_SIZE), random_state=0)\n", - "\n", - "# encode labels\n", - "label_encoder = LabelEncoder()\n", - "labels_train = label_encoder.fit_transform(df_train[LABEL_COL])\n", - "labels_val = label_encoder.transform(df_val[LABEL_COL])\n", - "labels_test = label_encoder.transform(df_test[LABEL_COL])\n", - "label_list = label_encoder.classes_\n", - "\n", - "num_labels = len(np.unique(labels_train))" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We check to ensure the label classes are balanced in the train and validation set." - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "telephone 16586\n", - "travel 15507\n", - "slate 15497\n", - "fiction 15478\n", - "government 15472\n", - "Name: genre, dtype: int64" - ] - }, - "execution_count": 8, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "df_train[LABEL_COL].value_counts()" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "telephone 8434\n", - "travel 7734\n", - "government 7715\n", - "fiction 7705\n", - "slate 7682\n", - "Name: genre, dtype: int64" - ] - }, - "execution_count": 9, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "df_test[LABEL_COL].value_counts()" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Number of unique labels: 5\n", - "Number of training examples: 78540\n", - "Number of testing examples: 13090\n", - "Number of testing examples: 39270\n" - ] - } - ], - "source": [ - "print(\"Number of unique labels: {}\".format(num_labels))\n", - "print(\"Number of training examples: {}\".format(df_train.shape[0]))\n", - "print(\"Number of testing examples: {}\".format(df_val.shape[0]))\n", - "print(\"Number of testing examples: {}\".format(df_test.shape[0]))" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### 3.2. Tokenize and Preprocess\n", - "Before training, we tokenize the text documents and convert them to lists of tokens. The following steps instantiate a XLNet tokenizer given the language, and tokenize the text of the training and testing sets.\n", - "\n", - "We perform the following preprocessing steps in the cell below:\n", - "- Convert the tokens into token indices corresponding to the XLNet-base tokenizer's vocabulary\n", - "- Add the special tokens [CLS] and [SEP] to mark the end of a sentence\n", - "- Pad or truncate the token lists to the specified max length\n", - "- Return id lists that indicate which word the tokens map to\n", - "- Return mask lists that indicate paddings' positions\n", - "- Return segment type id lists that indicates which segment each the tokens belongs to\n", - "\n", - "**See figure below for the step-by-step tokenization process** \n", - "\n", - "\n", - "*For more information on XLNet's input format, see transformer [implementation](https://github.com/huggingface/pytorch-transformers/blob/master/examples/utils_glue.py)*" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": {}, - "outputs": [], - "source": [ - "tokenizer = Tokenizer(LANGUAGE, cache_dir=XLNET_CACHE_DIR)\n", - "\n", - "train_input_ids, train_input_mask, train_segment_ids = tokenizer.preprocess_classification_tokens(list(df_train[TEXT_COL]), MAX_SEQ_LENGTH)\n", - "val_input_ids, val_input_mask, val_segment_ids = tokenizer.preprocess_classification_tokens(list(df_val[TEXT_COL]), MAX_SEQ_LENGTH)\n", - "test_input_ids, test_input_mask, test_segment_ids = tokenizer.preprocess_classification_tokens(list(df_test[TEXT_COL]), MAX_SEQ_LENGTH)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 4. Model Training\n", - "----------------------------\n", - "### 4.1. Create model\n", - "First, we create a sequence classifier that loads a pre-trained XLNet model, given the language and number of labels." - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "metadata": {}, - "outputs": [], - "source": [ - "classifier = XLNetSequenceClassifier(\n", - " language=LANGUAGE,\n", - " num_labels=num_labels,\n", - " cache_dir=XLNET_CACHE_DIR,\n", - " num_gpus=NUM_GPUS, \n", - " num_epochs=NUM_EPOCHS,\n", - " batch_size=BATCH_SIZE\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### 4.2. Train Model\n", - "\n", - "We train the classifier using the training examples. This involves fine-tuning the XLNet Transformer and learning a linear classification layer on top of that\n", - "\n", - "#### 4.2.1. Machine Specifications\n", - "\n", - "We're using two P4000 GPUs - each with 8GB of memory - to train this model. \n", - "\n", - "For a combined GPU memory of 16GB and sequence length of 128 tokens, the maximum batch size we could use for training is 32. Without validation, the maximum batch size for training is 56. \n", - "\n", - "#### 4.2.2. Shuffling of the training set before each epoch \n", - "\n", - "We shuffle data in the mini-batch training before each epoch to prevent overfitting that occurs when the order of data within every epoch is the same. " - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Iteration: 0%| | 1/4909 [00:00<55:30, 1.47it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "epoch:1/1; batch:1->491/4909; average training loss:1.656134\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Iteration: 10%|█ | 492/4909 [15:47<47:04, 1.56it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "epoch:1/1; batch:492->982/4909; average training loss:0.927851; average val loss:0.716668\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Iteration: 20%|██ | 983/4909 [34:12<42:38, 1.53it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "epoch:1/1; batch:983->1473/4909; average training loss:0.760149; average val loss:0.540830\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Iteration: 30%|███ | 1474/4909 [52:36<36:25, 1.57it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "epoch:1/1; batch:1474->1964/4909; average training loss:0.679937; average val loss:0.506850\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Iteration: 40%|████ | 1965/4909 [1:10:57<31:46, 1.54it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "epoch:1/1; batch:1965->2455/4909; average training loss:0.630495; average val loss:0.489841\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Iteration: 50%|█████ | 2456/4909 [1:29:22<26:32, 1.54it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "epoch:1/1; batch:2456->2946/4909; average training loss:0.592024; average val loss:0.409858\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Iteration: 60%|██████ | 2947/4909 [1:47:46<21:12, 1.54it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "epoch:1/1; batch:2947->3437/4909; average training loss:0.562752; average val loss:0.388481\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Iteration: 70%|███████ | 3438/4909 [2:06:12<16:04, 1.53it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "epoch:1/1; batch:3438->3928/4909; average training loss:0.536014; average val loss:0.362622\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Iteration: 80%|████████ | 3929/4909 [2:24:40<10:45, 1.52it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "epoch:1/1; batch:3929->4419/4909; average training loss:0.515133; average val loss:0.319612\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Iteration: 90%|█████████ | 4420/4909 [2:43:03<05:40, 1.44it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "epoch:1/1; batch:4420->4909/4909; average training loss:0.494290; average val loss:0.318628\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Iteration: 100%|██████████| 4909/4909 [3:01:23<00:00, 2.22s/it]\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "[Training time: 3.026 hrs]\n" - ] - } - ], - "source": [ - "with Timer() as t:\n", - " classifier.fit(\n", - " token_ids=train_input_ids,\n", - " input_mask=train_input_mask,\n", - " token_type_ids=train_segment_ids,\n", - " labels=labels_train, \n", - " val_token_ids=val_input_ids,\n", - " val_input_mask=val_input_mask,\n", - " val_token_type_ids=val_segment_ids,\n", - " val_labels=labels_val,\n", - " verbose=True,\n", - " logging_steps = LOGGING_STEPS,\n", - " save_steps = SAVE_STEPS,\n", - " val_steps = VAL_STEPS,\n", - " ) \n", - "print(\"[Training time: {:.3f} hrs]\".format(t.interval / 3600))" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### 4.3. MLFlow Train-Validation Loss Plot \n", - "\n", - "During training, MLflow logs the loss of the training and validation batches and can automatically generate figures of the losses over time. The figure below enables us to visualize the model performance against the number of training iterations. \n", - "" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 5. Evaluation\n", - "-------------------------\n", - "### 5.1. Predict\n", - "We score the test set using the trained classifier:" - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "39280it [07:46, 84.27it/s] \n" - ] - } - ], - "source": [ - "preds = classifier.predict(\n", - " token_ids=test_input_ids,\n", - " input_mask=test_input_mask,\n", - " token_type_ids=test_segment_ids,\n", - " num_gpus=NUM_GPUS,\n", - " batch_size=BATCH_SIZE,\n", - " probabilities=False\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### 5.2. Report Classification Metrics\n", - "Finally, we compute the accuracy, precision, recall, and F1 metrics of the evaluation on the test set." - ] - }, - { - "cell_type": "code", - "execution_count": 15, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - " precision recall f1-score support\n", - "\n", - " fiction 0.85 0.89 0.87 7705\n", - " government 0.91 0.90 0.90 7715\n", - " slate 0.78 0.77 0.77 7682\n", - " telephone 0.99 0.99 0.99 8434\n", - " travel 0.91 0.89 0.90 7734\n", - "\n", - " micro avg 0.89 0.89 0.89 39270\n", - " macro avg 0.89 0.89 0.89 39270\n", - "weighted avg 0.89 0.89 0.89 39270\n", - "\n" - ] - } - ], - "source": [ - "cls_report = classification_report(labels_test, preds, target_names=label_encoder.classes_,output_dict=True)\n", - "print(classification_report(labels_test, preds, target_names=label_encoder.classes_))\n", - "\n", - "cls_report_df = pd.DataFrame(cls_report)\n", - "cls_report_df.to_csv(path_or_buf=os.path.join(os.getcwd(),\"checkpoints\",\"cls_report.csv\"))\n", - "mlflow.log_artifact(os.path.join(os.getcwd(),\"checkpoints\",\"cls_report.csv\"))\n", - "mlflow.end_run()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### 5.3. Confusion Matrix\n", - "The following confusion matrix - created using the data visualization library 'Seaborn' - allows us to easily identify which classes the model performed better or worse in. " - ] - }, - { - "cell_type": "code", - "execution_count": 26, - "metadata": {}, - "outputs": [ - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAhkAAAFXCAYAAAAPoFwKAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nOzdd3xO1x/A8c+TnYgkYmYZGRJbKrFrFjFittrqDzVqj6pRaqvRoFpi1ay9FSFCUbVn7Z0EWQRBhkTm8/sjPPF4snki+L5fr7xenpNz7j33OPc833vuuTcKpVKpRAghhBDiLdN51xUQQgghxIdJggwhhBBCaIUEGUIIIYTQCgkyhBBCCKEVEmQIIYQQQiskyBBCCCGEVkiQIcRHLigoiI4dO1KxYkX69ev3xttr1KgRW7dufQs1y5+8vb3p3Lnzu66GEO8FhbwnQ4j8Kzg4GG9vb44dO0ZUVBTW1tY0bNiQHj16UKRIkbeyjzFjxhAaGsqUKVMwNTXFzMzsjbb3+PFjTExMMDIyeiv1ywsnT56kS5cu3LhxI8u8z549IzExEQsLizyomRDvN5nJECKfCggIoEOHDkRFRfH777/j5+fH5MmTefr0KevXr39r+wkJCcHV1RVra+s3DjAALC0t36sAI7uUSiWJiYkUKFBAAgwhskmCDCHyqYkTJ1KqVCkWLFiAm5sb1tbWuLm5MW3aNLp06aLKt3z5cho0aEClSpX46quvuHLliup3W7dupVGjRvj6+tKoUSPc3d0ZPXo0CQkJQOqtjePHjzNv3jycnZ3ZunWrqsyrXr9FsHPnTjw8PKhUqRJ16tRh7Nixqt+9frvk0qVLfPXVV1SqVIkGDRrw559/qm3b2dmZbdu20aVLF6pUqUKHDh24efNmhu1y8uRJnJ2dOXLkCM2aNcPV1ZUJEyaQlJTEzJkzcXd3p3Hjxhw9elRVJiAggJ49e1KjRg3c3Nz47rvvCA4OBlKDrJft6ezsrGqHl583b95M586dqVSpEsePH1dri/DwcNzd3dm5c6dqX3PnzqVp06bExcVleAxCfCwkyBAiH3r8+DGnTp2iW7duKBQKjd+/nHHw9fVlzpw5DBs2jG3btuHo6EivXr2IjY1V5Y2IiMDHx4cFCxYwZ84c9u7dy+bNmwHYvHkzrq6udO/enSNHjtCiRYss6/bgwQNGjhxJ//798fPzY+HChVSoUCHdvM+ePaNXr16ULVuWbdu2MXToUNWszKvmzp3Lt99+y7Zt27C0tGT06NFZ1mP58uX8/vvvzJ49my1bttCzZ0+MjIzYtGkTDRs2ZOTIkSQmJgIQGxuLh4cHa9euZe3atRgZGfHDDz8AYGVlhbe3NwBHjhzRaIe5c+fSuXNndu/eTcWKFdXqULx4cUaPHs3PP//Mw4cPuXr1Kn/88QdTp07F2Ng4y2MQ4kOn964rIITQFBwcjFKppEyZMpnmW7FiBZ07d6ZVq1YATJgwgcOHD+Pj48OXX34JQEJCAlOmTMHS0hIADw8PTp8+TadOnbC0tERfXx8TExOKFi2arbo9ePAAQ0NDGjdujImJCTY2NlSqVCndvD4+PhgaGjJ+/Hh0dXVxcHDgxo0b/Pnnn3h4eKjyffPNN6rZk759+/L111/z/PnzTG+7DBs2jHLlylGuXDlq1KjBo0ePGDBgAAB9+vRh1apVBAUF4eDgQKVKldTqOHHiRGrVqkVYWBjW1taYm5sDpNsGX375JU2bNs2wHm3btmXPnj2MHTuW0NBQOnXqhJubWyYtKMTHQ2YyhHiPBQYGUqVKFdVnPT09KlasSGBgoCqtSJEiqgADUr9IIyIicr1PFxcXnJ2d+eyzzxg5ciS7d+9WzRikV7+KFSuiq6urSqtatapa/QDKli2rVj9Inc3JjKOjo+rfhQsXxsHBQe3zq9uIiYlh0qRJNGvWjE8++YTGjRsDcO/evSyPt3z58lnmmTRpEidPniQ+Pp4hQ4ZkmV+Ij4UEGULkQ3Z2dgDcvn37jbelr6+v9lmhUJDZQ2U6Ojoav09KSlL9W09Pj1WrVjFr1iwKFy7MjBkz6NSpU7qBRnYfXtPTS5tUfXl7KCUlJdMyrx6XQqHQ+PzqNry8vDh9+jQ//fQTGzduZNOmTRrHlZHsLGK9fv06CQkJPH36lKioqCzzC/GxkCBDiHzI0tKS6tWrs2LFinS/qKOjowEoU6YMFy5cUKUnJSVx+fJl7O3tc73vQoUK8eTJE5KTk1Vprz/aqaurS82aNRk+fDibNm3i4sWL6S7WtLe35/Lly2rbOn/+/BvVLzfOnTvHl19+Sf369XF0dFS130svg5xX65ld0dHRjBkzhmHDhlGtWjXGjRv3VuosxIdAggwh8qlx48YRGBhI9+7dOXbsGCEhIfz333+MHj2aFStWANClSxdWrVrFrl27CAgIYMKECSQkJKjWaORGpUqVUCqVzJs3j7t377Jy5UrOnDmj+v2FCxdYtGgRV65cITQ0lG3btmFoaIiVlZXGtjw9PYmLi2PixIkEBASwc+dOVq9erfZ0TF6ws7PD19cXf39/zpw5g5eXl9rvra2tATh8+DCPHz9WPX2THVOnTsXKyoquXbsyadIkzp07x7Zt295q/YV4X0mQIUQ+5eTkxObNmylcuDDDhg2jefPmjBo1CnNzc77++msAWrVqRf/+/fHy8qJNmzbcunWLRYsWUaBAgVzv19LSkmnTprFt2zbatm3L9evXVfsDMDU15cSJE3Tv3p0WLVqwa9cuvL291dZ9vJp38eLFXL9+nTZt2jBjxgwGDRqUradY3qaRI0eiVCpp374948aNY/DgwWq/t7Kyok+fPowcOZJatWqpPZKamX///RdfX1+mTZuGjo4ORYsWZcyYMUyZMoXw8HBtHIoQ7xV546cQQgghtEJmMoQQQgihFRJkCCGEEEIrJMgQQgghhFZIkCGEEEIIrZAgQwghhBBaIX+75D1j7Dn/XVch33q8td+7rkK+Fhmb/qu/BVgU0M8600cqKVkeQMyMqaHmHzB8W4xdB+S4TNy5uVqoSe5JkCGEEELkR4r3/2aDBBlCCCFEfqTQ3ixJXpEgQwghhMiPZCZDCCGEEFohMxlCCCGE0AqZyRBCCCGEVshMhhBCCCG0QmYyhBBCCKEVH8BMxvsfJgkhhBAiX5KZDCGEECI/ktslQgghhNCKD+B2iQQZQgghRH4kMxlCCCGE0AqZyRBCCCGEVshMhhBCCCG0QoIMIYQQQmiFjtwuEUIIIYQ2yEyGEEIIIbRCFn4KIYQQQitkJkO8T2yLmDK9Zx0aVbVFoVDwz/kQhi85QvDDmCzL2hU1Zdw31alf2YbCZkaEPnrGliP+zNj0H7HxSap8xoZ6DO3gSsd6TtgWMSUi6jn/Xgpl0ppTBD2IVuUb/bU7Yzq5a+zH50QgHaf4vZ0DzqH79+4xc/o0Thw/ilKppEbN2gwf+RNWVtZZlo2Pj2ee9+/47vQhOjoKZ5dyDB4yjGpu6seYkpLC8qWL2bxpAxGPHlK6dBl69e3PZ02aqfKcPnWS77p3yXBfK9dsoHKVqrk/0Fx4EH6Pub9N58zJ4yhRUs29JgN/GEnxElZZlo2Pj2fZH97s3b2TmJhoHJ1c6DNgCFU+cVPLF/n0KSuWLuDY4X+JiHiIpWURatWtx7c9+2JRyFKV74+5v3Hi2CEe3L9PYmIixUtY8ZlHS778pitGRsZv/dizcv/ePWZ4vdJvatVmxI8/YWWd/X6zyyet33z/Qyb9ZuMGHr3oN7379uezps3U8u3Y9hcH/znA1SuXuXcvjNZt2vHz1F/e6vHm1P3795g1fRonThwDpZLqNWszdMSobJ9XC+bOxneXDzHRUZR1dmHQ98P45JX2uXvnNhvXr+XM6ZOEhoRgUqAAFSpUpO+AwZR1dlHb3oSxo7h08QIPH4STkqLE1s6Otu0/54svO6Grq/vWj/2NfQAzGQqlUql815UQ2WfsOT935Qz1ODWnI/GJyUxcfQqlUsn4/9XAxFAP94Eb1AKF15kY6nFidkf09XSYvPY0wQ+jcXMqxphO1dl16g6dp+9V5f1z2Gd41izD5DWnOev/ALuiBRnbyZ3kFCXVB23g2fPU/bwMMhqN2EpySloXfBz9HP+wyFwd4+Ot/XJVDiAuLo6OHdpgYGBA/4Hfo1DAPO/ZPI+LY9PWHRibmGRaftSPQzl86F+GDB2Bra0dG9at4eiRQ6xYswEXl3KqfN6zf2Pln0sZMGgI5StUwG+3L1s3b2TOvD/4tF59AGJiYggM8NfYx4Rxo4mKjGTP/n9zNSBGxibmuAzA8+dxdP+mAwb6BvToMxCFQsGShd7EP49j2dqtGBtn3jY/j/2RE0cP0WfQUKytbflr8zpOHj/C/KVrcCqb+iWgVCoZ8F1ngoPu0r1Xf0qVsefO7UCWLfTGtmRp5i9djeLFgDvL62ds7UpRslRp9PUNuHzpPKuXL8K9Zh2mzvTO1TFaFNDPVbm4uDg6tm+DvoEBAwal9pu5c2bz/HlqvzHJqt+MeKXf2Nmxft0ajh4+xMo1G3App95vVixfysDBQyhXPq3feM9P6zcAvXt248njx5SvWJG/9/jRqHGTNw4ykpJz/xURFxfH11+0xUDfgL4DB6NQKJjv/TvPnz9nw+btWZ5Xo0cO48jhfxk8ZDi2tnZs3LCWY0cOsXzVepxfnFcb1q1m6+aNtGrdFpdy5YmOjmbl8iXcuH6NZSvXUq58RdX2Ro34AddP3LC1s0OhUHD82BHWrlrBl1//j+EjR+fqGE0NtRcIGDedkeMycXuHa6EmufdRz2Q8evSI77//nqtXr5KQkECbNm2YMmVKjrfTs2dPWrZsSbt27bRQy7eje9PylCluRuW+awm8FwXApTsRXP7jG3p6VGDO9gsZlq1V3gonGwtajfNh/7lgAA5dCqNQQSO+b1cVY0M94uKTMDLQpUNdR2ZtOcdvf51XlX/wNJYdEz2pVc6KfS/Kv3TqRrhakPGubN28kdCQYLbt9KNkyVIAlC3rTOuWzdi8aQOdu3bLsOyN69fZvWsnE36eStt2HQCo5uZOh7YtWTB3NrPnLgTgcUQEK/9cSrcevejarQcA7tVrEhx0lzm/zVR9WZiammrMVISFhXI7MIDOXbvl+RXXzm2buRcawqpNO7G1KwmAg2NZvvm8JTu2buLLb7pmWNb/5nX27dnFj2N/poVn6vlR5RM3vv2qLcv+mMu0X+cCEBJ0l8sXzzN01Hhat/sCANdq1dFRKJjl9TPBQXcoWaoMAD/8OFZtH9Wq1yT+eRxrVizl6dMnWFgUeuttkJGtmzcSEhLM9p1+lCyV2m+cyjrTukUzNm/cQJdvM+83vrt2MnGyer9p36Yl8+fOZs681H4TERHBiuVL6d4zrd9Ur5Hab2a/0m8AFixaio5O6hT70SOHtXLMOfHXlk2EhgSzdcdu7F6cV05OzrTzbMaWzRv4X5eM2+fmjev4+e5k/KQptG6b2j6fuLnTsV0rFs6bw2/eCwBo6tGSjl99owpCIfW88vRozLrVq5g01UuVPm36LLV91Kpdl0cPHrBj25ZcBxla9QHMZLz/N3zewIYNG7CwsODs2bNcvnw5WwGGt7c3w4YNU0tbsmRJvg4wAFrWKM2pG+GqAAPgbng0x6/do1XN0pmWNdBL7SbRsQlq6ZHP4tFRKHh5Gujp6qCnq0N03Ov5Uj/r5OPHsf49eIBKlauoAgwAG1s7qrp+wsF/9mdRdj96evo082ihStPT06OZR0uOHT1CQkLq8R87epjExERaerZWK9+iVWtu3bpJaIh6APaqXT7bUSqVtG6T9/3s6KGDlK9YWRVgAFjZ2FKxsitHD/2TednDB9HT06NREw9Vmp6eHo2aenD6xFFV2yQmpc6yFChQQK28acGCQOrtgsyYmVuotp2XDv5zgMqVq6gCDADbbPabg/+k3288mmev37T0bM2tmzcJeaXfvAww8otDL84rO7XzypYqVV35N8vz6gB6evo0aabePk09WnD8WFr7FCpUSC3AAChYsCAlS5XmwYPwLOtobmGBrm4+vd5W6OT8J5/JfzXKQ2FhYTg4OGh00A9RuZKWXAl6rJF+NegJLnaW6ZRIc+B8CLdCnzL525q42BWigJEe9Svb0M+zMov9rqhutcTEJbLmwA36tapMvUrWFDDSo1zJQkztVosLgY/450KIxrZvLe9CzLY+3Fjamclda2Jk8G7uiwb4++PoVFYj3d7BMd1bF6+XtbG1wdhYfT2Ag6MjiYmJBAXdTc0X4I+BgYFaIJOaz+nF7wMy3IfPju2UK18h3Tpq251Af8o4OGmkl7Z34M7tjOv8sqyVta3GWonSZVLbJjQ4CIAy9o5UcXVj5dI/uH71MrGxsVy7cokVSxdSo/anlC7joLHtpKQkYmNjOXPqOBvXrqSFZztMTQu+wZHmXIC/Pw7p/J84ZKffBGSz3/hn0G9e/J8EZtJv3rXAAH9V/36VvYMTgYGZ1zswwB8bG832sXd0IjExkeAX7ZOeyMinBPjfooy9vcbvlEolSUlJREdFsf/vPezcsY1vunybvQPKawpFzn/ymXwavmnfyJEj8fHxQaFQsHLlSsqUKYO9vT0zZ84E4Ny5c0yfPp1bt25hZGRE7969KVWqFH/88QdKpZL9+/dTrFgx9uzZQ+fOnWndujVffPEFKSkp/PHHH2zatInY2Fjq1q3LuHHjMDMzIyQkhMaNGzNz5kxmzZpFbGws3bt3p3fv3lo/XktTQ57GxGukP4l+TiFTw0zLxicm0/jHv1g3qhnn5n+tSl+25ypDFh5Sy9tr9gF+7VWXPVPbqtJOXb9Pq7E7SExKuxoNuBfJmD+Pcz7wEUqlks9c7RjYpgpVHYrSapxPbg8z1yIjIzEzM9NINzc3JyoqKp0Sr5c1T6ds6tV1VGSkKl/BgmYaQa25ufmLfE/T3f6F8+cIunuHEe9oOjcqKrXerzMzMycmOvO2iYqKpGA67Wr28pijUttGoVDg9ft8powfRe9vv1Llq1WnHhOnzdIoHxhwi25fp83qNGvRmmE/TcjW8bxNedFvorLoN5EZ9Jv8IDIykoLpHqM50Vm2z9MMy0Ja+6Rn+rTJKFHS6X+at/IOHzrIkIF9gdR+92337/iud+7Xc2lVPpyZyKmPNsj45ZfUxVDFixdnyJAheHt7c/duamR87949evTowbhx42jZsiVxcXHcvXuXSpUq0bt3b+7evasKRl63detWtm7dyp9//omlpSUjRoxg4sSJ/Prrr6o8p0+fxs/PD39/f7788kuaNWtG6dKltX7M6S3xzc4sjqG+LqtGNKWouTHdft1H8MNo3MsWZ9RXbiQlpzB4QVqgMeF/Nfi6QVlGLj3KmVupCz9Hf+3GtgmtaDpqm2rWY/3Bm2r7OHA+hNBHz5jZqy4Nq9imO+uhbWk3ftJkZ1m0UqlMtx1fX1Od3Xyv89n+F3p6+jRv2SrrymhJet0kO2vGU/Nk75hnTJnA1csXGTpyHKVK23P3TiDLF81j3KghTPt1ntqtABvbkvzx53qeP4/j8sXzrFmxhOTkZMb+7KWxXW1L9/80OwXftN9kby/vXG77DsrclV225A/8fHcybuIUtds0L7l+4saqdZuIiY7h1MnjrFqxHIVCQf9BQ7Kuk8ixjzbIyIyPjw81a9akbdvUq3F9fX0qVaqU7bLffvstJUum3r8eOnQonp6eeHmlDX79+/fH0NCQChUq4OTkxI0bN7QeZDyJiadQQc0ZCwtTQ56kM8Pxqm+blKN+ZRvKf7ea2/dTrz6OXrlH5LN45g9syJLdV7h0J4JyJQsx/ItP6DPnH1b8fe1F6XucvhHO5UXf0K1peeb5XMxwPxsP3WJmr7pUcyqW50GGmZkZkVGaV0ZRUelfqb7K3Nyc+/fD0i0LaVftqVe3kRpfGi+veF+uK3hVQkICe/f48Wm9+hQqlPltLW0paGaW7lV5dHQUpunMcLzKzMycB/fva5Z9ecwvrlSPH/mX/Xt9mTV3CdWq1wRSF4ha2dgybGAvjh0+SN36jVTlDQ0NcXnx1EDVT9wpXKQov0waQ/uOnahQqUruDjQXzMzNiEznijoqgxkO9bLm3LuXdb8xy6jfRKa2oXk6/Sa/MDMzS3fGISoqKt0ZLrWy5ubcT7d9olS/f93mjeuZN+c3+g34njYvFtO+rmDBgpSvkDqeV69ZC319fZYsWsAXX3aiWPHiWR5TnsqHtz9y6v2fi9GCsLAwSpXSjICz48GDB9jY2Kg+29rakpycTEREhCqtcOHCqn8bGRnx7Nmz3Fc2m64FPaZ8Sc0vqXJ2hbgerLlW41UVShfmcfRzVYDx0pmbDwBwsUtdzV+xVOpxnb31QC1fwL1InsQ8x9kuu6v+8/4KzcHRkQD/WxrpgQEB2Ds4Zlk2NCSUuLg4jbL6+vqqe+kODk4kJCQQ/GIdQlo+/xe/11x3cPCf/URFReL5DhZ8vlS6jCN3AjXXF9y9HZDuWgm1svaO3AsL4flz9ba5ezu1bWxeLCYNDEhte5dXHjcEKPfiy+DuncBM9+NcrgIAoSFBmeZ72xwcMug3gdnoNw7Z6zeOji/6TVD6/cY+nX6TX9g7OBKQztqU24H+2NtnXm97B0dCQzXb53aAP/r6+hqzFLt8tvPLlIn8r0s3evTqk+06lqtQkZSUFEJD8372NEuy8PPDZGVlpbp18rqsbi8UK1aM0NBQ1eewsDB0dXXVAot3YdepO1R3Lk7p4mlXDyWLFaRWuRLsOnkn07LhT2KxLGiEvZX6lYe7c2rUHxaRGiTdfxILgFvZYmr5HK3NKWRqRFhE5i/9+qpB6gKxUzeyXhH+ttVv2IhLFy8QEpy2Uj80NIQL5/+jfoNGmZSE+g0bk5SUyN97014ilpSUxF4/X2rVrouBgQEAdep+ir6+Pr471dec+O7cgaNTWWxs7TS27bN9GxYWFmqPKea1OvUacvXyRcJC09rmXlgoly6cp069BlmWTUpK4uC+tHepJCUlcWCfH241aqvaxrJwEQCuXb2kVv7a5dSZryJF1fvU6y78dwYAaxvNNtSmBhn0m/Pn/qN+w8z7TYOX/WaPer/Z81q/qf2y3+xS7ze7XvQb23T6TX5Rv0EjLl+8oPYETFhoCOfPn6NeVudVg0YkJSWy7/Xzas9uataqo2ofgAP7/2biuJ9o2/5zhgz7MUd1/O/MaRQKBba2tjkqlyc+gCBDd8KECRPedSXelX379mFqakqtWrU4deoUkZGRNG3aFGtra7y8vLCxscHe3p5nz55x69YtihUrRkBAAGfOnKF9+/aqgOOvv/7C2dmZChUqkJSUxLJly2jQoAF6enpMnjyZsmXL0rx5c6Kioli5ciX9+/dX3V/esmULFSpUoNwrL97JzJR1p3N1rJfvRNCxnhPt6jhw7/EznGwsmDegAfGJyfT1/ke1KLNkUVNC1nYH4Mjl1KnKuw+i6dqkHJ41yhAVm0AhU0Pa13FgQucaXL4TwaQ1pwAIeRRDqxpl+PxTR5KTlRjo61C3ojVz+9dHR6Fg0Px/iXrxGOzx37/AxFCPQqaGOFib06tFBX7sWI1954Lx2vhfro7xxy813yCaXU5OZfHbvYt9f++haLFi3L1zm8kTxmFgaMiESVPQ108d0MLCQmlQtyagxM29OgBFihTldmAgG9avwcK8EFFRUcz57VcuX7rIlF9mUPTFF6SxiQlxcbGs/HMpRkbGJCQk8OfSxfy9dw/jJ06mdOkyanV6HBHBtCkTadO2A/UaNMz1sb0Un5j5Y6AZsXd04sDe3Rw88DdFihYjOOgOM6dOwMDQkBFjJqGvn/oiq/v3wmjdtC5KUm9hABQuXISgO4H8tXm9arHfonm/cf3KJcZM/IXCRYoCYG1tyx7fHRw+uB9DQyNiY2M5eewwc2d5UdDMnMHDR6Ovr0/ArRtMmTCKhIQEYqKiCLoTyK4dW1mxbCFuNWrzTdeeuTrG3D7V5OhUFj/fXfz99x6KFSvG3du3+flFv5k4aQr6Bmn9pn6dmiiVr/SbokW5fTuQ9evWYGGR2m9mz0rtN1O90vqNiYkJsbGxrFie1m+Wv9pvyqT1mwB/f86ePkVggD8H9u/DwMAAE2MTAgP8KWRpqfGkRna8yWtsHJ3KssfPl/1/76FoseLcvXOHKZPGYWhgyLiJk1Xn1b2wUBrXq4USqOaWdl7duR3IxvVrMbewIDoqCu/ff+XK5Yv8PHW6qn3+O3Oaod/3x8GxLN2/683D8HAehN/nQfh9njx5TJGiqX3s8KGDzJs9i+fPnxMVFYn/zZusXb2CjevX0P7zL/Fokbs1TwZ62rulMWX9mRw/XTLm69yPg9ogazLSYW1tzeLFi5kxYwYTJkzAxMSEPn36ULFiRTw8PNixYwc1atSgWLFi7Nq1S61shw4dCA8Pp3Pnzjx//pw6deowduzYDPaUd2Ljk2g+ZjvTe9Zh6Q+foQAOXgxh2OKjqrdwAqBQoKerg84rMzZBD6KpP2wLYzq5M+F/NShsZkTIoxiW7bmK14azqsWRKSlKWozZzogvqtHdozxjC1cnIuo5J67fZ9KaU2qvL78V+pQ+LStRwtIEXR0dAu9FMnX9GWZtOZdHLaLO2MSERctWMNNrGmNGjUCpVFK9Zi2G//gTJiZp725QKpUkJyeT8trIO3HyNObO+Y153r8T/eL1x/MWLqFc+Qpq+QYMGoKJiQlrV69UvR56+q+/pztbsmuXD0lJSXi2aavxu7xkbGzCb/OXMfc3L6ZMGIVSqaSaW00G/PCj2hstX7aN8rV3WowcO5nFC+awdKE3MTHRODg5M332Qsq6lFflKWBqyoJla1i+aD7rVi3nccRDLAsXpdanDej2XT/VfgpZFsbcvBCrly/m8eNHGBkaYWVjS79Bw2jZJv178NpkYmLC4mUrmOE1jdEjR7x4HX0tho/8CZMCmv3m9UWLkyZPw3v2b8ydk9Zv5v+h2W8GDn6t35Qpw4xff6fBa7Mle/fsZuH8uarPZ06f4szp1IuAJctXYlm9xttugkwZm5iwcMmfzJo+jXE/pbaPe41aDBsxKt3z6vW+M37SVOZ7/8aCubOJjo7CqawL3gsWqzgs8XoAACAASURBVLXP6VMnSEhI4Mb1q3Tv0kmtvJW1NTv9DgBga1eSFKWSBXNn8/hxBAULmmFXqhQTp3jh0bylFlvhDeTDmYmckteKv2dy+1rxj8GbvFb8Y5Db14p/DHL7WvGPwZu8VvxjoNXXirddlOMycdt6aaEmuSczGUIIIUR+9AHMZEiQIYQQQuRHH8AjrBJkCCGEEPnQh/AnLyTIEEIIIfIhCTKEEEIIoR3vf4whQYYQQgiRH8lMhhBCCCG0QoIMIYQQQmiFBBlCCCGE0AoJMoQQQgihHe9/jCF/hVUIIYQQ2iEzGUIIIUQ+JLdLhBBCCKEVEmQIIYQQQiskyBBCCCGEVkiQIYQQQgjteP9jDHm6RAghhMiPFApFjn9y4unTp/Tv3x9XV1caNGjAtm3bMsw7Z84c6tWrR7Vq1ejYsSPnz5/P1j5kJkMIIYTIh7R9u2TSpEno6+tz5MgRrl27Rq9evShXrhzOzs5q+Xx9fdm4cSOrV6+mZMmSrFy5kgEDBnD48OEs6ygzGUIIIUQ+lJuZjKioKEJCQjR+oqKi1LYdGxvL3r17GTx4MAUKFMDNzY3PPvuMHTt2aNQjJCSEatWqUbp0aXR0dOjQoQMPHz7kyZMnWR6DzGQIIYQQ+VEuJjJWrFjB3LlzNdIHDBjAwIEDVZ/v3LmDrq4uZcqUUaW5uLhw8uRJjbItW7bE19eXgIAASpUqxcaNG6lYsSKWlpZZ1keCjPfMoy1933UV8i3LtnPedRXytbvrpe+InNP5ABYfvq9yc7uka9eutGvXTiPdzMxM7XNsbCympqZqaaampsTGxmqULVKkCNWqVaNly5bo6OhgYWHBsmXLslUfCTKEEEKIfCg3QYaZmZlGQJEeExMTYmJi1NJiYmIwMTHRyDt37lwuXbrEwYMHKVKkCDt37qRnz574+flpBCqvkzUZQgghRD6kzadLSpcuTXJyMnfu3FGlXb9+HUdHR428N2/epEWLFpQoUQI9PT3atm1LYmIi/v7+We5HggwhhBAiH9JmkGFiYkKTJk2YM2cOsbGxnD17lv379+Pp6amRt3Llyvj5+fHw4UNSUlLw8fHh+fPnlCpVKsv9yO0SIYQQIj/S8nqY8ePHM3r0aGrXro25uTljx47FxcWFsLAwWrZsya5du7C2tua7777j8ePHtGvXjtjYWOzs7Jg9ezaFChXK+hCUSqVSu4ch3qZnCfLflZEi7bzfdRXyNVn4mTEzY/13XYV8KyVFxpzMmBhoLxKw6ftXjsuELtBc9PkuyUyGEEIIkQ99CH+7RNZkCCGEEEIrZCZDCCGEyIc+hJkMCTKEEEKI/Oj9jzEkyBBCCCHyI5nJEEIIIYRWSJAhhBBCCK2QIEMIIYQQWiFBhhBCCCG04/2PMSTIEEIIIfIjmckQQgghhFZIkCGEEEIIrfgAYgwJMoQQQoj8SGYyhBBCCKEVH0CMIUGGEEIIkR/JTIZ4r9y/f49fp0/j5PFjKJVKqteszbAfR2FlZZ1l2fj4eObPnc3unT5ER0dR1tmFQUOGUc3NXS3f6hXLOX36JNeuXOHRo4f06tufPv0GamwvOTmZdWtWsf2vLYSGhmBawJRKlavQu99Ayjo7v7Vjzi7bIqZM/64ejVztUCgU/HM+iOGLDhH8MCbLsnZFTRn3v1rUr2xLYTMjQiNi2HL4FjM2niE2PkmVz9hQj4ldatGhrhOWZkb4hz3l101nWX/whsY2jQx0Gfq5G181cMauWEGexsRz9lY4X03ZRWJSyls99twKv38P71nTOXPyOEqUuFWvyaChIylewirLsvHx8SxZ6M1e353ExETjVNaFPgOHUPUTN1UeX59tTJs4JsNtbPM7SOEiRd7KsbyJ+/fuMcNrGieOH0WpVFKjVm1G/PgTVtbZO6/mef/OLp/U88rZpRzf/6B5XqWkpLB86WI2b9zAo0cPKV26DL379uezps3U8o0dPYpLF8/zIDyclBQldnZ2tPv8C778qhO6urpv9biz6/79e8x8ZdypkYtxx/eVcWdwOuPOqhXLOXP6JFdfjDu9Mxh3Zv/2K0cP/8u9+/dISkykRAkrmrf0pHPXbhgbG7+1Y35bPoAYQ4KMj0VcXBy9e3yLgYEBEyf/gkKhYL737/Tu3pUNW7ZjbGKSaflJ40Zz+PC/fP/DcGxs7di4fi0D+vTkz9XrcXYpp8q3dcsmTE1NadCoMZs3rs9we/PnzmbFsiV069EL9xo1efrkCUsWLaB3jy6s37yd4iVKvK1Dz5KxoR67p7YnPjGZ72b9jRIY37kmftM64N5/jVqg8DoTQz12TWmHvq4OE1cfJ/hhDG5OxRjzTU0crS3o7OWnyrt+dEtquJRg4qrj3Ax5SpvaDiwf3gyFQsG6f66r8unp6rB9YhtKlzBjxsYzXAt6TFFzYxq5lkRXR0GiNhsjm54/j+P7vj3Q1zfgp4lTUKBgyQJvBvXuxp/rt2JsnHl/8vp5HMePHKLv4KFY29jy16Z1DB3Ym4XL1uDk7AJArbr1WLh8jVo5pVLJyCEDsLaxzRcBRlxcHN9174q+gQE/T/VCoYC5c2bTs3sXNm3dgUkW59WEsT9x+NC/DBk6Als7O9avW0PfXj1YuWYDLuXSzqt53rNZsXwpAwcPoVz5Cvjt9mXYD4Pxnv8Hn9arr8oXH/+crzv9D1u7kigUCo4dPcL0aVMIDrrLj6MyDti0JS4ujl49UsedSZN/gRfjTq/uXdmYjXFn4otxZ8gr407/Pj1Z8dq489eWTRTIxrjz7FkMrdu2p1TpMhgYGHDh/DmWLl7I1SuX+d17/ts67LdGR+f9jzIkyPhI/LVlE6EhwWz12U3JkqUAcCrrTNtWzdiyaQP/69otw7I3b1xnt+9Oxk+aQpt2HQCo5ubOF+1asWDeHH73XqDKu3nbTnR0dEhKSsr0ZPfZ/hdNmzWn/6DvVWlOZZ3p0KYFhw8d5POOX73pIWdb92YVKFPCjMq9VxF4LxKAS7cfcXlxF3o2r8ScbecyLFurvDVONoVoNWYb+88FAXDoYgiFChrxfftPMDbUIy4+idrlrWharRTf/fY3q/ddA2D/uSBsipgypVttNvx7g5QUJQDft3elqmMxqvVdTcijtJmUbccCtNUEOebz12bCQkNYs2UntnYlAXBwKkun9i3ZvmUTX/2va4Zl/W9e52+/XYwc9zMtW7cDoOonbnTp2JalC+fyy29zAShUyJJChSzVyl44d5bIyKd0791fS0eWM1s3byQkJJjtO/0oWSrtvGrdohmbN26gy7cZn1c3rl/Hd9dOJk6eSttXzqv2bVoyf+5s5sxbCEBERAQrli+le89edO3WA4DqNWoSHHSX2b/NVAsyps/8TW0ftevU5eGDB2zbuuWdBBkvx52/Xhl3ypZ1pk2rZmzetIHOmYw7N16MOxNeG3c+b9eK+fPmMDsX485PY8arfa5RsxbPn8exfOlinjx5QqFChd7kcN+6D2EmQ+ddV0DkjX8PHqBS5SqqEx3AxtaWKlVdOfjP/szL/nMAPT19mnq0UKXp6enR1KMFx48eISEhQZWuo5O9LpWUmEgBU1O1tIJmBYHUq9W81LKGPadu3FcFGAB3w6M4fvUerWraZ1rWQC/1eKNjE9TSI5/Fo6NQqF7YV90ldWZm75k7avn+PnsXq8Km1HBOm7np1aIyW4/cUgsw8psjhw5SvmJlVYABYG1jS8Uqrhw59E+WZfX09Gjc1EOVpqenR+NmHpw6cVStP71u987t6Ovr07hp8zc/iLfg4D8HqFy5iirAALC1taOq6ydZnlcH/9mPnp4+zV47rzyat+TYK+fVsaOHSUxMpKVna7XyLT1bc+vmTUJCgjPdj7mFBbp67+Z6UhvjTrM3GHfSY2Fhodq2ePskyHhHQkJCcHZ2Jikp46n4tynQ3x8HRyeNdAdHJwIDM79CDgjwx8bWRuOepYODE4mJiQQH3c1xfb74shO+O304eGA/MTExhAQHM23yJIoXL0FTj7z9AilXypIrdyM00q8GReBS0jKdEmkOnA/mVugTJnerg4udJQWM9Klf2ZZ+rauyePcl1a2W5BezFAmvraeIT0wGoHypwkDq+g67YgW5fT+SeQMbEb6pD0/+6ofvlHZUtn/3twdeuhPoj72DZn8qY+/AnSz60+0Af6ysbTEyUu9PZewdSUxMJDQ4KN1y8c+fc3DfXmrXrY/5iy+Gdy3A3x8Hp7Ia6Q4OjgQG+GdeNqPzyjG1HYJenFcB/v4YGBiofVGn7iO1/QMD1NtbqVSSlJREVFQU+/buwWf7X3Tu8m1OD+2tCPD3xzEfjTsvJSUlERv7jBPHj7Fq5Z+0adeBggUL5np72qJQKHL8k998VKFbYmIi+vr677oa70RkZCRmZuYa6WZm5kRHRWVaNiryKQXTKWtubq7adk71HTAIfQMDhg0ZSEpK6hdvqdKlWbR8JebmefsFYmlqxNOYeI30J9HPKWRqmGnZ+MRkGg/fzLqfWnBu4f9U6cv8LjNkwUHV55shTwCo7lyCvWfTBscaL2Y4ChU0AsDKMnV2Z+jnbpy9GU4Xr90Y6usy5pua7JnWgeoD1mRrMaq2RUVGUtDMTCPdzMycmOjM+1N0VMZlAaKi0u9Phw8e4NmzGDxatclFjbUj9bzSPBZzc3OisjivMjonX/b/qBfnVVRkJAULmml8gaSdf0/V0g/9e5BB/fsAqV9S3Xv2onffd3N7KTIyMv2xI5vjTrpj1huMOwD+t27yRfu0WaFWrdswdvykXG1L2/JhzJBjeRpkXLp0ibFjx3L37l0aN25MYmIijo6ODBw4kE2bNrF48WKePn2Kq6srEydOpESJEowfPx4DAwNGjx6t2s7333+Po6MjAwYMIDw8nClTpnD69GmMjY3p0qUL3377LQDe3t7cunULIyMj9u/fz5AhQ3jy5AkBAQGYmJjg5+eHlZUV06dPp0KFCgA0atSIb775hu3btxMcHEzz5s0ZPnw4o0eP5sSJE1SoUIHZs2djaZl6hXv+/Hl++eUX/P39sbKy4qeffqJWrVoAdO7cGXd3d06dOsXVq1dxdXXl119/xcLCgv/9L/ULyd09dZX0woULqVGjhlbbP70OqyTrWxNKZfp/p+dNbmts2rCOpYsW0KNXH9yr1+DpkycsX7qY/r16sHTFaooWK57rbedGeoeSnasCQ31dVo1sTlELE7rN3EPwg2jcnYsz6usaJCWnMHj+QQD2/RfEtaDH/Nq7Pj1n7eVGyBPa1HagY33nF/tPrcDLWd/Y+EQ6TPIh7sVMyH+3HnB5SVd6t6rMmOXH3vyA34b0+lM2+oRSqUy3bbMqu3vXdiwKWVKzzqfZrmJeSPdYslMwm+2QYXtlsJdPqrmxdsNmYmJiOHniOCuWL0OhUDBw8JDs1Oqte5NxJ2e/yB67kqVYvX4TcbFxXLhwjuVLFpGclMxUr5lvtF1tyI8zEzmVZ7dLEhISGDBgAO3atePUqVM0btyY/ftT78kdP36cmTNnMmvWLA4fPoy1tTXff5+6INDT05Pdu3eTnJw6rfzs2TMOHjyIp6cnKSkp9O3bF0dHR/79919WrFjB6tWr+eeftHvC+/fvp1GjRpw+fZovvvhClebh4cHp06epV68eU6ZMUavr7t27Wbp0KXv27OHff/+lR48e9OvXjxMnTqCvr8/y5csBCA8Pp1evXvTu3ZuTJ08yatQoBg8ezMOHD1Xb8vHxYfLkyRw9epS4uDj+/PNPAFavXg3A6dOnOXfunNYDDDMzs3Qj/+ioqHSvKtXKmpune3X58krt5RVVdkVGPuXX6dPo3LU7ffsPws29Bp819WD+oqU8efKYFcuX5Wh7b+pJTDyFCmrOWFiYGvIknRmOV33btAL1K9vSdvwO1v9zg6NXwvh96zlGLjlMr5aVqVQm9RZHcoqSTlN9eRafyMFfO3JvQ28mdKnFuBWpAcO9x88AiIh6DsCJq/dUAQZAyKMYbgQ/pop9sbdyzG+qoJkZ0ZGaV6LR0VGYFsxGf0qvL76YAUnv6vXRo4ecPXWCJh4t89W9czPz9M+rqAxmONTLmmvMQkDaTM7LK/aX59/rwUdU5MvzT33mr2DBglSoWIkaNWsx6Psf6NmrN8uWLCI8PDz7B/aWmJmZpft/HZWNccf8LY87LxkaGlKhQiXc3KvTo2dvRowczW7fnVy8cD5X29OmD+F2SZ4FGefPnyclJYUuXbqgr69P8+bNKV++PJD6Rdy+fXsqVqyIoaEhw4YN49KlS4SEhFCtWjUMDAw4efIkAH///Tdly5alVKlSXL58mYiICAYOHIiBgQF2dnZ89dVX+Pr6qvZbuXJlPDw80NHRwdAw9YvEzc2NevXqoaurS5s2bbh27ZpaXbt06ULRokUpVqwY7u7ulC9fnooVK2JgYMBnn32myr9jxw4+/fRTGjZsiK6uLrVr16Zq1aocPHhQta327dtTunRpjI2N8fDw4Pr167wL9o7p3yMODPDH3t4h07IODo6EhoQSFxenXjbQH319fexeu1eclbt37pCQkECFipXU0s3NLbC1K8nt23n7FMW1oAjKlyyskV7OzpLrQY8zLVuhdGEeRz/n9n31wfDMzdQB3cUubbX69eDH1By4Duduy/mk72qcui7n/ovg4vjVewDcvh9F7PPEDGdWUvJ4UWxGytg7cjtQsz/dCQygdBb9qbS9I/fCQnj+XL0/3QkMQF9fH5tXFpO+tNfXh+TkZJrno1slkHpuBPjf0kgPDAzA3sExy7LpnlcBqe3wcg2Go6MTCQkJBAcFvZYvtf3tHTJv7/IVKpKSkkJoFgtEtcHB0ZGAXI479m953MlI+QoVATTaNz9QKHL+k9/kWZDx8OFDihcvrhZplXjxLoQHDx5ga2urSi9QoAAWFhaEh4ejUCho1aoVPj4+AOzcuRNPT08gdfHkw4cPcXd3x83NDTc3N+bNm0dERNoiPut0XohTuHDaF4qxsTGxsbFqvy/yyvP3RkZGavmNjIx49iz1iyE0NJS9e/eq9u3m5sapU6fUZjJe3ZaxsbGqbF6r36ARly5eICQ4baAJCw3hwvlz1G/YKPOyDRuRlJTIvr1p73xISkpir99uataug4GBQY7q8rJNLl+6qJYeGfmU4KC7FMvjWyW7Tt6muksJSpdIu7IqWawgtcpbsetkYKZlw5/EYlnQCHsr9asqd+fUYwiL0Pz/DnoQzbUXwUufVpX5++xdVZCSlJyC35k71K5gjYlh2hW7XVFTytoW4uzNvL8aTU+deg25evkiYa98cd0LC+XShfPUrdcg07J16zUkKSmJf/btVaUlJSVx4G8/3GvWTrc/7dnlg4NTWdU7NPKLBg01z6vQ0BDOn/svy/OqQcPGJCUl8vce9fNqj58vtWrXVbVD7bqfoq+vj+8uH7Xyu3buwNGpLLa2dpnu5+yZUygUCmztMs+nDW8y7jR4Me78nc64UysX405Gzp45DfBO2icrH8JMRp7NOxYtWpTw8HC1+4v379/HycmJYsWKERoaqsobGxvL06dPKV48daD29PTk66+/ZvDgwZw6dQovLy8ArKyssLW1Ze/evZo7fEGbjW5lZUWbNm2YPHlyjsvmdWdo3+ELNqxbww+D+tFv4PepL+OaO5vixUvQ4YsvVfnCwkJp06Ip3/XuR68Xi8WcXcrR1KMFM72mkZSUhLWNLZs3riMsNIQpv8xQ28/VK5cICw1VvfPhdkCAKjip82l9jI2Nsbax5dP6DVj551J0dHT4xM2dyKdPWbF8CQkJiXzxZd69IwNSF2n2aVWZTWNbMXHVCZRKJeP+V5OQRzEs2X1Zla9k0YJcWdqVqetOMW3dKQBW7bvKoHaubJvYGq8Npwl+GEM1x2KM/Lo6Z2+Fc+xqmKr8sC/cCH4QRdjjZ9gVLUifVpWxLVqQRsM3qdXn59UnOfxbR/6a0JrZf53DyECXn76uwdOYeBbuvJA3jZIFz3Yd2LpxLaOGDqJn34EoFAqWLPSmWIkStG7fUZXv/r0wvmrbnK49+9Dtu74AODm70KiJB3N+9SIpKREra1u2bd7AvbBQxk720tjXjetXCQy4Rf/vh+fZ8WVX+887sn7tGgYP7MeAQYNRoGCe92yKlyjBF6+dV608mtCrTz/69BsAgEu5cjRr3oLpXlNJSkrCxtaWjevXERoSwrRX1gcULlyY/3X5lqWL/8DEpADlypdnj58vp06eUHuB1KF/D7L9r63Ub9CQElZWxD57xpEjh9iyaSOfd/wyz4N3SBt3hqQz7nz+Wvu0fjHu9M5g3LGxsWXTxnWEpjPuXHkx7ihfjDuBAQGq4KTui3Hn5o0b/ParF02aemBja0dCQgL/nT3DujUrqVO3HlWquuZRq2RfPowZcizPgoyqVauiUChYvXo1X3/9NQcOHODq1at8+umneHp6MmTIEDw9PXFwcGDWrFlUqlRJNbvh5OSEjY0No0aNonr16qqZhcqVK2NqasqiRYtUt2Fu375NbGwslStX1voxtW7dms8//5xDhw5Rp04dkpOTuXDhAjY2NunOoLzK0tISHR0dgoKCsLfP/F0Mb4OxiQl/LP2TX6dPY+xPI1JfK16jFsN+HIWJSYG0jEolycnJpCjVH7Wc8PNU5s35jfnes1Wv9527cDHlyldQy7dh7Rp8dmxTff57r5/qZN/ptw9jm9T/019m/MbqFcvx272LVSuXY1rAFJdy5Rm1cjzlK6jfRtG22Pgkmv+0lenf1WPp0KYogIMXghm26BDPnr/yfk1F6ts4dV4584MeRFP/h42M+aYGEzrXorCZMSGPolnmdxmvDafVbnsUMNJjQpdaWBU25WlMPH+fvUunqb4a78O4HvyY5j/9xeRudVj1oweJySkcuhhCx8k7efBUfer4XTE2NmH2wmV4/+rF5PGjUCqVVHOvyaChP6q95VL5oj8pU9T700/jJ7No/hyWLPAmJjoaBydnZs5ZiLNLeY19+e3cjq6uHk2bt9T6ceWUiYkJi5etYIbXNEaPHPHitdm1GD7yJ0wKpJ1XqnZ47XbXpMnT8J79G3Pn/K46r+b/sUTjvBo4eAgmJiasXb0y9bXiZcow49ffafDKbICdnR0pyhTmev/O44gICpqZUbJkKSZP86J5i1babYgMvBx3Zr427gzPYNxRvjbuTPx5KnNfG3fm5XDc2fVi3ClcuDAWFoVYuuQPIh49wsjICBtbO4YMHUG7Dl9orxHeQH6cmcgphTIP33x08eJFxowZQ3BwMA0bNuT58+dUqVKF3r17s27dOpYtW8bTp0+pWrUqEydOVPuiXrJkCTNmzMDLy4u2bduq0sPDw/Hy8uLkyZMkJCRQpkwZBg0aRN26dfH29ubu3bvMnJl2VfB62t27d2natCk3bqT+/YhGjRoxefJkateuDcDIkSMpXrw4Q4akrszeunUrmzZtYt26dQBcuHCBmTNncuPGDXR0dKhUqRLjx4/H1taWzp0707p1a9WC002bNrFjxw5WrVoFwOzZs1m3bh1JSUnMnz+f6tWrZ9mGzxLyxz35/KhIO+93XYV87e76vu+6CvmWmfHH+Wh7dryclRTpMzHQXiDgNjnzF9ul58yYhlqoSe7laZDxug4dOtC5c2e1oEFkToKMjEmQkTkJMjImQUbGJMjInDaDDPcpB3Nc5vToBm+9Hm8iT9/4efLkScLDw0lKSmLr1q0EBgZSr169vKyCEEII8V74EJ4uydMHzu/evcvQoUN59uwZdnZ2zJkzR/VSKyGEEEKk+RDWZORpkNGxY0c6duyYdUYhhBDiI/cBxBgf198uEUIIId4XMpMhhBBCCK34AGIMCTKEEEKI/EhmMoQQQgihFR9AjJG3j7AKIYQQ4uMhMxlCCCFEPiS3S4QQQgihFRJkCCGEEEIrPoAYI3tBxsWLFzEwMMDFxQWA/fv3s337dsqUKUO/fv0wNDTUaiWFEEKIj82HMJORrYWf48ePJzg4GICgoCB++OEHLCwsOHDgANOmTdNqBYUQQoiP0Yfwt0uyFWTcuXNHNYvh6+tLnTp1mDRpElOmTGHfvn1araAQQgjxMVIoFDn+yW+yFWTo6+uTkJAAwNGjR2nQoAEAhQoVIiYmRmuVE0IIIT5WH8JMRrbWZNSoUQMvLy/c3d25ePEis2bNAiAwMBArKyutVlAIIYT4GOnkx6ghh7I1kzFx4kRKlCjBqVOnmDlzJkWLFgXgwoULtGzZUqsVFEIIIT5G2p7JePr0Kf3798fV1ZUGDRqwbdu2DPMGBQXRq1cvXF1dqVGjBtOnT8/WPrI1k2FpacmkSZM00gcNGpStnQghhBAiZ7S9xmLSpEno6+tz5MgRrl27Rq9evShXrhzOzs5q+RISEvj222/p3Lkzv//+O7q6uty+fTtb+8gwyPjvv/+yXdFPPvkk23mFEEIIkTWdXMQYUVFRREVFaaSbmZlhZmam+hwbG8vevXvx8fGhQIECuLm58dlnn7Fjxw6GDx+uVnbr1q2UKFGCbt26qdJePgySlQyDjE6dOmVrAwqFgmvXrmUrrxBCCCGyJzczGStWrGDu3Lka6QMGDGDgwIGqz3fu3EFXV5cyZcqo0lxcXDh58qRG2QsXLmBtbU2PHj24fPkyZcuWZcyYMRozHunJMMi4cuVKloVF3nsUnfCuq5BvBW/o966rkK+5jtz1rquQb938rc27rkK+FZeY/K6rkK+ZGGjvxdm5uVvStWtX2rVrp5H+6iwGpM5kmJqaqqWZmpoSGxurUTY8PJyTJ08yf/58atWqxcqVK+nXrx+7d+/GwMAg0/pk2Dq6urqZFhRCCCGE9ijIeZTx+m2RjJiYmGi8giImJgYTExONvIaGhlSrVo369esD0KNHDxYsWEBgYGCWt02y9XSJUqlk1apVeHp64urqqnr75+LFi/H19c3OJoQQQgiRAzqKnP9kV+nSpUlOTubOnTuqg4SGxAAAIABJREFUtOvXr+Po6KiRNzu3RTI8huxkWrhwIatWraJHjx6kpKSo0q2trVm5cmWudy6EEEKI9GnzjZ8mJiY0adKEOXPmEBsby9mzZ9m/fz+enp4aeVu3bs2FCxc4duwYycnJrFixgkKFCmFvb5/lfrIVZGzZsoXJkyfTtm1bdHTSipQrV46AgIBsH5QQQggh8ofx48cTHx9P7dq1+eGHHxg7diwuLi6EhYXh6upKWFgYAPb29sycOZPx48fj7u7Ovn37mD9/fpbrMSCb78l4+PBhum/2TExMJDlZFgUJIYQQb5u2X/hpYWHBvHnzNNKtra05d+6cWlqTJk1o0qRJjveRrZmMypUr888//2ikb9iwAVdX1xzvVAghhBCZ01EocvyT32RrJmPEiBH07NmTy5cvk5SUxOLFi/H39+fWrVusXr1a23UUQgghPjr5MGbIsWzNZFSqVIndu3dja2tL/fr1CQkJoUqVKmzbtu2NVp0KIYQQIn0fwp96z/ZbRCwtLeVvlQghhBB5JB/GDDmW7SDj/v37bNiwQfVHUcqUKUPHjh3lT70LIYQQWpAf11jkVLZul+zbt48mTZpw7NgxihQpQpEiRTh+/DhNmzZl37592q6jEEII8dFR5OInv8nWTMYvv/xCnz596N+/v1r6ggULmDZtGp999plWKieEEEJ8rPLjGoucytZMRkREBC1bttRIb968OREREW+9UkIIIcTHTpuvFc8r2QoyGjVqlO5tkf3799OoUaO3XikhhBDiY/dBP12ycOFC1b9tbW1ZsGABhw4donLlyigUCi5evMiVK1f45ptv8qSiQgghxMckH8YMOZZhkHHo0CG1z87OziQlJfHff/+p0sqWLcuZM2e0VzshhBDiI5UfZyZyKsMgY+3atXlZDyGEEEK8Ij+uscipbL8nQwghhBB554OeyXjdsWPH8PPz4/79+yQlJan9btmyZW+9YkIIIcTH7P0PMbL5dMmaNWsYMGAACoWC48ePY21tTUpKChcuXKBcuXLarqMQQgjx0flo/grrypUrmTJlCs2bN2fHjh306tWLkiVLMm/ePMLDw7VdR/GWPAi/z8LZ0/nv9AlQKnF1r0nfwSMoViLrV8MvWzibm9eucuvGVaKjIhk2+meatmyjkW+v73ZOHDnIzWtXeRB+jyYtWjN8zGSNfCuXzGf1soUa6bU+bchEr9m5O8A3FH7/HnNmeXH6xHGUKHGrXovBQ3+khJV1lmXj4+NZvMCbvb4+RMdE41TWhX6DfqDqJ25q+Tq0asL/2bvvsCiOPoDj36MpSBGxUQUBwS6KAvbesbc3scaugD2xxNg1okYFe40tir3XKHbs3VhBRYqIioAC0u794/TwPLqUU+fjw5Pc3szuzDBz99vZ2eVFaIhS/llzvajboJH8dVxsLMuXLOTEsSNERb7FzLwU3Xv3o1nL1l9f0WwwLlyQSR0rUse+GBLg7INwJu+4Q0hEbLr5RrS0Y2RL+1Tfi0tIwnbEfoVtJQ0KMrq1PQ3Kl8BAW5OwyDj2Xgtm9t57ADjbGrFtWO00j9dm7mmuP43IWuW+0ovQUOZ6zuKC3zmkUilOzjUZM3Y8xpnsN4u9F3Bw/z6io6Owsy/LsBGjqeZYXSFdcnIya1evZPs2H16/CsfS0ooBg4fSuEkzhXR9e/fg6pVLSscZ/ds4uvfo/VX1zK6wF6EsnDebyxf9kEqlVK/hwrDRWRhXS7w5cmgf0dEp48qhmuK46tAqjXE1z4t6n42rof17c/3qZaV0w0b9Rtefe2ajdkJGMhVkvHjxgkqVKgGgo6NDdHQ0AK6urnTs2JGpU6fmXgmFHBEXF8uv7v3Q1NRkzO/TkUgk/L3CmzFufVm2YTva2jrp5t+zfTPWtnY41arLv4f2pZnu+JEDRL6NoGoNZ06fOJZhueYvW4eaWsqEmp6+QeYrlYPiYmPxGPQLmlpa/D5lJhKJhBVLvHAf+AvrfXZm2D6zpk7E7+xphg4bhYmZOTu2bmaE2wCWr91EGTvF2T4nl1r8MlDx6bkWpSwVXo8fM4w7t27Sf4gHFqUsOeX7L1Mn/oZUmkzzVm1ypM6ZVVBTHR+PWsQnJjNywzWkUhjjWpatHrVoMsuX2PikNPNuPv+Mk/+9VNimo6XOhqEuHLv9QmG7WRFtdo2sQ+DrGCZtu82r6A+YGelgWbSQPM2d55G0mat45xvAnJ+rUFhHi5vP8jbAiI2NpX/fXmhpaTF1xmwkEljsvZD+fXqybedetHXS7zeT/xjPmdOnGDHqV8zMzPHZvIkhA/uybpMP9vYp/Wax90LW/70aN48RlCtfnsOHDjJm5DC8Fi+nTt16CvssU8aO3ycpfiabmJrmXKWzIC42FveByuPKbeAvbMjkuDp/5jRDh4/C1DRlXK34O/Vx1ffLcWVpqbRPG9sy/DphssI2Y5OMA578oIITE1mWqSDD2NiY8PBwTE1NKVWqFGfOnKF8+fJcv34dTU3N3C6jSvH29ubZs2fMnTs3v4uSJYf27OBFSBCrt+zF1MwCACsbW/p0deXA7u10+l/6Ufyuo+dRU1MjOCgw3SBj1vxl8qDhyoVzGZbLvlxF1DXyf/3x3l3bCQkOYvPO/ZiZlwLA2rYM3dq3ZM+OrXTr3jvNvI8e3ufY4QOMnzSdVm3aA1ClqiPdu7Rl1bJFeM5frJDeoLAhFSpWTnN/N69f5aLfOYX9ObnUIjwsjCVef9GkeSvU1dW/ssaZ91OtUlgULUT9qcd5+uo9APdCojj9RyO617Zk5Qn/NPO+eBvHi7dxCts6VDdDU12N7RefK2yf1a0yL97G0XXhORKTpbKNjxWfKPwuLlFppsLUUBvbEnqsOPGYT9nyys7tWwkOes7u/YexsJD1mzJl7GjTqhnbt/nQo1efNPM+uH+fQwf2M3naTNq17whANcfqdGzXiqWLFrJwkWym783r16z/ezV9+g6gV5++AFSv4czzwGd4zZ+rFGToFCpEpcpVcqO6Wbbn47jasnM/Zh/bx8a2DF3btWT3jq38L4NxdfSQbFy1bvtxXFVzpHvntqxaugjPBamMq0ppj6tPdAoVylQ6VfA9LPzM1JqMtm3bcuPGDQD69+/P4sWLqV27NuPGjaNHjx65WsBv2dixY5k/f35+FwMAv7MnsS9fSR5gABibmFG+YhX8zvhmmP/z2YacSKdqzp72pXzFSvIAA8DE1IyKlR04cyr99jl7yhcNDQ0aNWku36ahoUHjpi245HeO+Pj4LJXl7u1bALjUqqOw3almLV6/Cufu7ZtZ2t/XalKxJNeevJEHGADPX8dwJeANTSuWzPL+OjlZ8DIqjlP3UmY4ShXVoX65Eqw9FZASYGRSxxrmqKlJlIKWvHDq5AkqVqosDzAATM3MqeJQlZO+xzPIexwNDU2aNW8p36ahoUGz5q04f+6svN+cP3eGhIQEWrkqzmC1bN2GR48eEhyU9/XOrLOnPo4ri1TG1cnMjavGTZXH1cVsjKtvkUSS9R9Vk6lvhMGDB9O7d28AGjRowIEDBxg/fjw+Pj4MHjw4N8sn5JBnT/yxLG2jtL2UlTWBTwPyoUQyP7dvSvPaVejevhmrFs/nw4e4jDPlgicBjyltbau03aq0NU8D0j5Tl+X1x9jUjILa2l/ktSEhIYGg54EK28+dPknDmtWo71yF/r3+x+kvvozU1GXDUuOLWUItTS0AAvwfZ65SOaSMsR4PQqOVtj8Mjca2pF6W9lWycEFqlinK7stBJH0WTDiWNgJk6zQ2ubnweH5rbnu2YH6PqhQulP5saUcnc24Fvk21jLnN//FjbGzLKG0vbW2T4e/J//FjTM1M0f6i31jbyPpNYOAzWTr/x2hpaSkEMrJ0th/fV+yf9+/fo7ZzNRyrlKdze1d27diW5XrllDTHlXXG4yrA3x+T1MaVdRrj6sxJGtSsRj2nKvTv+T9OpRHkPbx/nyZ1nahTozI9urRn3+4dWaxV3vlhFn5+ycLCAgsLC54+fUqnTp3Yvn17TpdLJaxcuZINGzYQHR2NkZERkydPVkozYsQILl26RFxcHOXKlWPKlCmULl0aHx8f9u3bh0QiYf369Tg4OLBmzRrCwsKYMWMGly9fRltbm549e8oDuNwUHRWJnp6+0nY9fQOio6Ny/fhfMjGzoO/g4ViXsUcikXD10nl2+mzg0cN7zF64Is/LExWZevvoG2TcPunlBYiKipRvq1W3PmXLVcDY1IyI16/YsXUz40Z78Me0P2nW0hUAi1JWgGxG4/PZjDsfZzCiIlP2lxcK62gRGZOgtP1tTDwGOlm7XNqxhjnqahK2fTHrUMKgIABzf3Zg5+UgFh99hGWxQoxtU44yxnq0nnMKaSoTHFWtDCldXJc/tt3OUjlySmRkJPr6yr97AwMDoqLS7zeyvMprkAwMCgMpv+fIj/3ry6lzg0/9K/KtfFs1R0datnalVClLoqOj2L93D1Mm/c6rV+H0Hzgka5XLAVGRkeil0j76mfjciUrjM0s+rj4bB7Xr1qds+QoYm5jx5s0rdvhsZtwo2bhq3spVnq5K1Wo0bdEKi1KWREdHc3j/HmZN/YNXr8Lp029QdquZa1QwZsiyr7oYHhcXx927d3OqLCrlyZMnbNy4kW3btlGiRAmCg4NJTEzk+vXrCunq1KnDtGnT0NTUZPbs2YwZM4YdO3bQtWtXrl+/TokSJRgxYgQgWyE+ePBg6tevz9y5cwkLC6NPnz6UKlWKBg0a5H6lUuuxqX1y54HGzRXvkqhWw4VixUqw9OPdL1WrO+d9oVJpn8w0jxRpqtdOpalkHvnrBIXXdRs0ZkDv/7Fs0QJ5kFHDuSaWVqVZMGcmenqzsLC04tSJfzl25CAAavnwGEApynXJzvXijjXMuf38LfdDFL9gPlXpwuNX/L5Vdrno/MNXRMcmsOSX6tQrW1xpASlAJydz4hOT2X0lKMtlySmSVJ5mkKl+I81cv8lsOoAhbsMUXjdo2JgRHkNZtWIZP/fohY5OIaU8uS31smcio1SaxphMZVz9pjiu6jVoTP9esnH1eZDRf7C7Qrq69RsydpQH61avoOtPPfKlfdLzw6zJ+BGpq6vz4cMHHj16RHx8vHzR65c6dOiArq4uBQoUwM3NjTt37hATE5PqPu/cucPr169xd3dHS0sLc3NzunXrxsGDB3O7Oujq6RMdpXwGHB0dlerZQn6o36QFAA/u3cnzY+vpG6TePmmcTX1OX99AYbYiJW+U/P20qKur07BxM16GveBVeDggu+483XM+BbW1GdjnZ1o0qMmKJQsZ5DYcAKOixTJdr5wQGRNPYR0tpe0G2pqpznCkpUqpwtiW1Et17UTEe9n19dP3wxW2n74ne13BTLkNtTTUcHUw5cTdMHn+vKavr09kKr/7qKjUZzg+Z2BgQORnsxCf54WUM3bZrEik0pfrp5kS/Y8zH2lp3rKV7LPs4cN00+UGPX2DVGfeoqMzN65SH5Of6p21cZWWJs1aEv/hA/6PHqWbLj+oZeNH1eT/sn4VZWFhwYQJE1i0aBEPHz6kdu3ajB8/XiFNUlIS8+bN48iRI0RERMgXPUZERKCTyq1rQUFBhIeHU716dYV9ODg45G5lkK29ePZE+Rpo4NMALCxL5/rxsyK1M8PcZlXamiepXEN/+iQAy9LWGeY97fsvcbGxCtePnz7xR1NTEzNzi3Ryp5yZfX7SYlXahnWbdxIaEkxsbCwWFqU46fsvABUr535/+dzD0GjKGCuvvbA11uPRi8yvg+jkZEFCUuqzDg8/rqdI6ww3tbWgTSqWpHAhLbZfDFR+M49Y29jg/1j5yynA35/S1sproL7Me+L4v8TGxiqsywjwl/WbT2swrK1tiY+P5/nzQIV1GZ/WfFhbp98/P01C5cdZsVVpa54EpDKuAjIxrqytOZXKuHoSkMlxhfK4SjWdfPyp3qyBKpYpq9INfMLCwtL9ef36dXrZv3murq5s2bKFU6dOoaWlhaenp8L7+/btw9fXl7///purV69y9OhRIO1Oa2xsjJmZGVeuXJH/XL9+PU8ey+5Spz737t4iNDjlA/5FaDB3b93ApXb9XD9+Zpw4KpvRsS9fMc+PXbteA+7euaWwUj80JJhbN65Tu276l7Jq12tAYmIiJ/49It+WmJjI8aOHqeFcEy0t5VmAz9Od+PcoJUoapzpDYWxiKv+y2uHzDzWca2b44ZrTjt1+QVVLQyyMUgJnsyLaOJYuovSsi7RoqktoU80U37thvHmnPOtw7WkEYZFx1C9XXGH7p9epPf+ik5M5b9594Pid/HsgYL0GDbl96yZBz1P6TXBwEDdvXKNe/YYZ5G1EYmICx44elm9LTEzk6OGDuNSsLe83tWrXQVNTk4P7FW8dP7h/Lza2ZTA1M0/3OIcO7qNgwYLYprJANbfVqdeAu7dTGVc3r1O7Xu6OK99jaY+rzx09fIACBQtibau8QDW/qUmy/qNq0p3JqFevXrqRVFrXCr8HAQEBhIWFUa1aNQoUKEDBggWJi1O88+H9+/doaWlhaGhIbGwsCxYsUHjfyMiIoKCUL/VKlSqhq6vLihUr6NmzJ5qamjx58oSYmBj5w85yS4s2Hdm7fQuTfvOg9wB3JBJYt3IxxUqUoFW7zvJ0YaEh9OrSiu59BtL9l5SFULeuX+FtxBsi3sgCy4f378rPLuo2bCpP9+yJv3zG5MOHD7x8EcrpE7Lgq5KDI4UNiwAwuFcXGrdwxdzCEiRw7dIF9mzfjKNzLapUq5GrbZGaNu07scPnH8aOdGfAEA+QwKql3pQoWZK2HVPa50VoCF3aNqd3v0H8MkC2kK6MXVkaNW2B17zZJCYmYmJqxq7tWwgNCWLS9NnyvMcOH+DMKV9catWheImSRLx5zY6tm3lw7y5TZs5RKM/6NSspaWxM0WLFCXsRys6tmwl7EcrSNRvzpkE+88/5Z/SuZ8XqgU7M2XcPKTC6lT0hEbFsPPtUns7UUJuzkxuz4NADFh5WnJpvVKEkhoW0lBZ8fpKULOXPvf8xv0dVZnarxOEboVgWK8QY17KcfxjOuYevFNIb6WpRr2xxNpx5muVbXnNSx45d8PlnE8M9hjDUfRgSiYQl3gspUbIknbp0lacLCQnGtUUTBgwawsDBbgDY25elWfOWzJk9k8SEREzNzNjms5ng4CBmzk55Dk8RIyO69+zNmlXLKVSoEPZly3H08EEuXbzAAu8l8nTXrl5hzaoVNGrcBBNTM95FR7Nv7y5O+p5g2IhRGT4YLDe06dCJ7T7/8NvHcSWRwMol3pQoUZJ2n42r0BDZuOrTX3lcLZz7cVyZmLHz47iaPCNlXB09fIAzJ32pWVs2rt68ec1On83cv3eXKbNSxtWNa1fZ8Pcq6jdsjLGxKe/eRXNw/x7OnvJlsMeIDB8Mlh9UMWjIqnSDjE9n5j+i+Ph4/vrrLx4/foyGhgYODg5MmzaNrVu3ytO0a9eOM2fOUKdOHQwNDXF3d8fHx0f+fqdOnRg2bBiOjo5UqVKFVatWsXTpUmbPnk2jRo2Ij4/HysoKDw+PXK+PtrYOnt6rWObliefU8UiRUqWaE4OH/6rw4SNFSnJSEsnJyQr5169awq3rV+Sv9+7Ywt4dWwA4ev6WfPup40cUHhd+89plbl6TPcZ3zqLV8iDDzMKSvds38+b1K5KTkzA2NefnXwbS5ee0H16Um7S1dfBatgavv2Yz9Y+xSKVSHKs7M2z0WIXFYFKplKSkJKXr4xMmTWf5koWsXOrFu+hobGztmOe9HLuy5eRpjE3NiHjzmsUL5xEVGUnBggUpW74Cf3kvx6mm4qOy4+JiWLHEi1fhL9HV08fJpRbTPedTIhOPgM9psfFJdPU6z6SOFVjQsyoSiYRzD8KZvOM2MZ897VMiAQ11tVQXpnZ2MififTzH76Q987H94nOSk6UMaWJLFycL3sYksOtyEH/u+U8pbXv5A73y71IJgLaODivWrGPu7Fn8Pu5XpFIpNZxdGPPb+FT7TfIXAdGU6bNY5DWfxd4LiI6OooydPYuXraJsufIK6dw8RqCjo8M/G9fz6uNjxT3nLVCYLSlarBhSaTJLFnvxNiICDQ1NbMvYMctzHi3y6XH02to6eC9fg9e82UydOBakUqrVcGb4F+MKUm+f3ydPZ9nihaxY8nFclbHjr0WK48rERDauFi2YR1TUx3FVrgJ/LVqO82fjqmjRokiTk1m5dBGRbyPQ0NDA2taOyTM9adq8VW43RbZ8DyfxEmlqS3UFlfXs9Yf8LoLKKlQg756C+S1yGHsgv4ugsh7OV/47PIJMTHxixol+YEaFcm9p45j9D7KcZ05ru1woSfaJhZ+CIAiCoIK+g4kMEWQIgiAIgipSxSd4ZpUIMgRBEARBBanicy+ySgQZgiAIgqCCvoOJjMwHSi9fvmTt2rVMnTqViAjZPevXrl3j+XPV/QuAgiAIgiDkn0wFGVevXqVFixacPHmSrVu38u7dOwAuXrzI3LlzM8gtCIIgCEJWfQ9/hTVTQcbs2bMZNmwY69atQ/OzPz9dq1Ytrl27lmuFEwRBEIQflUSS9R9Vk6k1GY8ePaJhQ+VH5BoaGvL2rfIf+BEEQRAE4et8D0/8zNRMhpGRUaprL65evYqZmVmOF0oQBEEQfnQ/zOWS7t27M2XKFPz8/AAIDAxk27Zt/Pnnn/Tq1StXCygIgiAIP6If5nJJ79690dbWZvz48cTGxtK3b1+KFCnCoEGD6NatW26XURAEQRB+ON/D5ZJMPyeja9eudO3alXfv3vH+/XtKlCiRm+USBEEQhB+ahG8/ysjyw7h0dXXR1dXNjbIIgiAIgvDRDzOT0bRp03T/5OyRI0dyrECCIAiCIPxAQUb//v0VXicmJvLgwQOOHTum9J4gCIIgCF8vvZP7b0WmgozOnTunur1KlSqcOHGC3r1752SZBEEQBOGH9z3MZHzVH3mrWrUqZ86cyamyCIIgCILw0fdwC2u2g4yIiAjWrl2LsbFxTpZHEARBEARy/2Fcb9++ZejQoTg4OFC/fn12796dYZ6xY8diZ2fHs2fPMnWMTF0uKV++vNK1oaSkJIyMjJgzZ06mDiQIgiAIQubl9uWSqVOnoqmpydmzZ7l37x4DBgygbNmy2NnZpZr+0qVLBAUFZekYmQoyVq1apfBaIpFQpEgRLC0t0dLSytIBBUEQBEHIWHYuf0RFRREVFaW0XV9fH319ffnrmJgYjh49yr59+yhUqBCOjo40btyYvXv3MmbMGKX88fHxTJs2jXnz5uHq6prp8mQYZMTHx7N7927c3NwwNzfP9I6F3FHCoEB+F0H4Rj1a0Da/i6CyDKu75XcRVFbE5UX5XYQfllo2Hsa1bt06Fi1S/p25ubnh7u4uf/306VPU1dWxsrKSb7O3t+fixYup7nfFihXUrl2bMmXKZKk8GQYZWlpanDx5kqFDh2Zpx4IgCIIg5K1evXrRvn17pe2fz2KAbCbjywdr6urqEhMTo5T36dOn7N27l127dmW5PJm6XNKuXTu2bt3K6NGjs3wAQRAEQRCyLjuXS768LJIWHR0d3r17p7Dt3bt36OjoKKWdPHkyI0eOpFChQlkuT6aCjKioKI4cOcLJkyext7dHW1tb4f1p06Zl+cCCIAiCIKQtNxd+WlpakpSUxNOnT7G0tATg/v372NjYKKX18/Pj0aNHCt/1Xbt2ZezYsbRr1y7d42QqyEhMTKRRo0by13FxcZnJJgiCIAhCNmX1ltSs0NHRoUmTJnh5eTF9+nTu3bvH8ePH2bRpk1Las2fPKryuXbs2y5Ytw97ePsPjpBtkhISEYGxsLG5TFQRBEIQ8ltsP15o0aRITJkygZs2aGBgYMHHiROzt7QkJCaFVq1YcOHAAExMTihUrppTX0NCQggULZngMiVQqlab1ZtmyZTl79ixGRkZfVxMhx8Ql5ncJBOH7I+4uSZu4uyR9BbP8t8wzb/WlwCzn6VvDIhdKkn3pNk868YcgCIIgCLlIFR8TnlUZxmDfw1+BEwRBEIRvzVf9cTEVkWGQMXToUDQ1NdNNs379+hwrkCAIgiAI38dJfoZBRpUqVbJ1b6wgCIIgCNn37YcYmQgy+vXrJxZ+CoIgCEIey81bWPNKukHG9zBVIwiCIAjfou/hG1jcXSIIgiAIKuh7OM9PN8i4f/9+XpVDEARBEITPfA9XE3LxMSKCIAiCIGTXD3ELqyAIgiAIeU/MZAiCIAiCkCu+/RBDBBmCIAiCoJK+h5mM7+GSjyAIgiAIKkjMZAiCIAiCCvoeZgFEkCFk2YvQUObMnsUFv3NIpVKcXGry62/jMTYxye+ifZWvqdeHDx9Y7L2AA/v2ER0dhZ19WYaPHE01x+oK6ZKTk1m7eiXbt/rw6lU4lpZWDBw8lMZNm8nThIe/5J+NG7jgd47AZ8/Q1NTEtowdg4a4Ke0vr6hK2wBMnDCO27du8DIsjORkKebm5rTv1Jmu3X5CXV09R+udGWYlCuM5uiMNneyRSMD34gPGzN3B8xcRGeYtZWLErBHtaOBkh6aGOlfuPGP8gt1c+0/xT3wbFS7EjGHtaFmvArraBbjzKISpSw/wr989hXTLJ3enRkVLTIoboKamRkBQOH/v8mP51tMkJ+fPc49Uqe/s3b2Lk74n+O/uHUJDQ2jTtj3TZv6Zo/XNSeJyifDDiY2Npf8vvXjyJIBpM2cz409PAp89o98vPYmJicnv4mXb19Zr8sTx7Ny+jSFuHngvWU7RYsUYPKAv9+8pfgks9l7I0sXedPvpZxYvW0nFylUYPXIYZ06fkqf57+5djhw+SP0GjZg734upM/6kQIEC9O3dg1MnfXO87hlRpbYB+PAhjv/91J05fy3kr4XeOLnUxHPWDOZ6zsrRemeGdkFNDq3woIxlCfr/sYG+E9djbVGcwys80CmolW7eIgaFOLF2BOWsjXGfvoV9RSxmAAAgAElEQVSeY9cCcHiFB3ZWJeTptDQ1OLTcg6a1yjFhwR66jV5FUFgEOxcOok41W6XyLN1yip9/XUO3USvxvfiAuWM64jmqQ85XPhNUre8c2L+XoOeBONesia6ubo7WNTdIsvGjaiTS7+ixnj169KBNmzZ07tz5q/azc+dOtm3bxubNm3OoZDknLjF/j79pwzrmev7Jnv2HsShVCoCgoOe0admM4SPH0LN3n/wtYDZ9Tb0e3L9Pl45tmTJ9Ju3adwQgMTGRDm1bYWlphdfiZQC8fv2aZo3q8Uu/AQxx85Dn7/9LLyIi3rB91z4AoqKi0NHRQUMjZaLx0/6MjIqydv2mHK9/elSpbdLy2+iRnD7li9/l69mqo2F1t2zlG/q/+swe1YFK7acS8PwVIJuduLPnDyYs3IPXxhNpl7lfM34f2JLKHabJ8+oU1OK//ZM5e/Ux3X9bA0C3ltVZO6MXTfst5MzVR/L8l3zG8SE+gTo95qZbxnWzetOiTgWK1x6drTpGXF6UrXygen0nOTkZNTXZuXWThnVxdq751TMZBXPxesCe2y+ynKdtxZK5UJLsU/mZjIYNG3L+/Pn8Lobw0UnfE1SqVFn+gQFgZmZOFYeqnPQ9no8l+zpfU6+TvsfR0NCkWfOW8m0aGho0b9GK8+fOEh8fD8D5c2dISEiglWsbhfytXNvw6OFDgoKeA6Cvr68QYHzan519WV6+DPuqemaHKrVNWgwKF0ZdI++v/raqV5FLt5/IgwSAZyGv8bsZQOv6FdPNW6OiFY8DwxXyxsTFc/66Py3qVEBdXfbx7FTRkpjYeIUAA+D4hfs4VrDEpJhBusd58/Y9iUnJWa1ajlC1vvMpwPhWqCHJ8o+q+bZaXMh3/o8fY21bRmm7tbUNAf6P86FEOeNr6uXv/xhTM1O0tbUV89rYkJCQQGDgM/kxtLS0sLAopZjOWjblHeDvn+YxEuLjuXXjBlalrTNVn5ykim0jlUpJTEwkKiqKf48eYd+eXfTo2TurVftqZa2Nufs4VGn7f/6h2JdO/4wyKTmZ+ETlqckP8YnoaGtR2qyoPF1CYlKq6QDK2SivbVBXV8NAV5t2jarws6tTujMquUkV+863RCLJ+o+qUekgY8yYMYSEhDBo0CAcHBxYvHgxN27coFu3bjg6OuLq6oqfn1+a+Xfs2EGLFi2oXr06ffr0ISgoSP6enZ0dGzdupHHjxjg5OTFjxgySkhQH8rx586hRowYNGzbkzJkz8u1hYWEMGjQIJycnmjRponBZxdvbm+HDhzN+/HiqVq1Kq1atuHv3rkJeDw8PXFxcaNiwIX///XcOtFTeiYyMRF9fX2m7gYEBUVFR+VCinPE19ZLlVT6bNDAoDEBUZKT8v3p6+kqLuQwMDD7u522ax1i6ZBFhYS/4pW//9CuSC1SxbU6fOkm1yuWp41Kd0SOH8b+fezBw8NDMVyqHFDHQ4W208tqCiMgYDPV00s376GkYNubFKWJQSL5NIpHgWKHUx33Ltj98+hIDPW2FdRoATpUs5WX4XIs6FXh3xYsXZ+awyfMXlm45xZ8rD2e5bjlBFfvOt0SSjX+qRqWDjDlz5mBiYsKyZcu4fv06nTp1YsCAAQwcOJCLFy8ybtw4hg0bRnh4uFLef//9l6VLl+Ll5YWfnx81atRg+PDhCn9Z9tChQ2zbto09e/Zw9uxZtmzZIn/v1q1blCpVCj8/P/r06cPvv/8uf2/UqFGUKFGC06dPs3DhQhYsWMC5c+fk7x8/fpzmzZtz+fJl6taty4wZMwDZ9cDBgwdjY2PDqVOnWLduHRs3bsTXN+8X832N1FY8fw8Le7JdL6k09bxfLHeSppUug6Mc3L+PNatWMGDQEKpWc8xMiXKcqrVN1WqO/OOznRWr/+aXfgNYt3YN3gvnZ6ZEOS61VW2ZuStg5fazqKlJWDWtB1ZmRSlZVJ+/fu2EpYkRIPu8APA5dIWXb6JZNbUH5W1MMCpciDG/NKV2VZuP6RQLcO76Y2r97EmLgV7MXXuM4T0bMXmo61fWMvtUre98S8RMRh7bu3cvderUoUGDBqirq1OzZk2qVKnCyZMnldL6+PgwYMAAbG1t0dDQYODAgfj7+/P8ecr1uQEDBmBoaEjJkiXp3bs3Bw8elL9nampKp06dUFdXp3379rx48YKIiAhCQ0O5evUqY8aMoUCBApQrV47OnTuzb1/K4iJHR0fq1q2Luro6bdu25d7HldB37tzh9evXuLu7o6Wlhbm5Od26dVM4rqrTN9An8uMZxOei0jhj+VZ8Tb30DQxSPVuKioqUv//pv1FRkUofklGRsjO6T2donzvpe4KJE8bRvkMnhUVteUkV20ZPT4/yFSri5OyCx/CR9BswkDWrVhAWlrdrViKiYjDUV56xKKyvTUQqMxyfexr8mj4T1uFQ1oL/9k3mybGZ1Khkhfcm2UnHi1eyuke+i+Wn0aswKqzLlW3jCfKdTa+2LkxfLvvcCH2l+LuJehfHtf8COXnpIZMW7cNz9VFG92mS4dqN3KCKfedb8j2syfimnpMRHBzM0aNHcXRMOZtLTEykSpUqqaadNWsWnp6e8m1JSUmEhYVhYWEBgMln92mbmJgozIgYGRnJ/79gwYIAvH//ntevX2NgYKBw+5OpqanCJZHP82pra8tv1QoKCiI8PJzq1VPu8U5KSsLBwSELrZC/rK1t8H/8SGl7QIA/pa1t8qFEOeNr6mVtbcOJf/8lNjZW4fpxgL8/mpqa8mvFNja2xMfH8zwwUGEh3Kdr06WtFddbXLzgx5iRw2jYuDETJ0/Ndt2+liq2zZfKla9AcnIywUHPKVGiRLppc9I9/1DKWRsrbS9b2pj7ARnfGbD7+A32+t7EtlRx4hOSeBL0ioXju/I89I3CczbOXfennOtkrC2Koa6mxqNnLxnZqxExsfHcuJf+othr/wWirq6GpakRIeHKX/i56VvoO6pMFWcmsuqbmskwNjambdu2XLlyRf5z48YNhgwZkmraKVOmKKS9deuWwhd8SEiI/P9DQ0MpVqxYhmUoXrw4kZGRvHv3TiFvZj7YjI2NMTMzUyjT9evXWbNmTYZ5VUX9Bg25fesmQZ/NCAUHB3Hj+jXqNWiYjyX7Ol9Tr/oNGpGYmMCxIynXvRMTEzly+CAuNWujpSV7XkLN2nXQ1NTk4AHF2zEP7N+LjW0ZzMzM5dtu3rjOMLchODm7MPPPOfm6Kl7V2iY1V69cQiKRYGaefrqcduDUbWpUtMTSNOXEwsK4CC6VS3Pg1O1M7SM5WcqDJ2E8CXqFcTEDOjWtyortZ1NN6x8YzsOnYegU1KRP+1r8c+AS72Pj091/nWo2JCcn8yT4deYrlkO+hb6jyr6HyyXqkydPnpzfhUjPgQMHMDExoUKFCpiamuLp6YmtrS1mZmYkJiZy7do1JBIJenp67Nq1Czs7O8qXL4+Ojg5LliyhevXqGBkZER0dzYkTJ7C1la04XrRoEWFhYTRs2JDIyEimTJlChw4dqFSpEvfu3eO///6jU6dOgOza6JIlS+jVqxempqacP3+eR48e4ezszOPHj5k5cyZDhw6lVKlSXLp0icjISJo2bQrIFi9t2LABd3d3ihcvzp49e4iMjKR8+fJIJBICAgIICgrK9NlXYv7ciSZnY1uGwwcPcOzYEYoXL86zJ0+YNvkPtAoUYMrUGWhqpf8AIlWV2XqFhARTr5YzUqkUx+o1ACharBhPngSwZfMmChc2JCoqioV/zePO7VvMnD2HYsWKA6Cjo0NMTAzr1q6mYEFt4uPjWbt6JceOHmHSlOlYWlkB8CTAn/6/9MbAwIARo8YQEfGGsLAX8p8SJfP2PnhVapvTp07iteAvPsTFERkZyaOHD9i4YR1b/tlEpy5dadGydbbqOHtl9i5Z3nkUQpfmjrRv7EBo+FtsS5Vg8cT/8SE+kcFTNsnvCrEwNiTIdzZI4OxV2Rm2hoYaf45oT8ECmhgXM6Bl3Qqsnt6TR0/DcJ+xRWGtxVT3NhTW16aYoR4Nne1YNbUnSUnJ9B7/N3EfEgBoXrs8U93bULCgFob6OlSwNcXt5wYM7lqP1TvPse3I1WzVceyAlhknSoMq9R2Q3Yly9fIlAvwfc+L4v2hpaaGjrUOA/2MMixRRupMlMzRyMf4PeBWb5YWf1sXSX3Cc11T+csmAAQOYPn06c+fOpW/fvixZsoS5c+cyevRo1NTUqFixIpMmTVLK16RJE96/f8/IkSMJDg5GT08PJycnWrRoIU/TrFkzOnXqRFRUFG3atKFbt26ZKtNff/3FpEmTqFOnDvr6+ri5uVGnTp0M86mrq7N06VJmz55No0aNiI+Px8rKCg+P/LnWnh06OjqsXLOOObNnMWHsr7LHBDu7MGbseHQKFcp4Byoqs/WSSqUkJSUpXf+dOn0W3gvns8hrAdHRUZSxs2fJ8lWULVdeIZ37sBHo6Ojwz8b1sscfW1kxZ94C6n92Vnfr5k2ioiKJioqkX5+eSmW9efdBDtc+farUNubm5iRLk1nkvYA3r1+jp6+PhUUpps+ane0A42vExMXTYqAXnqM7snpaTyQSCScvPWD0nB1fzDBI0NBQR02S8o0klYK1RXG6tHCksJ42wWFvWbfbD881R5RuWS1eRI85oztSrIge4W+i2et7i2lLDxARlbLuIyDoFWoSCZOHtKZYEV3eRsfiHxhOv4nr8TmcvQDja6lS3wE4euQQy5akPFzsyuVLXLl8CYBVa9dTpIZTTjfBV1FTwZmJrPqunviZFXZ2dhw9epRSn13D+xbk9xM/BeF7lN0nfv4IvuaJnz+C3Hzi54n7Wb/E1dDeKONEeUjlZzIEQRAE4UekimssskoEGYIgCIKgglTx4VpZ9cMGGQ8e5O11bUEQBEH40fywQYYgCIIgqLLvYeGnCDIEQRAEQQWJyyWCIAiCIOQKsfBTEARBEIRc8R3EGCLIEARBEARVpPYdTGWIIEMQBEEQVNC3H2KIIEMQBEEQVNN3EGWIIEMQBEEQVJC4u0QQBEEQhFzxHSzJEEGGIAiCIKii7yDGEEGGIAiCIKik7yDKEEGGIAiCIKggsSZDEARBEIRcIdZkCIIgCIKQK76DGAO1/C6AIAiCIAipkGTjJwvevn3L0KFDcXBwoH79+uzevTvVdLt27aJDhw5UrVqVOnXqMHPmTOLj4zN1DBFkCIIgCIIKkmTjX1ZMnToVTU1Nzp49y9y5c5k6dSoPHjxQShcbG8v48eO5cOECO3bs4OrVq6xYsSJTxxBBhiAIgiD8YGJiYjh69CjDhg2jUKFCODo60rhxY/bu3auU9qeffsLR0REtLS2KFy9O27ZtuXHjRqaOI9ZkfGMSkpLzuwgq63v4Y0K5KTY+Kb+LoLIiLi/K7yKoLMO2XvldBJUWe8Aj1/adnY+0qKgooqKilLbr6+ujr68vf/306VPU1dWxsrKSb7O3t+fixYsZHuPq1avY2tpmqjwiyBAEQRAEFZSd06Z169axaJFy0Ozm5oa7u7v8dUxMDLq6ugppdHV1iYmJSXf/e/fu5erVq0ycODFT5RFBhiAIgiCoomxEGb169aJ9+/ZK2z+fxQDQ0dHh3bt3CtvevXuHjo5Omvs+ceIEs2bNYs2aNRQtWjRT5RFBhiAIgiCooOw8jEtfX08poEiNpaUlSUlJPH36FEtLSwDu37+PjY1NqulPnz7NuHHjWL58OWXLls10ecTCT0EQBEFQQRJJ1n8yS0dHhyZNmuDl5UVMTAxXr17l+PHjuLq6KqX18/NjzJgxeHt7U6VKlSzVQQQZgiAIgqCCcvkxGUyaNIkPHz5Qs2ZNRo4cycSJE7G3tyckJAQHBwdCQkIAWLJkCdHR0QwcOBAHBwccHBxo1apV5uoglUqlWSyXkI+iP4i7S9Ii7i5Jn7i7JG26BcWV47SIu0vSl5t3l9wJfpdxoi9UMNXNOFEeEiNLEARBEFSQ+ANpgiAIgiDkiu9hclYEGYIgCIKggr6DGEMEGYIgCIKgkr6DKEMEGYIgCIKggsSaDEEQBEEQcoVYkyEIgiAIQq74DmIMEWQIgiAIgkr6DqIMEWQIgiAIggr6HtZkiMeKC4IgCIKQK8RMhiAIgiCoILHwUxAEQRCEXPEdxBgiyBAEQRAElfQdRBkiyBAEQRAEFfQ9LPwUQcYP5MWLUP7y/JOLF86DVEoNZxdG/TqOksYmGeb98OEDyxZ5cfDAXt5FR1PGzh734aOo6lhdnubZ0yds27KZK5cvEhwUhE4hHcqVr8hgNw/K2Nmnue+bN67Tr9fPSKVSLly7jYZG3nfLFy9Cmec5i4t+55FKpdRwrsno38ZhnMm2WbJoIYf27yM6OooydvZ4jBhNtc/aBmDjurVcvnyRe3fv8upVOAMGD2XQEHel/SUlJbF50wb27NpBcHAQuoV0qVipMgOHuFPGzi7H6pwVYS9C8fprNpcv+CFFimMNF4aN+i3TfWflUm+OHtxH9LtobMvYM8RjJFWqOiqlDX8Zxsql3vidO010VBRFixWnUdMWDHYfAcCr8HC2bdnI5Yt+BAU+Q1NTE2vbMvwyYEiq+8sLL0JDmTN7Fhf8ziGVSnFyqcmvv43H2CRzbbPYewEH9sn6jp19WYaPVO47ycnJrF29ku1bfXj1KhxLSysGDh5K46bN5GnCw1/yz8YNXPA7R+AzWdvYlrFj0BA3pf3lJbOiunj2r0tDB3MkEgm+NwIZs+I0z8Mz/jPm5sV0+aO7C/UqmWGkX5Dg1+/YceYRc7ZeIeZDojyddgENpvR0oWNtW4roF+RxyFvmbbvKlpMPFPa3YkRjejQup3ScRbuvM2blma+vbA77HtZkSKRSqTS/CyFkXvSH5Gzli4uN5X+d26GpqcUQ92EgkbDUeyFxcXFs2b4bbR2ddPP/PnYMZ8+cYtiI0ZiambPN5x/Onz3Dmg2bsbMvC4DP5k3s2r6V1m3aYV+2HNHRUaxfu5oH9++xev0/lC1XXmm/iQkJ/Ny1I2/fRvD61auvCjLUsjkiY2Nj6dapHVpaWgxxG4ZEImGJ9wLi4uLw2bEnw7aZ8Ntozpw5xfCRYzA1M2frln84f/Y0f2/cIm8bgA5tWqKrq4t92XJs37olzSDDe+FfrFuzij59B1DdyZm3ERGsWrGU8JdhbNm+hxIlS2avnvFJ2coXFxtLr/91QFNLiwGDPZBIJKxY4kVcXBzrfXairZ1++0ye8Ct+Z08zdNgoTMzM2bF1MxfOn2H52k2UsUtpn9CQYAb90h0TE1M6detOESMjQkOCCXoeyIAhHgCcO32SBXNn0dK1HeUrViYxIYGd27Zw4fwZZv+1iFp162erjroFs9fnYmNj6dKhLZpaWrh5DEcigUVeC4mLi2Xbzr3oZNB3xv06ijOnTzFi1K+YmZuzZfMmzp05zfpNPtiXTWkb74XzWbd2Ne7DRlC2XHkOHzrIzu1b8V6ynDp16wFw6qQvs2dNp227DlSqXIWEhAS2bvmHs2dOs3DRUurVb5CtOhq29cpWPpB9+V/y/okPCUlM2eCHFJjUwxmdAppUH7pJIVD4kk4BDS54/w9NdTWm/3OR5+HvcLQtzu8/O3PgYgA9Zh+Wp90ztS1O9iWZssGPh0FvaVvTmv4tK/LL3KNs9r0vT7diRGOaOVrSeep+hWO9ePOewPDobNUx9oBHtvJlxtNXcVnOY1m0YC6UJPvETMYPYteObQQHBbFj70HMLUoBYGtrRwfX5uzYvpXuPXunmffhg/scPrifP6bOoE27DgBUdaxOl/auLFvszXzvJQA0a96SLt1+QvLZl331Gs64Nm/M5o3rmTpzttK+1/+9BqlUSpt2HVm7ankO1jjzZG3znJ37DmHxqW3K2NGudTN2bPOhe68+aeZ9+OA+hw7uZ9LUGbRt3xGAao7V6dy+NUsXe7HAe6k87fbd+1FTUyMxMZHtW7ekuc99e3bRtFkLhnoMl2+zLWNHx7YtOXP6JJ26dPvaKmfJ3l3bCQkOYvPO/ZiZy9rH2rYM3dq3ZM+OrXTr3jvNvI8e3ufY4QOMnzSdVm3aA1ClqiPdu7Rl1bJFeM5fLE87Z+YUihUvjvfytWhoagLgUE3xDLySQ1U27zygEIjWcKlF9y5t2bR+TbaDjOzauX0rQUHP2bP/MBalUvpOm5bN2L7Vh5690+47D+7f5+CB/UyZPpN2n/WdDm1bsWTRQrwWLwPg9evXrFu7ml/6DaBXn74A1HBy5nngMxbOnysPMhyqVmPvgSMKbVOzVm06tG3F32tWZTvI+Bq/NCuPVUl9Kg3cQEBoJAC3n7zizsqe9GtREa/d19PM61LOBFtTQ1r/vpvj1wMBOH0rCEO9ggzvUBXtAhrEfkikZjljmlYrRf/5x9j47z0Ajl8PxLSoLjP61MTn1AOSk1POpRMSk7n04EUu1joHfQczGeI5GfkoKCgIOzs7EhPTjuZzyumTvlSoVFkeYACYmplRuYoDp32PZ5D3BBoamjRt1kK+TUNDg2bNW3Lh/Fni4+MBKGxoqBBgAOjq6WFRypLwly+V9hv0PJA1K5czdsIf+XKJ5JNTJ09QsVJleYABKW1zMoO2OeX7sW2at5Rv09DQoGnzlvidS2kbADW1zA23xIQECunqKmzT09cDID8mHs+e9qV8xUryAAPAxNSMipUdOHPKN/28p3zR0NCgUZPm8m0aGho0btqCS37n5O0T9DyQi37n6NT1Z3mAkRo9PX2lvqKhoYFtGftU+1huO+l7gkqVKssDDAAzM3OqOFTNsO+c9D2OhoYmzb7oO81btOL8Z33n/LkzJCQk0Mq1jUL+Vq5tePTwIUFBzwHQ10+9bezsy/LyZdhX1TO7WjmV5tKDF/IAA+BZWBR+/4XS2rl0unm1NGTjJTomXmF75PsPqElSVivUsJfN7B298lQh3bGrzzA20sXJLnszf6pAko1/qkYEGUDDhg05f/58fhcjVwX4P8baxlZpe2lrGwIC/NPN6+//GBNTUwpqayvmtbEhISGB54HP0swbGfkW/8ePsCyt/IEya/oUGjVpqrCuIz8EPE69baxtbDPVNqZmpmh/0TbW1rYZtk1aOnf9iYP793HyxHHevXtH0PPnzJo+lRIlStK0eYuMd5DDngQ8prS1cvtYlbbmaQbt8yTAH2NTM6W+Y1Va1neCnsvOUG/flJ3RFihQgGFD+lHfuQrN67sw7Y9xRL59m+4xEhLiuXvrBpZW6X9p5Qb/x4+xti2jtN3a2oYA/8fp502r73wcV4Ef+47/48doaWkpBMGyY8h+JwH+af8OEuLjuXXjBlalrTNVn5xWtlQR7j57rbT9v8DX2FsUSTfviRvPeRQcwfQ+tbA3L0KhgprUq2TGkDZVWHnotvxSS9LHWYr4RMVLyR8SZJcHy5UyUthezECb5//0J3qvG7dW9GBUp2qoqanelzPI1mRk9UfViMslGUhISEAznTOrb0VkZCT6+vpK2/UNDIiOiko3b1Q6eT+9n5Y5s2YgRcpP3XsqbD+4fy/37t5l+54DmSl+rpK1jYHSdn39zLTNW/RSyWvwsW0i02mbtAx280BTS4vRI9xJTpZ9cJaytGTF2vUYGBTO8v6+VlRkJHp6afSd6Iz7Tlp5AaKiZO3zKlw2CzFz6kSat3SlZ+9+BAUFsmzRAp4E+LNq/ZY0Z4JWL1/Cy5dh/DHDM0v1yglpjSsDAwOiMug7afW7T7/jT+PqUxt+OUuY0sfSDsKWLllEWNgLZnnOTb8iuaSIbkHevvugtD0iOg5D3QLp5v2QkESjMdvZPL4l15d1l29fc/gOI5aelL9+GBQBQA27khy9mhLUO32c4TDUS1mjcCvgFdcfv+S/Z28oqKVOGxdrpvaqibVJYYZ4pT/zlB9UMGbIsh9+JmPMmDGEhIQwaNAgHBwcWLx4MXZ2duzYsYOGDRvy008/ATBixAhq1apFtWrV6NGjBwEBAQDcvHkTZ2dnhUsep06donHjxoBsVfiKFSto0qQJTk5OeHh4EBERkfcVBaUPKYDMzL5LpdI08qafee2qFRw+uJ9fx/2ucJkmMvItC+Z6MsRjOEWMjNLZQ95J7QxASsaNI5Wm/kHwNZc1tvlsZvWKpfQdMIgVa9bhOW8BOjqFGDqgL+H5NO2dWgNlqu+Qub6T/PG1Q7XqjBo7kWo1nGnboQujxk7kwb27XPQ7l+r+jx7az8a/V9G73yCqOFTLREVyXqr1y0zGTI6rNMdfBkc5uH8fa1atYMCgIVStlj933kDq/SS1+nypgKY6G8a2oFhhHfrMPULjX7czbvUZOtUtw4LB9eXp/r0WyL3AN8wbWA8n+5IU1i1Ar6bl6FLP7uPxUwqwaM8Nlu67xalbQRy58oyh3idYvPcGfZqVx9pEOeDLb9/DTMYPH2TMmTMHExMTli1bxvXr12nbti0A58+fZ9++fWzcuBGAOnXqcOTIEc6fP4+trS1jxowBoHLlyujr63PuXMqH4P79+2ndujUAGzZs4OjRo6xbt44zZ85QuHBhJk2alMe1lF2vTe2sOjoqEr1UzsQ+Z2BgkEZe2Znap7PSz23fuoXFXvMZ7DZMviDyk6XeCyliZESTZs2JjooiOiqK+HjZ2c67d9HExsRkul45Ie22icqwbfQNDORn45/7dBZrkErbpCcy8i3zPGfRo9cvDB7qgWN1Jxo3bc6SFauJiHjDurVrsrS/nKCnb0B0KnWMjkp9luJz+vqpt4+873w8k/909l7dqaZCuhrOstcPH9xT2sfZ077MmDyB1m070G+QWyZqkvP0DVLvO2nN/inmNUh1FuJTe30aV5/62JfBR1Tkpz6mPLt10vcEEyeMo32HTgxxy727HzIS8e4DhnrKMxaFdQsQkcoMx+d6Ny1PvUpmtJu0ly2+Dzh3N4QFO68zdtUZBrSqREWrooDscslPMw/y/kMCJ+d1IdRnIJN7uvDHOtkl8KFteXEAAB/kSURBVNA379M9ztZTDwGoZlsiO1XMZZJs/KiWHz7ISIuHhweFChWiQAHZAOnQoQO6uroUKFAANzc37ty5Q8zHL8PWrVuzb98+AOLi4jh+/Diurq4A+Pj4MGLECExMTNDS0mL48OEcO3ZMYUFgXiidxjXigAB/Smdwvba0tQ0hwcHExcYq5vX3R1NTU2GWAuDAvj3MnjGV7j370HfAoFSP+fjRQxrVcaFBbSca1HZi3ZpVADSuW5Pfx/2a1ep9ldI2abSN/+MM28ba2obgoGBiv2ybgMeptk1Gnj19Snx8POUrVFTYbmBQGDNzC548SX8NRG6wKm3Nk1Ta5+mTACwzaB+r0taEBgcp9Z2nT2R9x8zcQp4O0j4T+/L25CuXLjDxt5HUbdCYXydMzmRNcp61tQ3+jx8pbQ8I8Ke0tU2GeVPtOx/H1ac1GDY2tsTHx/M8MPCLdLLfSWlrxd/BxQt+jBk5jIaNGzNx8tQs1ykn3Qt8TTkL5dnKsuZFuB/4Jt285S2NeBMdx5MXikHclYey2Tx7c0P5tvvP3+Dsvhm7PmupOngjtr3W8uJjcOH3X2i6x/nUtVTxaQ5iJuM7ZvLZg3SSkpLw9PSkUaNGVK1alaZNmwLIL3u4urpy/PhxYmNjOXHiBJaWllh/HPjBwcG4u7vj6OiIo6MjTZs2RUNDg/Dw8DytT936Dbhz66Z8JTpASHAwN29cp279hhnkbUhiYgL/Hj0i35aYmMixI4dwdqmFlpaWfLvv8WNM/WMC7Tp0Yvjo1IOFUb+OY9nqdQo/rdu0A2DJijUMzuMzr3r1G3L71k2Cnn/eNkHcvHGdeg3Sb5t6DT61Tco9+4mJiRw9fAjnmoptkxlFi8rOzu7cvqWwPTLyLc8Dn1G8eN6fbdWu14C7d24R/FnfCQ0J5taN69Sum/5tkbXrNSAxMZET/yr2neNHD1PDuaa8fcpXrIyRUVEunle8LHLx/FkAypZPCbru3LrB2JFuVKvuzKRpf2b6rp3cUL+Bct8JDg7ixvVrGfad+g0akZiYwLEjin3nyOGDuNSsLW+bmrXroKmpycED+xTyH9i/FxvbMpiZmcu33bxxnWFuQ3BydmHmn3PytW0ADlx8Qg37kliWTJnVsSiuh0s5Yw5cDEg3b1hEDEX0ClLaWHE2sLqdbAyEvFaeoQh8Gc29j8HLoNaVOHb1mVKQ8qWu9exITpZy5WHe352UkW9/HkMs/EzT59cM9+3bh6+vL3///TdmZmZERETg4uIij3ytrKwoXbo0J06cYP/+/fJZDICSJUsyc+ZMqlVTvl4cFBSU+xX5qH3Hzmzd8g+jPIYy2H0YEiQsW+xFyRIl6dC5izxdaEgw7Vo1o9/AwfQfNBQAO/uyNGnegnmes0hMTMDE1IztW7cQEhzE9Fkpi+2uXbnMhN9GY2NbhtZt23P75g35e5paWtiXLSff35euXr4EyJ6/kde3s3bo2BmfzZsY6TGEIe7DZQ/jWrSQEiVK0rFzV3m6kJBg2rZsSv+BQxgwOKVtmjZvydzZs0hMTPzYNpsJCQ5ixp9zFI7z393bhAQHy+/Zf+LvLw9OatWph7a2NiamZtSpV5/1f69GTU2Nqo7ViXz7lnVrVxEfn0Dnrnn7jAyANu07scPnH8aOdJc9FEsCq5Z6U6JkSdp27CxP9yI0hC5tm9O73yB+GTAEgDJ2ZWnUtAVe82bL22fX9i2EhgQxaXrKc1M0NDQY5D6CGZMn4DlzCvUaNCb4eSArlnjhUK061ao7AfDsSQCjhw3GoLAhP/Xsw/37/ymUtULFynnQIik6dOrCln82Mcx9CG4esnG12HshJUqWpPMXfad18yYMGDSEQUNkl3bsy5alWYuWeM6eSWJiIqZmZmzdspngoCBmzU5ZqGlkZET3nr1ZvXI5OjqFKFuuHEcOH+TSxQss+PiMGpDdyeM2eCCGhob06tOXe//dVShrpcpVcrk1lK05fIdBrSuxbWJrpmy4gFQq5Y/uzgS9eseqQ3fk6SyK6XF3dS9mbr7ErM2yz4IN//6HR3sHdk9pw2yfyzwPf0c1m+KM/V8Nrj4K4/x/IfL8ozs78vxlFCFv3mNeTI9BrSthVkyPhmO2KRxj9eimbDv9EP+QSApoqtPGpTQ9Gpdj1aHbGQYj+UEVZyaySgQZyM4eAwMDqVmzZqrvv3//Hi0tLQwNDYmNjWXBggVKaVxdXdm8eTO3b99m8uTJ8u3/+9//WLBgAbNmzcLMzIw3b95w7do1+cLQvKKto8OyVWuZ5/knk8b/hlQqpbqT7LHiOjqF5OmkUtnMzecPrwGYNHUmS7wXsHSRF9HRUdiWscdr6UrsP3uK5+VLF4mPj+fB/Xv07fmTQn5jExP2HVa91dsga5vlq/9mnucsJo7/VfZYcScXRv+m2DZIpbK2kSreKjd52kwWe81nifdC+WPFFy1bqfSEU59/NrFv727562NHD3PsY5Cx//C/aJuaAfDnnPlsXLeWw4cOsGH9WnQLyZ4SOm79JMqVV7yMkhe0tXXwWrYGr79mM/WPsUilUhyrOzNs9Ngv+o6sfb6cdp4waTrLlyxk5VIv3kVHY2Nrxzzv5diVVXy8c0vXdqipqbFx3WoO7t2Fvr4BTVu2ZpDbcHnQf+f2Tfk6HveByg+6Onf1rtK23KSjo8PKNeuYM3sWE8bK+o6Tswtjxo5Hp1DGbTN1+iy8F85nkdcCed9ZsnyVUt9xHzYCHR0d/tm4XvZYcSsr5sxbQP3PZktu3bxJVFQkUVGR9OujeDcXwM27D5S25baYD4m0GL8Tz/51WT2qKRLg5M3njF5xmvdxCSkJJaChrqZwWSzwZTT1Rm7l95+dmNzDBSN9bYJeRbPm8B1m+1xWWFBaqKAGk3u6YGyky9t3Hzh29Rk/zfx/e/ceFsV1/gH8i8tdoiDRyEUUDWAB0eWuKIlcFATkmsoS0QQMoSqggtGgkoZHbWsTpIAxgBps0JQQCJFGUoygiZJHrY2KRhQKqIBcoqAgwrLL+f1BmR/rgiBhBeH9+Pg8OzNnZt45O7v7zpnDnOOo+vX/H13e/FiIxuY2RPpZYIq6Khjrus0SmXwayd9KthyOFCPxuRfPih4rDuD777/Hzp070dLSguDgYMTHx+PatWvcFfWjR48QGRmJc+fOQUNDA2FhYdi6dStOnjwJXd2uH4aGhga89tprsLKywuHDh7ltd3Z24vDhw/jHP/6BhoYGaGhowNXVFVFRUaiqqoKjo6PEvvoz2MeKjwWDfaz4WDHYx4qPBYN9rPhY8FseKz4WyPKx4rUPOvov9ISpE0fWIxcoyXjBUJLRN0oyno6SjL5RktE3SjKeTqZJxsNBJBkTRlaSQZ8sQgghZAQaDZdNlGQQQgghI9BoaJylJIMQQggZgUZDx09KMgghhJCR6MXPMSjJIIQQQkaiUZBjUJJBCCGEjETUJ4MQQgghMkF9MgghhBAiE6OhJYMGSCOEEEKITFBLBiGEEDICjYaWDEoyCCGEkBGI+mQQQgghRCaoJYMQQgghMjEKcgxKMgghhJARaRRkGZRkEEIIISMQ9ckghBBCiEyMhj4Z9JwMQgghhMgEJRmEEELICCQ3iP/PoqmpCevWrQOfz8frr7+OnJycPsumpaVh4cKFMDc3x5YtW9De3j6gfVCSQQghhIxEMs4yYmNjoaCggDNnzuCjjz5CbGwsbty4IVXuxx9/REpKCj777DMUFhaipqYG8fHxA9oHJRmEEELICCQ3iH8PHz5EVVWV1P+HDx9KbLu1tRX5+fmIiIjA+PHjYWlpCScnJxw7dkwqjpycHPj5+cHAwAATJ07E2rVr8c033wzoGKjj5wvmJSXKC8ngjFekjzt5do+/DR/uEMYsFYVnX+fAp4eRlJQkNX/9+vUICwvjpisrK8Hj8aCvr8/Nmz17Ns6dOye1bmlpKRwdHSXK3bt3D/fv38ekSZOeGg996xBCCCGjxOrVq+Ht7S01f8KECRLTra2tUFNTk5inpqaG1tZWqXWfLNv9urW1lZIMQgghZKyYMGGCVELRG1VVVbS0tEjMa2lpgaqqar9lu1/3VvZJ1PZOCCGEjDEzZsyAWCxGZWUlN6+kpASvvvqqVFkDAwOJDqElJSXQ1NTstxUDoCSDEEIIGXNUVVXh7OyMhIQEtLa24uLFizh58iQ8PDykynp6euKrr75CWVkZHjx4gP3798PT03NA+5FjjLGhDp4QQgghI1tTUxO2bduGs2fPYuLEidi4cSO8vLxQU1MDNzc3fPvtt9DW1gbQ9ZyM1NRUtLa2YsmSJYiNjYWSklK/+6AkgxBCCCEyQbdLCCGEECITlGQQQgghRCYoySCEEEKITFCSMYb9+uuvWLlyJczNzWFqaopt27YNajtr1qzB119/PcTRkRddYmIioqKihjuM5yowMBCZmZm/eTvZ2dkQCARDEBHpVlVVBSMjI4hEouEOZUyhJGMMy8jIgLq6Oi5evIirV69i165d/a7T2w/HgQMHen3CHBleL9KX6tatW7F3797hDqNXDg4OKCoqGu4wRg2qz7GFkowxrKamBrNmzYKc3LMOEEz60tHRMdwhEPLCos/P6ENJxhi1detW5OTk4ODBg+Dz+fDx8ZFoofj5558hEAhgaWmJhQsX4vPPP8cPP/yA5ORk5OXlgc/nY+nSpQAkm4g7Ozuxf/9+ODg4wNbWFlFRUdzof91X1rm5uVi8eDFsbGyQnJw8pMdVXFwMLy8v8Pl8REVFISIiAomJiQCAzMxMLFmyBNbW1nj33XdRW1sLAPjggw+kWnE2bNjADTJUV1eH8PBwzJ8/Hw4ODkhLS+PKJSYmIjw8HO+99x4sLCyQkZGBxMREbNiwAdHR0TA3N4ebmxuuXbvGrePg4ICDBw9i+fLl4PP5iI6ORmNjI9auXQtzc3MEBgbi/v37XPlLly7B398flpaW8PDwwE8//cQtCwwMREJCAnfbKzg4GE1NTQCAlStXAgCsrKzA5/N7HfhoqKSmpsLe3h58Ph9OTk44c+aMVJmNGzfCzs4OFhYWCAwMRHl5OYCuFrXc3FzuXAwKCgLw9Hp/XjZv3oyamhqEhoaCz+dj3759T30/npSVlQVXV1dYWVnh7bffRlVVFbfMyMgI6enpcHJygo2NDXbt2gWxWCyx/scffwxra2s4ODjgxx9/5ObX1dUhNDQUNjY2cHZ2xhdffMEt6+/8G8567a0+jYyMkJWVBQcHBwQEBADo+1y5fPkybG1tJVrnTp8+DScnJwBd3z8pKSlwdnaGjY0NwsPD0djY+NyOj/SCkTFry5YtLC4ujjHGWEJCAouMjGSMMVZTU8P4fD77+uuvmVAoZA8ePGBXrlyRKtdt5cqV7Msvv2SMMZaZmcmcnJzYrVu3WHNzM/vDH/7ANm3axBhj7M6dO8zQ0JDt2LGDtbW1satXrzITExNWUVExJMfT3t7O7O3tWVpaGhMKhez48ePMxMSEJSQksKKiImZtbc2Ki4tZW1sb++Mf/8hWrFjBGGPswoULzM7OjolEIsYYYy0tLWzu3LmssrKSicVi5u3tzf72t7+x9vZ2dvv2bebo6MgKCgq4+jA2NmZ5eXlMLBaztrY2lpCQwExNTdnp06eZSCRif/7zn5lAIODiXLx4MfP19WX19fWsrq6OLViwgHl7e7Pi4mLW3t7O3n77bfbRRx8xxhirra1lVlZWrKCggIlEInb27FlmZWXF6uvrubp3cnJiFRUVrLW1lQkEArZ3716J+u7o6BiS+u1LeXk5s7e3Z7W1tYwxxqqqqlhlZaXUuZKVlcWam5tZW1sb+/DDD5mPjw+3rOe5yBjrt96fp8WLF7OzZ88yxgb2fnR/Fk6cOMEcHR3ZzZs3WUdHB/vkk0+Yr68v6+zsZIwxZmhoyAICAtj9+/fZ3bt3mYuLC0tPT2eMddWVsbExy8zMZCKRiP39739n9vb2XExvvvkmi4mJYW1tbezatWvM2tqanTlzhjHGnnr+jYR67Vmf3efopk2bWEtLC2tra+OOv69zxdnZmZ06dYqbjoqK4s75tLQ05uvry6qrq1l7ezvbsWMHCwsLk9iXrD8PRBK1ZBApubm5sLW1hZeXFxQUFDBhwgTMmTNnwOu+9dZb0NPTg5qaGiIjI5GXlydx5bFu3TooKSnBxMRE6pn4v8WlS5fQ2dmJVatWQUFBAa6urjA2Nubi8vHxgampKZSUlBAVFYXi4mJUVVXBwsICioqK3JX+iRMnYGhoiOnTp+Pq1au4d+8ewsLCoKioiGnTpsHf3x/Hjx/n9mtmZgYXFxeMGzeOewKepaUl7O3twePx4OnpievXr0vEumrVKkyePBlTpkyBlZUVjI2NYWpqCkVFRTg5OXHljx07hkWLFmHx4sXg8XhYsGAB5s2bh1OnTnHb8vHxwYwZM6CiogIXFxeUlJQMSX0OFI/HQ3t7O0pLSyEUCqGjo4Pp06dLlfPx8YGamhqUlJSwfv16XL16tdcRHwEMqN6Hw0Dej24ZGRkICQmBgYEB5OXl8e677+K///0v7ty5w5UJCQmBhoYGpk6dirfeekvi+HR0dODn5wcejwdvb2/U1taisbERd+/excWLF7F582YoKSnB2NgYb7zxBnJzc7l1+zr/Rmq9hoeHY/z48dzn52nniru7O3esbW1tEo/CzsjIwMaNG6GtrQ1FRUVs2LABJ06cgFAoHJ4DIzQKK5FWU1PT64/EQNTX10NHR4eb1tXVhVgsxr1797h5mpqa3GtlZWU8evRo8MH20NDQgFdeeUWij8nUqVO5uExMTLj548ePh7q6Ourq6qCrq8t9cS1YsAD//Oc/uS+tqqoqNDQ0wMrKiltXLBaDz+dz092P3e2p5zGqqKhI/Zi+/PLL3GtlZeU+66S6uhr5+fmwtLTklotEIsybN6/XbamoqAxZfQ6Unp4etm3bhqSkJNy8eRMLFy5EdHS0RBmxWIyPP/4Y//rXv9DY2Ihx47qubxobG3sdyXEg9T4cBvJ+9Cz7pz/9CXv27OHmicVi1NXVQU9PD4DkuaOtrY2GhgZu+slzAgAePXqEe/fuYeLEiRJDb+vo6EjcEunr/Bup9dqzHvo7Vzw8PODj44PHjx+jsLAQM2bMwKxZswB01XlYWBi3DgDIy8tL1Ct5vijJIFK0tLRw+fLlXpf110l0ypQpqK6u5qZramrA4/GgqanJ9YGQlcmTJ6Ourg6MMS7O2tpaGBgYSMXV2tqKpqYmvPLKKwAADw8PCAQCRERE4Pz58/jLX/4CoKsudHV1kZ+f3+d+ZdlxVktLC56enti5c+czr/s8O/R6eHjAw8MDzc3N+PDDD7Fnzx7o6+tzy3Nzc1FYWIi0tDTo6uqisbER8+fPB/vfqAZPxjqQeh8Oz/J+aGlpITQ0FMuXL++zTE1NDQwMDAAAd+/exeTJk/vd7pQpU/DgwQO0tLRwicbdu3e5c7m/mEZivfZ8//s7V/T19TFz5kwUFBRIXBAAXRcVu3fvhoWFhdQ+evaHIc8P3S4hUro7s+Xm5qKjowMPHz7E1atXAXRdIVVXV6Ozs7PXdd3d3XH48GHcuXMHjx49wt69e+Hi4gJ5ednns/PmzYOcnBzS09MhEomQn5+PX375hTum7OxsXL9+HUKhEHFxcZgzZw50dXUBdA1lrKOjg/fffx/W1tbclaCZmRnU1NSQkpKCtrY2iMVilJWV4cqVKzI/HgBYvnw5CgsL8cMPP0AsFkMoFOLChQuoqanpd91JkyZh3LhxuH37tkxjLC8vx08//QShUAglJSUoKytLXEkCXVfgioqK0NDQwOPHjxEfHy+xXFNTU+JHYLjrvaeXX36Zq8NneT/8/f2RkpKC0tJSAEBzczPy8vIkyhw4cABNTU2oq6tDWloaXF1d+41HS0sLfD4fcXFxaG9vR0lJCTIzM+Hm5tbvuiOhXnvWZ2/6O1eArs/zF198gaKiIixbtoybLxAIEB8fz51L9+/fx/fffz/0B0EGjJIMIkVbWxupqalIT0+Hra0t3NzcuJYNFxcXAICNjU2vX2q+vr5Yvnw5AgMD4ejoCAUFBcTExDyXuBUVFZGUlITMzExYWVnhu+++g729PRQVFTF//nxEREQgPDwcdnZ2uHXrFuLi4iTW9/DwQFFREdzd3bl5PB4P+/fvR0lJCRwdHWFra4vo6GjuL2ZkTUtLC5988glSU1Mxf/582NvbIyUlpc8krycVFRWEhoYiICAAlpaWOH/+vExi7E7abGxsYGdnh/r6emzevFmijJeXF7S0tLBo0SK4u7tLNc/7+fmhrKwMlpaWWLNmzbDXe08hISH49NNPYWlpiZycnAG/H87OzlizZg02bdoEc3NzuLu7o7CwUKLM0qVL4efnBw8PDyxYsAD+/v4DiikuLg7V1dVYtGgR1q9fj/Xr12PRokX9rjcS6rVnffbsR9Ktv3MFANzc3PCf//wH8+bNk2jBWbVqFRwcHBAcHAxzc3O88cYbuHTpkkyPhzwdjcJKRjVfX18EBgbCy8truEMhRIKRkRHy8/MH3f+JkBcBtWSQUeXcuXOoq6uDSCRCdnY2ysvLYW9vP9xhEULImEQdP8mocuvWLURGRuLRo0eYNm0aEhISMGnSpOEOixBCxiS6XUIIIYQQmaDbJYQQQgiRCUoyCCGEECITlGQQQgghRCYoySCEAOj6yxwjIyNuOjExEYGBgcMaw1BzcHBAdnb2oNfvHkmYnh5JyMBQkkHICBcYGAgjIyMYGRmBz+fDz89PYthvWQkKCkJiYuKAyj6vhETWSQghZGhRkkHICyAoKAhnzpxBdnY2jI2NsXbtWty6davXskM14mT3IHKEEDJYlGQQ8gJQVVXF5MmToa+vj5iYGPB4PBQVFQHoaun461//ivfffx98Ph8JCQkAgGvXriEwMBBmZmZwcHBAUlISxGIxt80bN27Ax8cHc+bMgUAgkBiCHJBunRCJRNi7dy/s7e1hZmbGjXGTnZ2NpKQknD9/nmtx6b6d8FtjeFa7du2Co6Mj5s6dCzc3t16HMG9sbMQ777wDMzMzuLq64t///rfE8qKiIvj4+MDMzAxLly7F0aNH+9zfnTt3uEdYdz/Guq/kj5CxiB7GRcgLRl5eHvLy8ujo6ODmHT16FOvWrcM333wDHo+HxsZGBAUFISQkBDt37kRtbS127NgBVVVVBAUFQSwWIywsDLNnz8aePXtQWlqKXbt2PXW/iYmJyMnJQUxMDAwNDVFWVoZx48Zh2bJlKC0txc8//8zdXpk0aZJMYuiPuro69u7dCw0NDRQVFeG9997DrFmzJG6xJCcnIzIyElu3bsXRo0exdu1aFBQUQE1NDeXl5QgLC0N0dDSsra1RVlaG6OhoaGpqYunSpVL7i42NhYaGBr766ivIycmhuLhYanA4QsYySjIIeYF0dHTgs88+Q0tLCywtLbn58+bNw5o1a7jppKQkLFiwAMHBwQCA6dOnIywsDPv27eNuvdTX1yMrKwsvvfQSXn31VVy/fh3Jycm97retrQ2HDh1CfHw8HB0dAQB6enrcclVVVSgoKEgMVX7kyJEhjWEg1q1bx71esWIFCgoKcOLECYkkw97eHitWrAAAREdHo6CgALm5uRAIBEhNTYVAIICvry8AYNq0aVi9ejW+/PLLXpOM2tpauLu7Y+bMmQAgMbw9IYSSDEJeCMnJyTh06BDa29uhpqaGmJgYGBsbc8t/97vfSZS/efMmCgoKJEawFIvF3GihFRUV0NfXx0svvcQtnzt3bp/7v3XrFoRCIaysrAYc81DHMBA5OTn4/PPPUVVVBaFQCKFQKJH4AF3DnXfj8XgwMTFBRUUFF/PNmzdx5MgRroxIJIK2tnav+xMIBNi2bRuOHTsGOzs7LFu2DFOnTv1Nx0DIaEJJBiEvAH9/fwQGBnJ9M56koqIiMd3a2goPDw+Ehob2uU05OTmJ6aeNMDCY0QeGOob+XLx4Edu3b8eWLVtgYWGB8ePHY/fu3RCJRE/d55MxBwcHw9vbW2K+vHzvX5UBAQFYtGgRCgoKUFhYiMTERBw8eBDm5uaDPg5CRhNKMgh5AUycOPGZhgSfPXs2zp8/3+c6+vr6qKioQEtLC9TU1AAAxcXFfW5v+vTpUFRUxIULF7jbJT3Jy8tLdOiURQz9uXTpEgwNDbnOqowx3L59GxoaGhLlrly5wr3u7OzEL7/8Ajs7Oy7mysrKZ6rr7lsqq1evxjvvvIPjx49TkkHI/1CSQcgo9OabbyIjIwMxMTEICAiAoqIiSkpKcPv2bYSGhmLhwoXQ1NTE9u3bERYWhtLSUmRlZfW5PRUVFaxevRqxsbFgjMHIyAjl5eVQVlaGjY0NtLW1UVlZifLycqirq0NdXX3IY+jp+vXrEtNqamrQ09NDWVkZTp06BT09PRw5cgR1dXVS654+fRqZmZmwsLDA0aNH0dzcDA8PDwBAcHAwBAIBEhIS4Obmhs7OTly+fBkdHR0QCARS29q9ezdef/116Onpoba2Fjdu3MBrr702oGMgZCygJIOQUUhLSwvp6enYs2cPBAIB5OTkMHPmTO4qn8fjISkpCdHR0fD09ISpqSkiIiKwffv2PrcZEREBxhg++OADNDc3Y/r06Vz5JUuWIC8vD76+vmhtbcXJkyehq6s75DF08/Lykph2dHTEvn378Pvf/x6bN2/GuHHj4OfnB2dnZ6l1Q0JC8N133yE2NhY6OjpISkriWlJMTU1x6NAhxMXF4cCBA1BWVoahoSFCQkJ6jUMkEmHHjh2or6+HhoYG3N3de01GCBmraKh3QgghhMgE/UE3IYQQQmSCkgxCCCGEyAQlGYQQQgiRCUoyCCGEECITlGQQQgghRCYoySCEEEKITFCSQQghhBCZoCSDEEIIITJBSQYhhBBCZOL/AHkmmWdpS+CwAAAAAElFTkSuQmCC\n", - "text/plain": [ - "
" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "plot_confusion_matrix(labels_test,preds,label_encoder.classes_, normalize=True)" - ] - }, - { - "cell_type": "code", - "execution_count": 27, - "metadata": {}, - "outputs": [], - "source": [ - "mlflow.end_run()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## References\n", - "---------------\n", - "\n", - "1. Yang, Zhilin, Zihang Dai, Yiming Yang, Jaime Carbonell, Ruslan Salakhutdinov, and Quoc V. Le. [*XLNet: Generalized Autoregressive Pretraining for Language Understanding.*](https://arxiv.org/abs/1906.08237), 2019.\n", - "2. Devlin, Jacob and Chang, Ming-Wei and Lee, Kenton and Toutanova, Kristina, [*BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding*](https://arxiv.org/abs/1810.04805), ACL, 2018.\n", - "3. Dai, Zihang, Zhilin Yang, Yiming Yang, William W. Cohen, Jaime Carbonell, Quoc V. Le, and Ruslan Salakhutdinov. [*Transformer-xl: Attentive language models beyond a fixed-length context.*](https://arxiv.org/pdf/1901.02860), 2019.\n", - "4. Adina Williams, Nikita Nangia, Samuel R. Bowman. [*A Broad-Coverage Challenge Corpus for Sentence Understanding through Inference*](https://www.nyu.edu/projects/bowman/multinli/paper.pdf), 2016. Dataset available at (https://www.nyu.edu/projects/bowman/multinli/).\n", - "5. Transformers: a library of state-of-the-art pre-trained models for Natural Language Processing (NLP). Repository available at (https://github.com/huggingface/transformers)." - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python (nlp_gpu)", - "language": "python", - "name": "nlp_gpu" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.6.8" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/tests/conftest.py b/tests/conftest.py index 5db8c5e5b..b5c84ca97 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -76,12 +76,6 @@ def notebooks(): "tc_mnli_transformers": os.path.join( folder_notebooks, "text_classification", "tc_mnli_transformers.ipynb" ), - "tc_dac_bert_ar": os.path.join( - folder_notebooks, "text_classification", "tc_dac_bert_ar.ipynb" - ), - "tc_bbc_bert_hi": os.path.join( - folder_notebooks, "text_classification", "tc_bbc_bert_hi.ipynb" - ), "tc_multi_languages_transformers": os.path.join( folder_notebooks, "text_classification", "tc_multi_languages_transformers.ipynb" ), diff --git a/tests/unit/test_xlnet_common.py b/tests/unit/test_xlnet_common.py deleted file mode 100644 index 6a34f8872..000000000 --- a/tests/unit/test_xlnet_common.py +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. - -import pytest - -def test_preprocess_classification_tokens(xlnet_english_tokenizer): - text = ["Hello World.", - "How you doing?", - "greatttt", - "The quick, brown fox jumps over a lazy dog.", - " DJs flock by when MTV ax quiz prog", - "Quick wafting zephyrs vex bold Jim", - "Quick, Baz, get my woven flax jodhpurs!" - ] - seq_length = 5 - input_ids, input_mask, segment_ids = xlnet_english_tokenizer.preprocess_classification_tokens(text, seq_length) - - assert len(input_ids) == len(text) - assert len(input_mask) == len(text) - assert len(segment_ids) == len(text) - - - for sentence in range(len(text)): - assert len(input_ids[sentence]) == seq_length - assert len(input_mask[sentence]) == seq_length - assert len(segment_ids[sentence]) == seq_length - \ No newline at end of file diff --git a/tests/unit/test_xlnet_sequence_classification.py b/tests/unit/test_xlnet_sequence_classification.py deleted file mode 100644 index 12a5a6922..000000000 --- a/tests/unit/test_xlnet_sequence_classification.py +++ /dev/null @@ -1,44 +0,0 @@ -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. - -import pytest - -from utils_nlp.models.xlnet.common import Language -from utils_nlp.models.xlnet.sequence_classification import XLNetSequenceClassifier - - -@pytest.fixture() -def data(): - return ( - ["hi", "hello", "what's wrong with us", "can I leave?"], - [0, 0, 1, 2], - ["hey", "i will", "be working from", "home today"], - [2, 1, 1, 0], - ) - - -def test_classifier(xlnet_english_tokenizer, data): - token_ids, input_mask, segment_ids = xlnet_english_tokenizer.preprocess_classification_tokens( - data[0], max_seq_length=10 - ) - - val_data = xlnet_english_tokenizer.preprocess_classification_tokens(data[2], max_seq_length=10) - - val_token_ids, val_input_mask, val_segment_ids = val_data - - classifier = XLNetSequenceClassifier(language=Language.ENGLISHCASED, num_labels=3) - classifier.fit( - token_ids=token_ids, - input_mask=input_mask, - token_type_ids=segment_ids, - labels=data[1], - val_token_ids=val_token_ids, - val_input_mask=val_input_mask, - val_labels=data[3], - val_token_type_ids=val_segment_ids, - ) - - preds = classifier.predict( - token_ids=token_ids, input_mask=input_mask, token_type_ids=segment_ids - ) - assert len(preds) == len(data[1]) From b0dc696acc6d369422f0605730be0341c840a933 Mon Sep 17 00:00:00 2001 From: saidbleik Date: Tue, 26 Nov 2019 21:10:01 +0000 Subject: [PATCH 08/18] remove obsolete tests and links --- README.md | 2 + examples/text_classification/README.md | 3 -- .../test_notebooks_text_classification.py | 50 +------------------ 3 files changed, 4 insertions(+), 51 deletions(-) diff --git a/README.md b/README.md index 39c90ccf9..f0d9426f8 100755 --- a/README.md +++ b/README.md @@ -85,6 +85,8 @@ The following is a list of related repositories that we like and think are usefu |[AzureML-BERT](https://github.com/Microsoft/AzureML-BERT)|End-to-end recipes for pre-training and fine-tuning BERT using Azure Machine Learning service.| |[MASS](https://github.com/microsoft/MASS)|MASS: Masked Sequence to Sequence Pre-training for Language Generation.| |[MT-DNN](https://github.com/namisan/mt-dnn)|Multi-Task Deep Neural Networks for Natural Language Understanding.| +|[UniLM](https://github.com/microsoft/unilm)|Unified Language Model Pre-training.| + ## Build Status diff --git a/examples/text_classification/README.md b/examples/text_classification/README.md index e5071aab2..0ba711dcb 100644 --- a/examples/text_classification/README.md +++ b/examples/text_classification/README.md @@ -19,8 +19,5 @@ The following summarizes each notebook for Text Classification. Each notebook pr |Notebook|Environment|Description|Dataset| |---|---|---|---| |[BERT for text classification on AzureML](tc_bert_azureml.ipynb) |Azure ML|A notebook which walks through fine-tuning and evaluating pre-trained BERT model on a distributed setup with AzureML. |[MultiNLI](https://www.nyu.edu/projects/bowman/multinli/)| -|[XLNet for text classification with MNLI](tc_mnli_xlnet.ipynb)|Local| A notebook which walks through fine-tuning and evaluating a pre-trained XLNet model on a subset of the MultiNLI dataset|[MultiNLI](https://www.nyu.edu/projects/bowman/multinli/)| -|[BERT for text classification of Hindi BBC News](tc_bbc_bert_hi.ipynb)|Local| A notebook which walks through fine-tuning and evaluating a pre-trained BERT model on Hindi BBC news data|[BBC Hindi News](https://github.com/NirantK/hindi2vec/releases/tag/bbc-hindi-v0.1)| -|[BERT for text classification of Arabic News](tc_dac_bert_ar.ipynb)|Local| A notebook which walks through fine-tuning and evaluating a pre-trained BERT model on Arabic news articles|[DAC](https://data.mendeley.com/datasets/v524p5dhpj/2)| |[Text Classification of MultiNLI Sentences using Multiple Transformer Models](tc_mnli_transformers.ipynb)|Local| A notebook which walks through fine-tuning and evaluating a number of pre-trained transformer models|[MultiNLI](https://www.nyu.edu/projects/bowman/multinli/)| |[Text Classification of Multi Language Datasets using Transformer Model](tc_multi_languages_transformers.ipynb)|Local|A notebook which walks through fine-tuning and evaluating a pre-trained transformer model for multiple datasets in different language|[MultiNLI](https://www.nyu.edu/projects/bowman/multinli/)
[BBC Hindi News](https://github.com/NirantK/hindi2vec/releases/tag/bbc-hindi-v0.1)
[DAC](https://data.mendeley.com/datasets/v524p5dhpj/2) diff --git a/tests/integration/test_notebooks_text_classification.py b/tests/integration/test_notebooks_text_classification.py index 6bab6923f..8f00107eb 100644 --- a/tests/integration/test_notebooks_text_classification.py +++ b/tests/integration/test_notebooks_text_classification.py @@ -37,50 +37,6 @@ def test_tc_mnli_transformers(notebooks, tmp): assert pytest.approx(result["f1"], 0.89, abs=ABS_TOL) -@pytest.mark.gpu -@pytest.mark.integration -def test_tc_dac_bert_ar(notebooks, tmp): - notebook_path = notebooks["tc_dac_bert_ar"] - pm.execute_notebook( - notebook_path, - OUTPUT_NOTEBOOK, - kernel_name=KERNEL_NAME, - parameters=dict( - NUM_GPUS=1, - DATA_FOLDER=tmp, - BERT_CACHE_DIR=tmp, - MAX_LEN=175, - BATCH_SIZE=16, - NUM_EPOCHS=1, - TRAIN_SIZE=0.8, - NUM_ROWS=8000, - RANDOM_STATE=0, - ), - ) - result = sb.read_notebook(OUTPUT_NOTEBOOK).scraps.data_dict - assert pytest.approx(result["accuracy"], 0.871, abs=ABS_TOL) - assert pytest.approx(result["precision"], 0.865, abs=ABS_TOL) - assert pytest.approx(result["recall"], 0.852, abs=ABS_TOL) - assert pytest.approx(result["f1"], 0.845, abs=ABS_TOL) - - -@pytest.mark.gpu -@pytest.mark.integration -def test_tc_bbc_bert_hi(notebooks, tmp): - notebook_path = notebooks["tc_bbc_bert_hi"] - pm.execute_notebook( - notebook_path, - OUTPUT_NOTEBOOK, - kernel_name=KERNEL_NAME, - parameters=dict(NUM_GPUS=1, DATA_FOLDER=tmp, BERT_CACHE_DIR=tmp, NUM_EPOCHS=1), - ) - result = sb.read_notebook(OUTPUT_NOTEBOOK).scraps.data_dict - assert pytest.approx(result["accuracy"], 0.71, abs=ABS_TOL) - assert pytest.approx(result["precision"], 0.25, abs=ABS_TOL) - assert pytest.approx(result["recall"], 0.28, abs=ABS_TOL) - assert pytest.approx(result["f1"], 0.26, abs=ABS_TOL) - - @pytest.mark.integration @pytest.mark.azureml @pytest.mark.gpu @@ -118,6 +74,7 @@ def test_tc_bert_azureml( if os.path.exists("outputs"): shutil.rmtree("outputs") + @pytest.mark.gpu @pytest.mark.integration def test_multi_languages_transformer(notebooks, tmp): @@ -126,10 +83,7 @@ def test_multi_languages_transformer(notebooks, tmp): notebook_path, OUTPUT_NOTEBOOK, kernel_name=KERNEL_NAME, - parameters={ - "QUICK_RUN": True, - "USE_DATASET": "dac" - }, + parameters={"QUICK_RUN": True, "USE_DATASET": "dac"}, ) result = sb.read_notebook(OUTPUT_NOTEBOOK).scraps.data_dict assert pytest.approx(result["precision"], 0.94, abs=ABS_TOL) From 0b4b25638e3c839809f09c0d1e0e1d7ccfe163c9 Mon Sep 17 00:00:00 2001 From: hlums Date: Tue, 26 Nov 2019 22:44:21 +0000 Subject: [PATCH 09/18] Add missing tmp directories. --- tests/unit/test_bert_sentence_encoding.py | 3 +- ..._models_transformers_question_answering.py | 64 +++++++++++++++---- 2 files changed, 52 insertions(+), 15 deletions(-) diff --git a/tests/unit/test_bert_sentence_encoding.py b/tests/unit/test_bert_sentence_encoding.py index 1443d63a7..c19b1713f 100644 --- a/tests/unit/test_bert_sentence_encoding.py +++ b/tests/unit/test_bert_sentence_encoding.py @@ -19,7 +19,7 @@ def data(): @pytest.mark.cpu -def test_sentence_encoding(data): +def test_sentence_encoding(tmp, data): se = BERTSentenceEncoder( language=Language.ENGLISH, num_gpus=0, @@ -27,6 +27,7 @@ def test_sentence_encoding(data): max_len=128, layer_index=-2, pooling_strategy=PoolingStrategy.MEAN, + cache_dir=tmp, ) result = se.encode(data, as_numpy=False) diff --git a/tests/unit/test_models_transformers_question_answering.py b/tests/unit/test_models_transformers_question_answering.py index daa0c76fd..8a61b0910 100644 --- a/tests/unit/test_models_transformers_question_answering.py +++ b/tests/unit/test_models_transformers_question_answering.py @@ -12,12 +12,22 @@ ) import torch +from tempfile import TemporaryDirectory NUM_GPUS = max(1, torch.cuda.device_count()) BATCH_SIZE = 8 -@pytest.fixture() +@pytest.fixture(scope="module") +def tmp(tmp_path_factory): + td = TemporaryDirectory(dir=tmp_path_factory.getbasetemp()) + try: + yield td.name + finally: + td.cleanup() + + +@pytest.fixture(scope="module") def qa_test_data(qa_test_df, tmp): train_dataset = QADataset( @@ -63,7 +73,7 @@ def qa_test_data(qa_test_df, tmp): qa_id_col=qa_test_df["qa_id_col"], ) - qa_processor_bert = QAProcessor() + qa_processor_bert = QAProcessor(cache_dir=tmp) train_features_bert = qa_processor_bert.preprocess( train_dataset, batch_size=BATCH_SIZE, @@ -86,7 +96,7 @@ def qa_test_data(qa_test_df, tmp): feature_cache_dir=tmp, ) - qa_processor_xlnet = QAProcessor(model_name="xlnet-base-cased") + qa_processor_xlnet = QAProcessor(model_name="xlnet-base-cased", cache_dir=tmp) train_features_xlnet = qa_processor_xlnet.preprocess( train_dataset, batch_size=BATCH_SIZE, @@ -109,7 +119,7 @@ def qa_test_data(qa_test_df, tmp): feature_cache_dir=tmp, ) - qa_processor_distilbert = QAProcessor(model_name="distilbert-base-uncased") + qa_processor_distilbert = QAProcessor(model_name="distilbert-base-uncased", cache_dir=tmp) train_features_distilbert = qa_processor_distilbert.preprocess( train_dataset, batch_size=BATCH_SIZE, @@ -149,26 +159,40 @@ def qa_test_data(qa_test_df, tmp): def test_QAProcessor(qa_test_data, tmp): for model_name in ["bert-base-cased", "xlnet-base-cased", "distilbert-base-uncased"]: - qa_processor = QAProcessor(model_name=model_name) - qa_processor.preprocess(qa_test_data["train_dataset"], is_training=True) - qa_processor.preprocess(qa_test_data["train_dataset_list"], is_training=True) - qa_processor.preprocess(qa_test_data["test_dataset"], is_training=False) + qa_processor = QAProcessor(model_name=model_name, cache_dir=tmp) + qa_processor.preprocess( + qa_test_data["train_dataset"], is_training=True, feature_cache_dir=tmp + ) + qa_processor.preprocess( + qa_test_data["train_dataset_list"], is_training=True, feature_cache_dir=tmp + ) + qa_processor.preprocess( + qa_test_data["test_dataset"], is_training=False, feature_cache_dir=tmp + ) # test unsupported model type with pytest.raises(ValueError): - qa_processor = QAProcessor(model_name="abc") + qa_processor = QAProcessor(model_name="abc", cache_dir=tmp) # test training data has no ground truth exception with pytest.raises(Exception): - qa_processor.preprocess(qa_test_data["test_dataset"], is_training=True) + qa_processor.preprocess( + qa_test_data["test_dataset"], is_training=True, feature_cache_dir=tmp + ) # test when answer start is a list, but answer text is not with pytest.raises(Exception): - qa_processor.preprocess(qa_test_data["train_dataset_start_text_mismatch"], is_training=True) + qa_processor.preprocess( + qa_test_data["train_dataset_start_text_mismatch"], + is_training=True, + feature_cache_dir=tmp, + ) # test when training data has multiple answers with pytest.raises(Exception): - qa_processor.preprocess(qa_test_data["train_dataset_multi_answers"], is_training=True) + qa_processor.preprocess( + qa_test_data["train_dataset_multi_answers"], is_training=True, feature_cache_dir=tmp + ) def test_AnswerExtractor(qa_test_data, tmp): @@ -194,7 +218,7 @@ def test_AnswerExtractor(qa_test_data, tmp): def test_postprocess_bert_answer(qa_test_data, tmp): - qa_processor = QAProcessor() + qa_processor = QAProcessor(cache_dir=tmp) test_features = qa_processor.preprocess( qa_test_data["test_dataset"], is_training=False, @@ -210,6 +234,9 @@ def test_postprocess_bert_answer(qa_test_data, tmp): results=predictions, examples_file=os.path.join(tmp, CACHED_EXAMPLES_TEST_FILE), features_file=os.path.join(tmp, CACHED_FEATURES_TEST_FILE), + output_prediction_file=os.path.join(tmp, "qa_predictions.json"), + output_nbest_file=os.path.join(tmp, "nbest_predictions.json"), + output_null_log_odds_file=os.path.join(tmp, "null_odds.json"), ) qa_processor.postprocess( @@ -218,11 +245,14 @@ def test_postprocess_bert_answer(qa_test_data, tmp): features_file=os.path.join(tmp, CACHED_FEATURES_TEST_FILE), unanswerable_exists=True, verbose_logging=True, + output_prediction_file=os.path.join(tmp, "qa_predictions.json"), + output_nbest_file=os.path.join(tmp, "nbest_predictions.json"), + output_null_log_odds_file=os.path.join(tmp, "null_odds.json"), ) def test_postprocess_xlnet_answer(qa_test_data, tmp): - qa_processor = QAProcessor(model_name="xlnet-base-cased") + qa_processor = QAProcessor(model_name="xlnet-base-cased", cache_dir=tmp) test_features = qa_processor.preprocess( qa_test_data["test_dataset"], is_training=False, @@ -238,6 +268,9 @@ def test_postprocess_xlnet_answer(qa_test_data, tmp): results=predictions, examples_file=os.path.join(tmp, CACHED_EXAMPLES_TEST_FILE), features_file=os.path.join(tmp, CACHED_FEATURES_TEST_FILE), + output_prediction_file=os.path.join(tmp, "qa_predictions.json"), + output_nbest_file=os.path.join(tmp, "nbest_predictions.json"), + output_null_log_odds_file=os.path.join(tmp, "null_odds.json"), ) qa_processor.postprocess( @@ -246,4 +279,7 @@ def test_postprocess_xlnet_answer(qa_test_data, tmp): features_file=os.path.join(tmp, CACHED_FEATURES_TEST_FILE), unanswerable_exists=True, verbose_logging=True, + output_prediction_file=os.path.join(tmp, "qa_predictions.json"), + output_nbest_file=os.path.join(tmp, "nbest_predictions.json"), + output_null_log_odds_file=os.path.join(tmp, "null_odds.json"), ) From a39143f7817c6ad74539bee9730f29c29aadd193 Mon Sep 17 00:00:00 2001 From: Daisy Deng Date: Wed, 27 Nov 2019 03:16:52 +0000 Subject: [PATCH 10/18] fix import error and max_nodes for the cluster --- examples/entailment/entailment_xnli_bert_azureml.ipynb | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/examples/entailment/entailment_xnli_bert_azureml.ipynb b/examples/entailment/entailment_xnli_bert_azureml.ipynb index 138e10600..243d20cf7 100644 --- a/examples/entailment/entailment_xnli_bert_azureml.ipynb +++ b/examples/entailment/entailment_xnli_bert_azureml.ipynb @@ -45,7 +45,7 @@ "from azureml.core.runconfig import MpiConfiguration\n", "from azureml.core import Experiment\n", "from azureml.widgets import RunDetails\n", - "from azureml.core.compute import ComputeTarget\n", + "from azureml.core.compute import ComputeTarget, AmlCompute\n", "from azureml.exceptions import ComputeTargetException\n", "from utils_nlp.azureml.azureml_utils import get_or_create_workspace, get_output_files" ] @@ -169,7 +169,7 @@ "except ComputeTargetException:\n", " print(\"Creating new compute target: {}\".format(cluster_name))\n", " compute_config = AmlCompute.provisioning_configuration(\n", - " vm_size=\"STANDARD_NC6\", max_nodes=1\n", + " vm_size=\"STANDARD_NC6\", max_nodes=NODE_COUNT\n", " )\n", " compute_target = ComputeTarget.create(ws, cluster_name, compute_config)\n", " compute_target.wait_for_completion(show_output=True)\n", @@ -524,9 +524,9 @@ "metadata": { "celltoolbar": "Tags", "kernelspec": { - "display_name": "Python 3", + "display_name": "Python (nlp_gpu_transformer_bug_bash)", "language": "python", - "name": "python3" + "name": "nlp_gpu_transformer_bug_bash" }, "language_info": { "codemirror_mode": { From d13cce1e374dc3df48be7a1b66b9b9bb9e19f48d Mon Sep 17 00:00:00 2001 From: hlums Date: Wed, 27 Nov 2019 19:46:42 +0000 Subject: [PATCH 11/18] Minor edits. --- tests/conftest.py | 9 ++ tests/unit/test_bert_sentence_encoding.py | 1 - ..._models_transformers_question_answering.py | 128 +++++++++--------- 3 files changed, 74 insertions(+), 64 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index b5c84ca97..c1428c41b 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -98,6 +98,15 @@ def tmp(tmp_path_factory): td.cleanup() +@pytest.fixture(scope="module") +def tmp_module(tmp_path_factory): + td = TemporaryDirectory(dir=tmp_path_factory.getbasetemp()) + try: + yield td.name + finally: + td.cleanup() + + @pytest.fixture(scope="module") def ner_test_data(): UNIQUE_LABELS = ["O", "I-LOC", "I-MISC", "I-PER", "I-ORG", "X"] diff --git a/tests/unit/test_bert_sentence_encoding.py b/tests/unit/test_bert_sentence_encoding.py index c19b1713f..717fda735 100644 --- a/tests/unit/test_bert_sentence_encoding.py +++ b/tests/unit/test_bert_sentence_encoding.py @@ -18,7 +18,6 @@ def data(): ] -@pytest.mark.cpu def test_sentence_encoding(tmp, data): se = BERTSentenceEncoder( language=Language.ENGLISH, diff --git a/tests/unit/test_models_transformers_question_answering.py b/tests/unit/test_models_transformers_question_answering.py index 8a61b0910..ca7a5a69d 100644 --- a/tests/unit/test_models_transformers_question_answering.py +++ b/tests/unit/test_models_transformers_question_answering.py @@ -12,23 +12,13 @@ ) import torch -from tempfile import TemporaryDirectory NUM_GPUS = max(1, torch.cuda.device_count()) BATCH_SIZE = 8 @pytest.fixture(scope="module") -def tmp(tmp_path_factory): - td = TemporaryDirectory(dir=tmp_path_factory.getbasetemp()) - try: - yield td.name - finally: - td.cleanup() - - -@pytest.fixture(scope="module") -def qa_test_data(qa_test_df, tmp): +def qa_test_data(qa_test_df, tmp_module): train_dataset = QADataset( df=qa_test_df["test_df"], @@ -73,7 +63,7 @@ def qa_test_data(qa_test_df, tmp): qa_id_col=qa_test_df["qa_id_col"], ) - qa_processor_bert = QAProcessor(cache_dir=tmp) + qa_processor_bert = QAProcessor(cache_dir=tmp_module) train_features_bert = qa_processor_bert.preprocess( train_dataset, batch_size=BATCH_SIZE, @@ -82,7 +72,7 @@ def qa_test_data(qa_test_df, tmp): max_question_length=16, max_seq_length=64, doc_stride=32, - feature_cache_dir=tmp, + feature_cache_dir=tmp_module, ) test_features_bert = qa_processor_bert.preprocess( @@ -93,10 +83,10 @@ def qa_test_data(qa_test_df, tmp): max_question_length=16, max_seq_length=64, doc_stride=32, - feature_cache_dir=tmp, + feature_cache_dir=tmp_module, ) - qa_processor_xlnet = QAProcessor(model_name="xlnet-base-cased", cache_dir=tmp) + qa_processor_xlnet = QAProcessor(model_name="xlnet-base-cased", cache_dir=tmp_module) train_features_xlnet = qa_processor_xlnet.preprocess( train_dataset, batch_size=BATCH_SIZE, @@ -105,7 +95,7 @@ def qa_test_data(qa_test_df, tmp): max_question_length=16, max_seq_length=64, doc_stride=32, - feature_cache_dir=tmp, + feature_cache_dir=tmp_module, ) test_features_xlnet = qa_processor_xlnet.preprocess( @@ -116,10 +106,12 @@ def qa_test_data(qa_test_df, tmp): max_question_length=16, max_seq_length=64, doc_stride=32, - feature_cache_dir=tmp, + feature_cache_dir=tmp_module, ) - qa_processor_distilbert = QAProcessor(model_name="distilbert-base-uncased", cache_dir=tmp) + qa_processor_distilbert = QAProcessor( + model_name="distilbert-base-uncased", cache_dir=tmp_module + ) train_features_distilbert = qa_processor_distilbert.preprocess( train_dataset, batch_size=BATCH_SIZE, @@ -128,7 +120,7 @@ def qa_test_data(qa_test_df, tmp): max_question_length=16, max_seq_length=64, doc_stride=32, - feature_cache_dir=tmp, + feature_cache_dir=tmp_module, ) test_features_distilbert = qa_processor_distilbert.preprocess( @@ -139,7 +131,7 @@ def qa_test_data(qa_test_df, tmp): max_question_length=16, max_seq_length=64, doc_stride=32, - feature_cache_dir=tmp, + feature_cache_dir=tmp_module, ) return { @@ -157,27 +149,28 @@ def qa_test_data(qa_test_df, tmp): } -def test_QAProcessor(qa_test_data, tmp): +@pytest.mark.gpu +def test_QAProcessor(qa_test_data, tmp_module): for model_name in ["bert-base-cased", "xlnet-base-cased", "distilbert-base-uncased"]: - qa_processor = QAProcessor(model_name=model_name, cache_dir=tmp) + qa_processor = QAProcessor(model_name=model_name, cache_dir=tmp_module) qa_processor.preprocess( - qa_test_data["train_dataset"], is_training=True, feature_cache_dir=tmp + qa_test_data["train_dataset"], is_training=True, feature_cache_dir=tmp_module ) qa_processor.preprocess( - qa_test_data["train_dataset_list"], is_training=True, feature_cache_dir=tmp + qa_test_data["train_dataset_list"], is_training=True, feature_cache_dir=tmp_module ) qa_processor.preprocess( - qa_test_data["test_dataset"], is_training=False, feature_cache_dir=tmp + qa_test_data["test_dataset"], is_training=False, feature_cache_dir=tmp_module ) # test unsupported model type with pytest.raises(ValueError): - qa_processor = QAProcessor(model_name="abc", cache_dir=tmp) + qa_processor = QAProcessor(model_name="abc", cache_dir=tmp_module) # test training data has no ground truth exception with pytest.raises(Exception): qa_processor.preprocess( - qa_test_data["test_dataset"], is_training=True, feature_cache_dir=tmp + qa_test_data["test_dataset"], is_training=True, feature_cache_dir=tmp_module ) # test when answer start is a list, but answer text is not @@ -185,101 +178,110 @@ def test_QAProcessor(qa_test_data, tmp): qa_processor.preprocess( qa_test_data["train_dataset_start_text_mismatch"], is_training=True, - feature_cache_dir=tmp, + feature_cache_dir=tmp_module, ) # test when training data has multiple answers with pytest.raises(Exception): qa_processor.preprocess( - qa_test_data["train_dataset_multi_answers"], is_training=True, feature_cache_dir=tmp + qa_test_data["train_dataset_multi_answers"], + is_training=True, + feature_cache_dir=tmp_module, ) -def test_AnswerExtractor(qa_test_data, tmp): +@pytest.mark.gpu +def test_AnswerExtractor(qa_test_data, tmp_module): # test bert - qa_extractor_bert = AnswerExtractor(cache_dir=tmp) + qa_extractor_bert = AnswerExtractor(cache_dir=tmp_module) qa_extractor_bert.fit(qa_test_data["train_features_bert"], cache_model=True) # test saving fine-tuned model - model_output_dir = os.path.join(tmp, "fine_tuned") + model_output_dir = os.path.join(tmp_module, "fine_tuned") assert os.path.exists(os.path.join(model_output_dir, "pytorch_model.bin")) assert os.path.exists(os.path.join(model_output_dir, "config.json")) - qa_extractor_from_cache = AnswerExtractor(cache_dir=tmp, load_model_from_dir=model_output_dir) + qa_extractor_from_cache = AnswerExtractor( + cache_dir=tmp_module, load_model_from_dir=model_output_dir + ) qa_extractor_from_cache.predict(qa_test_data["test_features_bert"]) - qa_extractor_xlnet = AnswerExtractor(model_name="xlnet-base-cased", cache_dir=tmp) + qa_extractor_xlnet = AnswerExtractor(model_name="xlnet-base-cased", cache_dir=tmp_module) qa_extractor_xlnet.fit(qa_test_data["train_features_xlnet"], cache_model=False) qa_extractor_xlnet.predict(qa_test_data["test_features_xlnet"]) - qa_extractor_distilbert = AnswerExtractor(model_name="distilbert-base-uncased", cache_dir=tmp) + qa_extractor_distilbert = AnswerExtractor( + model_name="distilbert-base-uncased", cache_dir=tmp_module + ) qa_extractor_distilbert.fit(qa_test_data["train_features_distilbert"], cache_model=False) qa_extractor_distilbert.predict(qa_test_data["test_features_distilbert"]) -def test_postprocess_bert_answer(qa_test_data, tmp): - qa_processor = QAProcessor(cache_dir=tmp) +@pytest.mark.gpu +def test_postprocess_bert_answer(qa_test_data, tmp_module): + qa_processor = QAProcessor(cache_dir=tmp_module) test_features = qa_processor.preprocess( qa_test_data["test_dataset"], is_training=False, max_question_length=16, max_seq_length=64, doc_stride=32, - feature_cache_dir=tmp, + feature_cache_dir=tmp_module, ) - qa_extractor = AnswerExtractor(cache_dir=tmp) + qa_extractor = AnswerExtractor(cache_dir=tmp_module) predictions = qa_extractor.predict(test_features) qa_processor.postprocess( results=predictions, - examples_file=os.path.join(tmp, CACHED_EXAMPLES_TEST_FILE), - features_file=os.path.join(tmp, CACHED_FEATURES_TEST_FILE), - output_prediction_file=os.path.join(tmp, "qa_predictions.json"), - output_nbest_file=os.path.join(tmp, "nbest_predictions.json"), - output_null_log_odds_file=os.path.join(tmp, "null_odds.json"), + examples_file=os.path.join(tmp_module, CACHED_EXAMPLES_TEST_FILE), + features_file=os.path.join(tmp_module, CACHED_FEATURES_TEST_FILE), + output_prediction_file=os.path.join(tmp_module, "qa_predictions.json"), + output_nbest_file=os.path.join(tmp_module, "nbest_predictions.json"), + output_null_log_odds_file=os.path.join(tmp_module, "null_odds.json"), ) qa_processor.postprocess( results=predictions, - examples_file=os.path.join(tmp, CACHED_EXAMPLES_TEST_FILE), - features_file=os.path.join(tmp, CACHED_FEATURES_TEST_FILE), + examples_file=os.path.join(tmp_module, CACHED_EXAMPLES_TEST_FILE), + features_file=os.path.join(tmp_module, CACHED_FEATURES_TEST_FILE), unanswerable_exists=True, verbose_logging=True, - output_prediction_file=os.path.join(tmp, "qa_predictions.json"), - output_nbest_file=os.path.join(tmp, "nbest_predictions.json"), - output_null_log_odds_file=os.path.join(tmp, "null_odds.json"), + output_prediction_file=os.path.join(tmp_module, "qa_predictions.json"), + output_nbest_file=os.path.join(tmp_module, "nbest_predictions.json"), + output_null_log_odds_file=os.path.join(tmp_module, "null_odds.json"), ) -def test_postprocess_xlnet_answer(qa_test_data, tmp): - qa_processor = QAProcessor(model_name="xlnet-base-cased", cache_dir=tmp) +@pytest.mark.gpu +def test_postprocess_xlnet_answer(qa_test_data, tmp_module): + qa_processor = QAProcessor(model_name="xlnet-base-cased", cache_dir=tmp_module) test_features = qa_processor.preprocess( qa_test_data["test_dataset"], is_training=False, max_question_length=16, max_seq_length=64, doc_stride=32, - feature_cache_dir=tmp, + feature_cache_dir=tmp_module, ) - qa_extractor = AnswerExtractor(model_name="xlnet-base-cased", cache_dir=tmp) + qa_extractor = AnswerExtractor(model_name="xlnet-base-cased", cache_dir=tmp_module) predictions = qa_extractor.predict(test_features) qa_processor.postprocess( results=predictions, - examples_file=os.path.join(tmp, CACHED_EXAMPLES_TEST_FILE), - features_file=os.path.join(tmp, CACHED_FEATURES_TEST_FILE), - output_prediction_file=os.path.join(tmp, "qa_predictions.json"), - output_nbest_file=os.path.join(tmp, "nbest_predictions.json"), - output_null_log_odds_file=os.path.join(tmp, "null_odds.json"), + examples_file=os.path.join(tmp_module, CACHED_EXAMPLES_TEST_FILE), + features_file=os.path.join(tmp_module, CACHED_FEATURES_TEST_FILE), + output_prediction_file=os.path.join(tmp_module, "qa_predictions.json"), + output_nbest_file=os.path.join(tmp_module, "nbest_predictions.json"), + output_null_log_odds_file=os.path.join(tmp_module, "null_odds.json"), ) qa_processor.postprocess( results=predictions, - examples_file=os.path.join(tmp, CACHED_EXAMPLES_TEST_FILE), - features_file=os.path.join(tmp, CACHED_FEATURES_TEST_FILE), + examples_file=os.path.join(tmp_module, CACHED_EXAMPLES_TEST_FILE), + features_file=os.path.join(tmp_module, CACHED_FEATURES_TEST_FILE), unanswerable_exists=True, verbose_logging=True, - output_prediction_file=os.path.join(tmp, "qa_predictions.json"), - output_nbest_file=os.path.join(tmp, "nbest_predictions.json"), - output_null_log_odds_file=os.path.join(tmp, "null_odds.json"), + output_prediction_file=os.path.join(tmp_module, "qa_predictions.json"), + output_nbest_file=os.path.join(tmp_module, "nbest_predictions.json"), + output_null_log_odds_file=os.path.join(tmp_module, "null_odds.json"), ) From 6c2ab2a07f57dd08ed69d91337eecd5f4a2fa390 Mon Sep 17 00:00:00 2001 From: hlums Date: Wed, 27 Nov 2019 22:13:07 +0000 Subject: [PATCH 12/18] Attempt to fix test device error. --- utils_nlp/models/transformers/common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils_nlp/models/transformers/common.py b/utils_nlp/models/transformers/common.py index 26d387b38..e6186bd0e 100644 --- a/utils_nlp/models/transformers/common.py +++ b/utils_nlp/models/transformers/common.py @@ -145,7 +145,7 @@ def fine_tune( # multi-gpu training (should be after apex fp16 initialization) if n_gpu > 1: - self.model = torch.nn.DataParallel(self.model) + self.model = torch.nn.DataParallel(self.model, device_ids=[0, 1, 2, 3]) # Distributed training (should be after apex fp16 initialization) if local_rank != -1: From 4b13b9d5ee3b1bc8149049f1be6eaed7764b53f1 Mon Sep 17 00:00:00 2001 From: hlums Date: Wed, 27 Nov 2019 22:33:50 +0000 Subject: [PATCH 13/18] Temporarily pin transformers version --- tools/generate_conda_file.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/generate_conda_file.py b/tools/generate_conda_file.py index afb82199a..b4a0f4fe1 100644 --- a/tools/generate_conda_file.py +++ b/tools/generate_conda_file.py @@ -83,7 +83,7 @@ "https://github.com/explosion/spacy-models/releases/download/" "en_core_web_sm-2.1.0/en_core_web_sm-2.1.0.tar.gz" ), - "transformers": "transformers>=2.0.0", + "transformers": "transformers==2.1.1", "gensim": "gensim>=3.7.0", "nltk": "nltk>=3.4", "seqeval": "seqeval>=0.0.12", From 3e72fb034eeb3eefebcd9a9d06fb8b1d2103ef7c Mon Sep 17 00:00:00 2001 From: hlums Date: Wed, 27 Nov 2019 23:08:28 +0000 Subject: [PATCH 14/18] Remove gpu tags temporarily --- tests/unit/test_models_transformers_question_answering.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/tests/unit/test_models_transformers_question_answering.py b/tests/unit/test_models_transformers_question_answering.py index ca7a5a69d..010bf5c5d 100644 --- a/tests/unit/test_models_transformers_question_answering.py +++ b/tests/unit/test_models_transformers_question_answering.py @@ -190,7 +190,6 @@ def test_QAProcessor(qa_test_data, tmp_module): ) -@pytest.mark.gpu def test_AnswerExtractor(qa_test_data, tmp_module): # test bert qa_extractor_bert = AnswerExtractor(cache_dir=tmp_module) @@ -217,7 +216,6 @@ def test_AnswerExtractor(qa_test_data, tmp_module): qa_extractor_distilbert.predict(qa_test_data["test_features_distilbert"]) -@pytest.mark.gpu def test_postprocess_bert_answer(qa_test_data, tmp_module): qa_processor = QAProcessor(cache_dir=tmp_module) test_features = qa_processor.preprocess( @@ -252,7 +250,6 @@ def test_postprocess_bert_answer(qa_test_data, tmp_module): ) -@pytest.mark.gpu def test_postprocess_xlnet_answer(qa_test_data, tmp_module): qa_processor = QAProcessor(model_name="xlnet-base-cased", cache_dir=tmp_module) test_features = qa_processor.preprocess( From 40ae2b71701a309645bbc66dbfb129d7ce0e2964 Mon Sep 17 00:00:00 2001 From: hlums Date: Wed, 27 Nov 2019 23:09:09 +0000 Subject: [PATCH 15/18] Test whether device error also occurs for SequenceClassifier. --- tests/unit/test_transformers_sequence_classification.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/test_transformers_sequence_classification.py b/tests/unit/test_transformers_sequence_classification.py index 156854200..a000727dc 100644 --- a/tests/unit/test_transformers_sequence_classification.py +++ b/tests/unit/test_transformers_sequence_classification.py @@ -12,7 +12,7 @@ def data(): return (["hi", "hello", "what's wrong with us", "can I leave?"], [0, 0, 1, 2]) -@pytest.mark.cpu +@pytest.mark.gpu def test_classifier(data, tmpdir): df = pd.DataFrame({"text": data[0], "label": data[1]}) From 321032e0e50b72dd0413da3481f8a2a04322a07a Mon Sep 17 00:00:00 2001 From: hlums Date: Wed, 27 Nov 2019 23:25:49 +0000 Subject: [PATCH 16/18] Revert temporary changes. --- tests/unit/test_transformers_sequence_classification.py | 4 ++-- utils_nlp/models/transformers/common.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/unit/test_transformers_sequence_classification.py b/tests/unit/test_transformers_sequence_classification.py index a000727dc..c35b2e31b 100644 --- a/tests/unit/test_transformers_sequence_classification.py +++ b/tests/unit/test_transformers_sequence_classification.py @@ -9,10 +9,10 @@ @pytest.fixture() def data(): - return (["hi", "hello", "what's wrong with us", "can I leave?"], [0, 0, 1, 2]) + return (["hi", "hello", "what's wrong with us", "can I leave?"]) -@pytest.mark.gpu +@pytest.mark.cpu def test_classifier(data, tmpdir): df = pd.DataFrame({"text": data[0], "label": data[1]}) diff --git a/utils_nlp/models/transformers/common.py b/utils_nlp/models/transformers/common.py index e6186bd0e..26d387b38 100644 --- a/utils_nlp/models/transformers/common.py +++ b/utils_nlp/models/transformers/common.py @@ -145,7 +145,7 @@ def fine_tune( # multi-gpu training (should be after apex fp16 initialization) if n_gpu > 1: - self.model = torch.nn.DataParallel(self.model, device_ids=[0, 1, 2, 3]) + self.model = torch.nn.DataParallel(self.model) # Distributed training (should be after apex fp16 initialization) if local_rank != -1: From 3bb5cce06896abd4ed365df670f86185a56aed7b Mon Sep 17 00:00:00 2001 From: hlums Date: Wed, 27 Nov 2019 23:27:01 +0000 Subject: [PATCH 17/18] Revert temporary changes. --- tests/unit/test_transformers_sequence_classification.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/test_transformers_sequence_classification.py b/tests/unit/test_transformers_sequence_classification.py index c35b2e31b..156854200 100644 --- a/tests/unit/test_transformers_sequence_classification.py +++ b/tests/unit/test_transformers_sequence_classification.py @@ -9,7 +9,7 @@ @pytest.fixture() def data(): - return (["hi", "hello", "what's wrong with us", "can I leave?"]) + return (["hi", "hello", "what's wrong with us", "can I leave?"], [0, 0, 1, 2]) @pytest.mark.cpu From 75e6eb9c95126d16b35fbcb1b1e487ae2e0a6eeb Mon Sep 17 00:00:00 2001 From: Emmanuel Awa Date: Tue, 3 Dec 2019 15:13:27 +0000 Subject: [PATCH 18/18] update: major release version to 2.0.0 --- utils_nlp/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils_nlp/__init__.py b/utils_nlp/__init__.py index 09e0f7e3c..96087d9ef 100755 --- a/utils_nlp/__init__.py +++ b/utils_nlp/__init__.py @@ -5,7 +5,7 @@ __author__ = "AI CAT at Microsoft" __license__ = "MIT" __copyright__ = "Copyright 2018-present Microsoft Corporation" -__version__ = "1.0.0" +__version__ = "2.0.0" # Synonyms TITLE = __title__