From 4338f20df63dc5e4d837f0ac4f8d414a27bc997d Mon Sep 17 00:00:00 2001 From: jirka Date: Thu, 18 Jul 2024 22:04:27 +0200 Subject: [PATCH 01/17] ci: fix label change --- .github/label-change.yml | 18 +++++++++++------- .pre-commit-config.yaml | 2 +- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/.github/label-change.yml b/.github/label-change.yml index 6ca0572b6..6a0edff8a 100644 --- a/.github/label-change.yml +++ b/.github/label-change.yml @@ -1,8 +1,12 @@ -documentation: - - _docs/**/* +"topic: documentation": + - changed-files: + - any-glob-to-any-file: + - _docs/**/* -CI/CD: - - .actions/**/* - - .azure-*/**/* - - .github/**/* - - _dockers/**/* +"topic: CI/CD": + - changed-files: + - any-glob-to-any-file: + - .actions/**/* + - .azure-*/**/* + - .github/**/* + - _dockers/**/* diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index c3760ae38..41b5535e1 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,5 +1,5 @@ default_language_version: - python: python3.8 + python: python3 ci: autofix_prs: true From d047ff8fdc366b57b03038fce5ca407024a6a035 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 18 Jul 2024 22:11:15 +0200 Subject: [PATCH 02/17] [pre-commit.ci] pre-commit suggestions (#321) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Jirka Borovec <6035284+Borda@users.noreply.github.com> Co-authored-by: jirka --- .pre-commit-config.yaml | 8 ++++---- .../Transformers_MHAttention.py | 12 ++++++------ .../06-graph-neural-networks/GNN_overview.py | 10 +++++----- .../Deep_Energy_Models.py | 10 +++++----- .../08-deep-autoencoders/Deep_Autoencoders.py | 2 +- .../09-normalizing-flows/NF_image_modeling.py | 8 ++++---- .../Autoregressive_Image_Modeling.py | 2 +- .../11-vision-transformer/Vision_Transformer.py | 8 ++++---- course_UvA-DL/12-meta-learning/Meta_Learning.py | 12 ++++++------ course_UvA-DL/13-contrastive-learning/SimCLR.py | 2 +- .../image_classification/image_classification.py | 2 +- 11 files changed, 38 insertions(+), 38 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 41b5535e1..2f57eeb69 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -9,7 +9,7 @@ ci: repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.5.0 + rev: v4.6.0 hooks: - id: end-of-file-fixer - id: trailing-whitespace @@ -23,7 +23,7 @@ repos: - id: detect-private-key - repo: https://github.com/codespell-project/codespell - rev: v2.2.6 + rev: v2.3.0 hooks: - id: codespell additional_dependencies: [tomli] @@ -37,7 +37,7 @@ repos: args: ["--in-place"] - repo: https://github.com/pre-commit/mirrors-prettier - rev: v3.1.0 + rev: v4.0.0-alpha.8 hooks: - id: prettier files: \.(json|yml|yaml|toml) @@ -54,7 +54,7 @@ repos: - mdformat_frontmatter - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.3.5 + rev: v0.5.0 hooks: # try to fix what is possible - id: ruff diff --git a/course_UvA-DL/05-transformers-and-MH-attention/Transformers_MHAttention.py b/course_UvA-DL/05-transformers-and-MH-attention/Transformers_MHAttention.py index 02f233849..f3ba2d528 100644 --- a/course_UvA-DL/05-transformers-and-MH-attention/Transformers_MHAttention.py +++ b/course_UvA-DL/05-transformers-and-MH-attention/Transformers_MHAttention.py @@ -87,7 +87,7 @@ os.makedirs(file_path.rsplit("/", 1)[0], exist_ok=True) if not os.path.isfile(file_path): file_url = base_url + file_name - print("Downloading %s..." % file_url) + print(f"Downloading {file_url}...") try: urllib.request.urlretrieve(file_url, file_path) except HTTPError as e: @@ -796,7 +796,7 @@ def _create_model(self): num_heads=self.hparams.num_heads, dropout=self.hparams.dropout, ) - # Output classifier per sequence lement + # Output classifier per sequence element self.output_net = nn.Sequential( nn.Linear(self.hparams.model_dim, self.hparams.model_dim), nn.LayerNorm(self.hparams.model_dim), @@ -948,8 +948,8 @@ def _calculate_loss(self, batch, mode="train"): acc = (preds.argmax(dim=-1) == labels).float().mean() # Logging - self.log("%s_loss" % mode, loss) - self.log("%s_acc" % mode, acc) + self.log(f"{mode}_loss", loss) + self.log(f"{mode}_acc", acc) return loss, acc def training_step(self, batch, batch_idx): @@ -1419,8 +1419,8 @@ def _calculate_loss(self, batch, mode="train"): preds = preds.squeeze(dim=-1) # Shape: [Batch_size, set_size] loss = F.cross_entropy(preds, labels) # Softmax/CE over set dimension acc = (preds.argmax(dim=-1) == labels).float().mean() - self.log("%s_loss" % mode, loss) - self.log("%s_acc" % mode, acc, on_step=False, on_epoch=True) + self.log(f"{mode}_loss", loss) + self.log(f"{mode}_acc", acc, on_step=False, on_epoch=True) return loss, acc def training_step(self, batch, batch_idx): diff --git a/course_UvA-DL/06-graph-neural-networks/GNN_overview.py b/course_UvA-DL/06-graph-neural-networks/GNN_overview.py index 6767f0a61..5b53866f0 100644 --- a/course_UvA-DL/06-graph-neural-networks/GNN_overview.py +++ b/course_UvA-DL/06-graph-neural-networks/GNN_overview.py @@ -61,7 +61,7 @@ os.makedirs(file_path.rsplit("/", 1)[0], exist_ok=True) if not os.path.isfile(file_path): file_url = base_url + file_name - print("Downloading %s..." % file_url) + print(f"Downloading {file_url}...") try: urllib.request.urlretrieve(file_url, file_path) except HTTPError as e: @@ -616,7 +616,7 @@ def forward(self, data, mode="train"): elif mode == "test": mask = data.test_mask else: - assert False, "Unknown forward mode: %s" % mode + assert False, f"Unknown forward mode: {mode}" loss = self.loss_module(x[mask], data.y[mask]) acc = (x[mask].argmax(dim=-1) == data.y[mask]).sum().float() / mask.sum() @@ -671,7 +671,7 @@ def train_node_classifier(model_name, dataset, **model_kwargs): trainer.logger._default_hp_metric = None # Optional logging argument that we don't need # Check whether pretrained model exists. If yes, load it and skip training - pretrained_filename = os.path.join(CHECKPOINT_PATH, "NodeLevel%s.ckpt" % model_name) + pretrained_filename = os.path.join(CHECKPOINT_PATH, f"NodeLevel{model_name}.ckpt") if os.path.isfile(pretrained_filename): print("Found pretrained model, loading...") model = NodeLevelGNN.load_from_checkpoint(pretrained_filename) @@ -790,7 +790,7 @@ def print_results(result_dict): # %% print("Data object:", tu_dataset.data) print("Length:", len(tu_dataset)) -print("Average label: %4.2f" % (tu_dataset.data.y.float().mean().item())) +print(f"Average label: {tu_dataset.data.y.float().mean().item():4.2f}") # %% [markdown] # The first line shows how the dataset stores different graphs. @@ -957,7 +957,7 @@ def train_graph_classifier(model_name, **model_kwargs): trainer.logger._default_hp_metric = None # Check whether pretrained model exists. If yes, load it and skip training - pretrained_filename = os.path.join(CHECKPOINT_PATH, "GraphLevel%s.ckpt" % model_name) + pretrained_filename = os.path.join(CHECKPOINT_PATH, f"GraphLevel{model_name}.ckpt") if os.path.isfile(pretrained_filename): print("Found pretrained model, loading...") model = GraphLevelGNN.load_from_checkpoint(pretrained_filename) diff --git a/course_UvA-DL/07-deep-energy-based-generative-models/Deep_Energy_Models.py b/course_UvA-DL/07-deep-energy-based-generative-models/Deep_Energy_Models.py index f9f15ccb5..f4e6d8c77 100644 --- a/course_UvA-DL/07-deep-energy-based-generative-models/Deep_Energy_Models.py +++ b/course_UvA-DL/07-deep-energy-based-generative-models/Deep_Energy_Models.py @@ -68,7 +68,7 @@ os.makedirs(file_path.rsplit("/", 1)[0], exist_ok=True) if not os.path.isfile(file_path): file_url = base_url + file_name - print("Downloading %s..." % file_url) + print(f"Downloading {file_url}...") try: urllib.request.urlretrieve(file_url, file_path) except HTTPError as e: @@ -770,7 +770,7 @@ def train_model(**kwargs): rand_imgs = torch.rand((128,) + model.hparams.img_shape).to(model.device) rand_imgs = rand_imgs * 2 - 1.0 rand_out = model.cnn(rand_imgs).mean() - print("Average score for random images: %4.2f" % (rand_out.item())) + print(f"Average score for random images: {rand_out.item():4.2f}") # %% [markdown] # As we hoped, the model assigns very low probability to those noisy images. @@ -781,7 +781,7 @@ def train_model(**kwargs): train_imgs, _ = next(iter(train_loader)) train_imgs = train_imgs.to(model.device) train_out = model.cnn(train_imgs).mean() - print("Average score for training images: %4.2f" % (train_out.item())) + print(f"Average score for training images: {train_out.item():4.2f}") # %% [markdown] # The scores are close to 0 because of the regularization objective that was added to the training. @@ -803,8 +803,8 @@ def compare_images(img1, img2): plt.xticks([(img1.shape[2] + 2) * (0.5 + j) for j in range(2)], labels=["Original image", "Transformed image"]) plt.yticks([]) plt.show() - print("Score original image: %4.2f" % score1) - print("Score transformed image: %4.2f" % score2) + print(f"Score original image: {score1:4.2f}") + print(f"Score transformed image: {score2:4.2f}") # %% [markdown] diff --git a/course_UvA-DL/08-deep-autoencoders/Deep_Autoencoders.py b/course_UvA-DL/08-deep-autoencoders/Deep_Autoencoders.py index 015cdb15a..7819bd8cc 100644 --- a/course_UvA-DL/08-deep-autoencoders/Deep_Autoencoders.py +++ b/course_UvA-DL/08-deep-autoencoders/Deep_Autoencoders.py @@ -64,7 +64,7 @@ file_path = os.path.join(CHECKPOINT_PATH, file_name) if not os.path.isfile(file_path): file_url = base_url + file_name - print("Downloading %s..." % file_url) + print(f"Downloading {file_url}...") try: urllib.request.urlretrieve(file_url, file_path) except HTTPError as e: diff --git a/course_UvA-DL/09-normalizing-flows/NF_image_modeling.py b/course_UvA-DL/09-normalizing-flows/NF_image_modeling.py index 00c53aa01..9b0c02e67 100644 --- a/course_UvA-DL/09-normalizing-flows/NF_image_modeling.py +++ b/course_UvA-DL/09-normalizing-flows/NF_image_modeling.py @@ -68,7 +68,7 @@ file_path = os.path.join(CHECKPOINT_PATH, file_name) if not os.path.isfile(file_path): file_url = base_url + file_name - print("Downloading %s..." % file_url) + print(f"Downloading {file_url}...") try: urllib.request.urlretrieve(file_url, file_path) except HTTPError as e: @@ -518,7 +518,7 @@ def visualize_dequantization(quants, prior=None): plt.plot([inp[indices[0][-1]]] * 2, [0, prob[indices[0][-1]]], color=color) x_ticks.append(inp[indices[0][0]]) x_ticks.append(inp.max()) - plt.xticks(x_ticks, ["%.1f" % x for x in x_ticks]) + plt.xticks(x_ticks, [f"{x:.1f}" for x in x_ticks]) plt.plot(inp, prob, color=(0.0, 0.0, 0.0)) # Set final plot properties plt.ylim(0, prob.max() * 1.1) @@ -1199,8 +1199,8 @@ def print_num_params(model): table = [ [ key, - "%4.3f bpd" % flow_dict[key]["result"]["val"][0]["test_bpd"], - "%4.3f bpd" % flow_dict[key]["result"]["test"][0]["test_bpd"], + "{:4.3f} bpd".format(flow_dict[key]["result"]["val"][0]["test_bpd"]), + "{:4.3f} bpd".format(flow_dict[key]["result"]["test"][0]["test_bpd"]), "%2.0f ms" % (1000 * flow_dict[key]["result"]["time"]), "%2.0f ms" % (1000 * flow_dict[key]["result"].get("samp_time", 0)), "{:,}".format(sum(np.prod(p.shape) for p in flow_dict[key]["model"].parameters())), diff --git a/course_UvA-DL/10-autoregressive-image-modeling/Autoregressive_Image_Modeling.py b/course_UvA-DL/10-autoregressive-image-modeling/Autoregressive_Image_Modeling.py index 2fdf597ea..3367a1496 100644 --- a/course_UvA-DL/10-autoregressive-image-modeling/Autoregressive_Image_Modeling.py +++ b/course_UvA-DL/10-autoregressive-image-modeling/Autoregressive_Image_Modeling.py @@ -92,7 +92,7 @@ file_path = os.path.join(CHECKPOINT_PATH, file_name) if not os.path.isfile(file_path): file_url = base_url + file_name - print("Downloading %s..." % file_url) + print(f"Downloading {file_url}...") try: urllib.request.urlretrieve(file_url, file_path) except HTTPError as e: diff --git a/course_UvA-DL/11-vision-transformer/Vision_Transformer.py b/course_UvA-DL/11-vision-transformer/Vision_Transformer.py index 73d231325..fd7a6d45a 100644 --- a/course_UvA-DL/11-vision-transformer/Vision_Transformer.py +++ b/course_UvA-DL/11-vision-transformer/Vision_Transformer.py @@ -69,7 +69,7 @@ os.makedirs(file_path.rsplit("/", 1)[0], exist_ok=True) if not os.path.isfile(file_path): file_url = base_url + file_name - print("Downloading %s..." % file_url) + print(f"Downloading {file_url}...") try: urllib.request.urlretrieve(file_url, file_path) except HTTPError as e: @@ -353,8 +353,8 @@ def _calculate_loss(self, batch, mode="train"): loss = F.cross_entropy(preds, labels) acc = (preds.argmax(dim=-1) == labels).float().mean() - self.log("%s_loss" % mode, loss) - self.log("%s_acc" % mode, acc) + self.log(f"{mode}_loss", loss) + self.log(f"{mode}_acc", acc) return loss def training_step(self, batch, batch_idx): @@ -396,7 +396,7 @@ def train_model(**kwargs): # Check whether pretrained model exists. If yes, load it and skip training pretrained_filename = os.path.join(CHECKPOINT_PATH, "ViT.ckpt") if os.path.isfile(pretrained_filename): - print("Found pretrained model at %s, loading..." % pretrained_filename) + print(f"Found pretrained model at {pretrained_filename}, loading...") # Automatically loads the model with the saved hyperparameters model = ViT.load_from_checkpoint(pretrained_filename) else: diff --git a/course_UvA-DL/12-meta-learning/Meta_Learning.py b/course_UvA-DL/12-meta-learning/Meta_Learning.py index 5e17b5692..6cb110823 100644 --- a/course_UvA-DL/12-meta-learning/Meta_Learning.py +++ b/course_UvA-DL/12-meta-learning/Meta_Learning.py @@ -92,7 +92,7 @@ os.makedirs(file_path.rsplit("/", 1)[0], exist_ok=True) if not os.path.isfile(file_path): file_url = base_url + file_name - print("Downloading %s..." % file_url) + print(f"Downloading {file_url}...") try: urllib.request.urlretrieve(file_url, file_path) except HTTPError as e: @@ -525,8 +525,8 @@ def calculate_loss(self, batch, mode): preds, labels, acc = self.classify_feats(prototypes, classes, query_feats, query_targets) loss = F.cross_entropy(preds, labels) - self.log("%s_loss" % mode, loss) - self.log("%s_acc" % mode, acc) + self.log(f"{mode}_loss", loss) + self.log(f"{mode}_acc", acc) return loss def training_step(self, batch, batch_idx): @@ -573,7 +573,7 @@ def train_model(model_class, train_loader, val_loader, **kwargs): # Check whether pretrained model exists. If yes, load it and skip training pretrained_filename = os.path.join(CHECKPOINT_PATH, model_class.__name__ + ".ckpt") if os.path.isfile(pretrained_filename): - print("Found pretrained model at %s, loading..." % pretrained_filename) + print(f"Found pretrained model at {pretrained_filename}, loading...") # Automatically loads the model with the saved hyperparameters model = model_class.load_from_checkpoint(pretrained_filename) else: @@ -947,8 +947,8 @@ def outer_loop(self, batch, mode="train"): opt.step() opt.zero_grad() - self.log("%s_loss" % mode, sum(losses) / len(losses)) - self.log("%s_acc" % mode, sum(accuracies) / len(accuracies)) + self.log(f"{mode}_loss", sum(losses) / len(losses)) + self.log(f"{mode}_acc", sum(accuracies) / len(accuracies)) def training_step(self, batch, batch_idx): self.outer_loop(batch, mode="train") diff --git a/course_UvA-DL/13-contrastive-learning/SimCLR.py b/course_UvA-DL/13-contrastive-learning/SimCLR.py index 40a71192d..cc8d027cb 100644 --- a/course_UvA-DL/13-contrastive-learning/SimCLR.py +++ b/course_UvA-DL/13-contrastive-learning/SimCLR.py @@ -760,7 +760,7 @@ def train_resnet(batch_size, max_epochs=100, **kwargs): # Check whether pretrained model exists. If yes, load it and skip training pretrained_filename = os.path.join(CHECKPOINT_PATH, "ResNet.ckpt") if os.path.isfile(pretrained_filename): - print("Found pretrained model at %s, loading..." % pretrained_filename) + print(f"Found pretrained model at {pretrained_filename}, loading...") model = ResNet.load_from_checkpoint(pretrained_filename) else: L.seed_everything(42) # To be reproducible diff --git a/flash_tutorials/image_classification/image_classification.py b/flash_tutorials/image_classification/image_classification.py index ba34a8b0e..f656f09dc 100644 --- a/flash_tutorials/image_classification/image_classification.py +++ b/flash_tutorials/image_classification/image_classification.py @@ -1,5 +1,5 @@ # %% [markdown] -# In this tutorial, we'll go over the basics of lightning Flash by finetuning/predictin with an ImageClassifier on [Hymenoptera Dataset](https://www.kaggle.com/ajayrana/hymenoptera-data) containing ants and bees images. +# In this tutorial, we'll go over the basics of lightning Flash by finetuning/prediction with an ImageClassifier on [Hymenoptera Dataset](https://www.kaggle.com/ajayrana/hymenoptera-data) containing ants and bees images. # # # Finetuning # From 65710b9e88783e16d405f7eec0887196e18a8dd4 Mon Sep 17 00:00:00 2001 From: Jirka Borovec <6035284+Borda@users.noreply.github.com> Date: Fri, 19 Jul 2024 18:53:11 +0200 Subject: [PATCH 03/17] ci/test: use bare `ipython` instead of `nbval` (#324) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .actions/assistant.py | 51 ++++++++++++++----- .../{ipynb-tests.yml => ipynb-validate.yml} | 13 +++-- .../{ci_test-acts.yml => ci_internal.yml} | 0 README.md | 4 +- _requirements/default.txt | 1 + _requirements/devel.txt | 2 +- .../01-introduction-to-pytorch/.meta.yml | 1 + ...Introduction_to_PyTorch.py => notebook.py} | 0 .../.meta.yml | 4 +- .../{Deep_Energy_Models.py => notebook.py} | 24 ++++----- pyproject.toml | 1 - 11 files changed, 65 insertions(+), 36 deletions(-) rename .azure/{ipynb-tests.yml => ipynb-validate.yml} (88%) rename .github/workflows/{ci_test-acts.yml => ci_internal.yml} (100%) rename course_UvA-DL/01-introduction-to-pytorch/{Introduction_to_PyTorch.py => notebook.py} (100%) rename course_UvA-DL/07-deep-energy-based-generative-models/{Deep_Energy_Models.py => notebook.py} (99%) diff --git a/.actions/assistant.py b/.actions/assistant.py index 8439680f8..32e8a3b6f 100644 --- a/.actions/assistant.py +++ b/.actions/assistant.py @@ -113,7 +113,7 @@ def get_running_cuda_version() -> str: return "" -def get_running_torch_version(): +def get_running_torch_version() -> str: """Extract the version of actual PyTorch for this runtime.""" try: import torch @@ -322,7 +322,13 @@ def bash_render(folder: str, output_file: str = PATH_SCRIPT_RENDER) -> Optional[ # dry run does not execute the notebooks just takes them as they are cmd.append(f"cp {ipynb_file} {pub_ipynb}") # copy and add meta config - cmd += [f"cp {meta_file} {pub_meta}", f"cat {pub_meta}", f"git add {pub_meta}"] + cmd += [ + f"cp {meta_file} {pub_meta}", + 'echo "#====== START OF YAML FILE ======#"', + f"cat {pub_meta}", + 'echo "#======= END OF YAML FILE =======#"', + f"git add {pub_meta}", + ] else: pip_req, pip_args = AssistantCLI._parse_requirements(folder) cmd += [f"pip install {pip_req} --quiet {pip_args}", "pip list"] @@ -335,7 +341,13 @@ def bash_render(folder: str, output_file: str = PATH_SCRIPT_RENDER) -> Optional[ # Export the actual packages used in runtime cmd.append(f"meta_file=$(python .actions/assistant.py update-env-details {folder})") # copy and add to version the enriched meta config - cmd += ["echo $meta_file", "cat $meta_file", "git add $meta_file"] + cmd += [ + "echo $meta_file", + 'echo "#====== START OF YAML FILE ======#"', + "cat $meta_file", + 'echo "#======= END OF YAML FILE =======#"', + "git add $meta_file", + ] # if thumb image is linked to the notebook, copy and version it too if thumb_file: cmd += [f"cp {thumb_file} {pub_thumb}", f"git add {pub_thumb}"] @@ -347,7 +359,7 @@ def bash_render(folder: str, output_file: str = PATH_SCRIPT_RENDER) -> Optional[ fopen.write(os.linesep.join(cmd)) @staticmethod - def bash_test(folder: str, output_file: str = PATH_SCRIPT_TEST) -> Optional[str]: + def bash_test(folder: str, output_file: str = PATH_SCRIPT_TEST, virtualenv: bool = False) -> Optional[str]: """Prepare bash script for running tests of a particular notebook. Args: @@ -364,11 +376,12 @@ def bash_test(folder: str, output_file: str = PATH_SCRIPT_TEST) -> Optional[str] # prepare isolated environment with inheriting the global packages path_venv = os.path.join(folder, "venv") - cmd += [ - f"python -m virtualenv --system-site-packages {path_venv}", - f"source {os.path.join(path_venv, 'bin', 'activate')}", - "pip --version", - ] + if virtualenv: + cmd += [ + f"python -m virtualenv --system-site-packages {path_venv}", + f"source {os.path.join(path_venv, 'bin', 'activate')}", + "pip --version", + ] cmd.append(f"# available: {AssistantCLI.DEVICE_ACCELERATOR}") if AssistantCLI._valid_accelerator(folder): @@ -378,8 +391,14 @@ def bash_test(folder: str, output_file: str = PATH_SCRIPT_TEST) -> Optional[str] # Export the actual packages used in runtime cmd.append(f"meta_file=$(python .actions/assistant.py update-env-details {folder} --base_path .)") # show created meta config - cmd += ["echo $meta_file", "cat $meta_file"] - cmd.append(f"python -m pytest {ipynb_file} -v --nbval --nbval-cell-timeout=300") + cmd += [ + "echo $meta_file", + 'echo "#====== START OF YAML FILE ======#"', + "cat $meta_file", + 'echo "#======= END OF YAML FILE =======#"', + ] + # use standard jupyter's executable via CMD + cmd.append(f"jupyter execute {ipynb_file} --inplace") else: pub_ipynb = os.path.join(DIR_NOTEBOOKS, f"{folder}.ipynb") pub_meta = pub_ipynb.replace(".ipynb", ".yaml") @@ -387,12 +406,15 @@ def bash_test(folder: str, output_file: str = PATH_SCRIPT_TEST) -> Optional[str] cmd += [ f"mkdir -p {os.path.dirname(pub_meta)}", f"cp {meta_file} {pub_meta}", + 'echo "#====== START OF YAML FILE ======#"', f"cat {pub_meta}", + 'echo "#======= END OF YAML FILE =======#"', f"git add {pub_meta}", ] warn("Invalid notebook's accelerator for this device. So no tests will be run!!!", RuntimeWarning) # deactivate and clean local environment - cmd += ["deactivate", f"rm -rf {os.path.join(folder, 'venv')}"] + if virtualenv: + cmd += ["deactivate", f"rm -rf {os.path.join(folder, 'venv')}"] if not output_file: return os.linesep.join(cmd) with open(output_file, "w") as fopen: @@ -707,7 +729,10 @@ def update_env_details(folder: str, base_path: str = DIR_NOTEBOOKS) -> str: Args: folder: path to the folder - base_path: + base_path: base path with notebooks + + Returns: + path the updated YAML file """ meta = AssistantCLI._load_meta(folder) diff --git a/.azure/ipynb-tests.yml b/.azure/ipynb-validate.yml similarity index 88% rename from .azure/ipynb-tests.yml rename to .azure/ipynb-validate.yml index 61863bb4b..d0dc2dd47 100644 --- a/.azure/ipynb-tests.yml +++ b/.azure/ipynb-validate.yml @@ -19,9 +19,12 @@ jobs: displayName: "Install dependencies" - bash: | - head=$(git rev-parse origin/main) - printf "Head: $head\n" - git diff --name-only $head --output=target-diff.txt + git fetch --all # some issues with missing main :/ + # head=$(git rev-parse origin/main) + # printf "Head: $head\n" # this shall be commit hash + # git diff --name-only $head --output=target-diff.txt + git diff --name-only origin/main HEAD --output=target-diff.txt + cat target-diff.txt python .actions/assistant.py group-folders --fpath_gitdiff=target-diff.txt printf "Changed folders:\n" cat changed-folders.txt @@ -35,7 +38,7 @@ jobs: - bash: echo '$(mtrx.dirs)' | python -m json.tool displayName: "Show matrix" - - job: nbval + - job: ipython dependsOn: check_diff strategy: matrix: $[ dependencies.check_diff.outputs['mtrx.dirs'] ] @@ -96,4 +99,4 @@ jobs: env: KAGGLE_USERNAME: $(KAGGLE_USERNAME) KAGGLE_KEY: $(KAGGLE_KEY) - displayName: "PyTest notebook" + displayName: "Execute notebook" diff --git a/.github/workflows/ci_test-acts.yml b/.github/workflows/ci_internal.yml similarity index 100% rename from .github/workflows/ci_test-acts.yml rename to .github/workflows/ci_internal.yml diff --git a/README.md b/README.md index 5d8baa460..a5ee506fa 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # PytorchLightning Tutorials -[![CI internal](https://github.com/Lightning-AI/tutorials/actions/workflows/ci_test-acts.yml/badge.svg?event=push)](https://github.com/Lightning-AI/tutorials/actions/workflows/ci_test-acts.yml) +[![CI internal](https://github.com/Lightning-AI/tutorials/actions/workflows/ci_internal.yml/badge.svg?event=push)](https://github.com/Lightning-AI/tutorials/actions/workflows/ci_internal.yml) [![Build Status](https://dev.azure.com/Lightning-AI/Tutorials/_apis/build/status/Lightning-AI.tutorials%20%5Bpublish%5D?branchName=main)](https://dev.azure.com/Lightning-AI/Tutorials/_build/latest?definitionId=29&branchName=main) [![codecov](https://codecov.io/gh/Lightning-AI/tutorials/branch/main/graph/badge.svg?token=C6T3XOOR56)](https://codecov.io/gh/Lightning-AI/tutorials) [![Deploy Docs](https://github.com/Lightning-AI/tutorials/actions/workflows/docs-deploy.yml/badge.svg)](https://github.com/Lightning-AI/tutorials/actions/workflows/docs-deploy.yml) @@ -91,7 +91,7 @@ On the back side of publishing workflow you can find in principle these three st # 1) convert script to notebooks jupytext --set-formats ipynb,py:percent notebook.py -# 2) testing the created notebook +# 2) [OPTIONAL] testing the created notebook pytest -v notebook.ipynb --nbval # 3) generating notebooks outputs diff --git a/_requirements/default.txt b/_requirements/default.txt index 3d3d8bfb2..3ea49e720 100644 --- a/_requirements/default.txt +++ b/_requirements/default.txt @@ -2,6 +2,7 @@ setuptools>=68.0.0, <69.1.0 matplotlib>=3.0.0, <3.9.0 ipython[notebook]>=8.0.0, <8.17.0 urllib3 # for ipython +numpy <2.0 # needed for older Torch torch>=1.8.1, <2.1.0 pytorch-lightning>=1.4, <2.1.0 torchmetrics>=0.7, <1.3 diff --git a/_requirements/devel.txt b/_requirements/devel.txt index e0ad0dce6..cff867415 100644 --- a/_requirements/devel.txt +++ b/_requirements/devel.txt @@ -2,5 +2,5 @@ virtualenv>=20.10 jupytext>=1.10, <1.15 # converting pytest>=6.0, <7.0 # testing with own fork with extended cell timeout -https://github.com/Borda/nbval/archive/refs/heads/timeout-limit.zip +# https://github.com/Borda/nbval/archive/refs/heads/timeout-limit.zip papermill>=2.3.4, <2.5.0 # render diff --git a/course_UvA-DL/01-introduction-to-pytorch/.meta.yml b/course_UvA-DL/01-introduction-to-pytorch/.meta.yml index 479f30e90..03dad1cc4 100644 --- a/course_UvA-DL/01-introduction-to-pytorch/.meta.yml +++ b/course_UvA-DL/01-introduction-to-pytorch/.meta.yml @@ -3,6 +3,7 @@ author: Phillip Lippe created: 2021-08-27 updated: 2023-03-14 license: CC BY-SA +build: 1 description: | This tutorial will give a short introduction to PyTorch basics, and get you setup for writing your own neural networks. This notebook is part of a lecture series on Deep Learning at the University of Amsterdam. diff --git a/course_UvA-DL/01-introduction-to-pytorch/Introduction_to_PyTorch.py b/course_UvA-DL/01-introduction-to-pytorch/notebook.py similarity index 100% rename from course_UvA-DL/01-introduction-to-pytorch/Introduction_to_PyTorch.py rename to course_UvA-DL/01-introduction-to-pytorch/notebook.py diff --git a/course_UvA-DL/07-deep-energy-based-generative-models/.meta.yml b/course_UvA-DL/07-deep-energy-based-generative-models/.meta.yml index d3511f6ca..c86b83169 100644 --- a/course_UvA-DL/07-deep-energy-based-generative-models/.meta.yml +++ b/course_UvA-DL/07-deep-energy-based-generative-models/.meta.yml @@ -3,7 +3,7 @@ author: Phillip Lippe created: 2021-07-12 updated: 2023-03-14 license: CC BY-SA -build: 0 +build: 1 tags: - Image description: | @@ -22,7 +22,7 @@ requirements: - torchvision - matplotlib - tensorboard - - lightning>=2.0.0 + - pytorch-lightning>=2.0.0 accelerator: - CPU - GPU diff --git a/course_UvA-DL/07-deep-energy-based-generative-models/Deep_Energy_Models.py b/course_UvA-DL/07-deep-energy-based-generative-models/notebook.py similarity index 99% rename from course_UvA-DL/07-deep-energy-based-generative-models/Deep_Energy_Models.py rename to course_UvA-DL/07-deep-energy-based-generative-models/notebook.py index f4e6d8c77..7df72cbbe 100644 --- a/course_UvA-DL/07-deep-energy-based-generative-models/Deep_Energy_Models.py +++ b/course_UvA-DL/07-deep-energy-based-generative-models/notebook.py @@ -9,9 +9,6 @@ import urllib.request from urllib.error import HTTPError -# PyTorch Lightning -import lightning as L - # Plotting import matplotlib import matplotlib.pyplot as plt @@ -20,6 +17,9 @@ import matplotlib_inline.backend_inline import numpy as np +# PyTorch Lightning +import pytorch_lightning as pl + # PyTorch import torch import torch.nn as nn @@ -28,7 +28,7 @@ # Torchvision import torchvision -from lightning.pytorch.callbacks import Callback, LearningRateMonitor, ModelCheckpoint +from pytorch_lightning.callbacks import Callback, LearningRateMonitor, ModelCheckpoint from torchvision import transforms from torchvision.datasets import MNIST @@ -41,7 +41,7 @@ CHECKPOINT_PATH = os.environ.get("PATH_CHECKPOINT", "saved_models/tutorial8") # Setting the seed -L.seed_everything(42) +pl.seed_everything(42) # Ensure that all operations are deterministic on GPU (if used) for reproducibility torch.backends.cudnn.deterministic = True @@ -465,7 +465,7 @@ def generate_samples(model, inp_imgs, steps=60, step_size=10, return_img_per_ste # %% -class DeepEnergyModel(L.LightningModule): +class DeepEnergyModel(pl.LightningModule): def __init__(self, img_shape, batch_size, alpha=0.1, lr=1e-4, beta1=0.0, **CNN_args): super().__init__() self.save_hyperparameters() @@ -640,7 +640,7 @@ def on_epoch_end(self, trainer, pl_module): # %% def train_model(**kwargs): # Create a PyTorch Lightning trainer with the generation callback - trainer = L.Trainer( + trainer = pl.Trainer( default_root_dir=os.path.join(CHECKPOINT_PATH, "MNIST"), accelerator="auto", devices=1, @@ -660,7 +660,7 @@ def train_model(**kwargs): print("Found pretrained model, loading...") model = DeepEnergyModel.load_from_checkpoint(pretrained_filename) else: - L.seed_everything(42) + pl.seed_everything(42) model = DeepEnergyModel(**kwargs) trainer.fit(model, train_loader, test_loader) model = DeepEnergyModel.load_from_checkpoint(trainer.checkpoint_callback.best_model_path) @@ -709,7 +709,7 @@ def train_model(**kwargs): # %% model.to(device) -L.seed_everything(43) +pl.seed_everything(43) callback = GenerateCallback(batch_size=4, vis_steps=8, num_steps=256) imgs_per_step = callback.generate_imgs(model) imgs_per_step = imgs_per_step.cpu() @@ -770,7 +770,7 @@ def train_model(**kwargs): rand_imgs = torch.rand((128,) + model.hparams.img_shape).to(model.device) rand_imgs = rand_imgs * 2 - 1.0 rand_out = model.cnn(rand_imgs).mean() - print(f"Average score for random images: {rand_out.item():4.2f}") + print(f"Average score for random images: {rand_out.item()}") # %% [markdown] # As we hoped, the model assigns very low probability to those noisy images. @@ -803,8 +803,8 @@ def compare_images(img1, img2): plt.xticks([(img1.shape[2] + 2) * (0.5 + j) for j in range(2)], labels=["Original image", "Transformed image"]) plt.yticks([]) plt.show() - print(f"Score original image: {score1:4.2f}") - print(f"Score transformed image: {score2:4.2f}") + print(f"Score original image: {score1}") + print(f"Score transformed image: {score2}") # %% [markdown] diff --git a/pyproject.toml b/pyproject.toml index 56d0c2b6e..a0e64ce97 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -85,7 +85,6 @@ ignore = [ # TODO: we shall format all long comments as it comes from text cells "E501", # Line too long ] -ignore-init-module-imports = true [tool.ruff.lint.per-file-ignores] "setup.py" = ["D100", "SIM115"] From 8334f3466edafb427fbe2740f50aa3d5510e3f48 Mon Sep 17 00:00:00 2001 From: jirka Date: Fri, 19 Jul 2024 19:07:28 +0200 Subject: [PATCH 04/17] bash: use `set -ex` for early exit publication --- .actions/assistant.py | 2 +- .azure/ipynb-publish.yml | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/.actions/assistant.py b/.actions/assistant.py index 32e8a3b6f..3849982b5 100644 --- a/.actions/assistant.py +++ b/.actions/assistant.py @@ -154,7 +154,7 @@ class AssistantCLI: "lightning_examples": "Lightning-Examples", "flash_tutorials": "Kaggle", } - _BASH_SCRIPT_BASE = ("#!/bin/bash", "set -e", "") + _BASH_SCRIPT_BASE = ("#!/bin/bash", "set -ex", "") _EXT_ARCHIVE_ZIP = (".zip",) _EXT_ARCHIVE_TAR = (".tar", ".gz") _EXT_ARCHIVE = _EXT_ARCHIVE_ZIP + _EXT_ARCHIVE_TAR diff --git a/.azure/ipynb-publish.yml b/.azure/ipynb-publish.yml index 150cee435..c0ccefcda 100644 --- a/.azure/ipynb-publish.yml +++ b/.azure/ipynb-publish.yml @@ -24,6 +24,7 @@ jobs: printf "commit hash:\n $(COMMIT_HASH)\n" printf "commit message:\n $(COMMIT_MSG)\n" displayName: "Set Git user" + timeoutInMinutes: "5" - bash: | set -e git fetch --all @@ -59,8 +60,10 @@ jobs: git commit -m "prune: $(COMMIT_HASH)" condition: gt(variables['dropped.folders'], 0) displayName: "Prune notebook" + timeoutInMinutes: "10" - bash: | + set -ex git status git push https://$(PAT_GHOST)@github.com/Lightning-AI/tutorials.git $(PUB_BRANCH) displayName: "Finish push" @@ -118,6 +121,7 @@ jobs: printf "commit hash:\n $(COMMIT_HASH)\n" printf "commit message:\n $(COMMIT_MSG)\n" displayName: "Set Git user" + timeoutInMinutes: "5" - bash: | set -e git fetch --all @@ -128,6 +132,7 @@ jobs: git show-ref $(PUB_BRANCH) git pull displayName: "Git check & switch branch" + timeoutInMinutes: "5" - bash: | set -e @@ -136,6 +141,7 @@ jobs: # todo: adjust torch ecosystem versions pip install -r requirements.txt -r _requirements/data.txt displayName: "Install dependencies" + timeoutInMinutes: "15" - bash: | set -e @@ -146,6 +152,7 @@ jobs: - bash: python .actions/assistant.py convert-ipynb $(notebook) displayName: "Generate notebook" + timeoutInMinutes: "5" - bash: | set -e @@ -161,6 +168,7 @@ jobs: displayName: "Render notebook" - bash: | + set -ex git status git show-ref $(PUB_BRANCH) git push https://$(PAT_GHOST)@github.com/Lightning-AI/tutorials.git $(PUB_BRANCH) From 52c0010460819dca0ed717689b21d3b84ae7df4e Mon Sep 17 00:00:00 2001 From: jirka Date: Fri, 19 Jul 2024 19:55:09 +0200 Subject: [PATCH 05/17] ci: condition passing init job to continue --- .azure/ipynb-publish.yml | 10 +++++----- .azure/ipynb-validate.yml | 5 +++-- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/.azure/ipynb-publish.yml b/.azure/ipynb-publish.yml index c0ccefcda..99c1d0893 100644 --- a/.azure/ipynb-publish.yml +++ b/.azure/ipynb-publish.yml @@ -59,17 +59,19 @@ jobs: git status git commit -m "prune: $(COMMIT_HASH)" condition: gt(variables['dropped.folders'], 0) - displayName: "Prune notebook" + displayName: "Prune notebooks" timeoutInMinutes: "10" - bash: | set -ex git status git push https://$(PAT_GHOST)@github.com/Lightning-AI/tutorials.git $(PUB_BRANCH) - displayName: "Finish push" + displayName: "Finish and push" - job: papermill dependsOn: sync_pub + # run if the initial job succeeded and the strategy matrix is not empty + condition: and(succeeded(), ne(dependencies.sync_pub.outputs['mtrx.dirs'], '')) strategy: # generated matrix with changed notebooks, include fields: "notebook", "agent-pool" and "docker-image" matrix: $[ dependencies.sync_pub.outputs['mtrx.dirs'] ] @@ -97,8 +99,6 @@ jobs: COMMIT_HASH: "$(Build.SourceVersion)" DEVICES: $( python -c 'print("$(Agent.Name)".split("_")[-1])' ) - condition: ne(dependencies.sync_pub.outputs['mtrx.dirs'], '') - steps: - bash: | echo "##vso[task.setvariable variable=CUDA_VISIBLE_DEVICES]$(DEVICES)" @@ -172,4 +172,4 @@ jobs: git status git show-ref $(PUB_BRANCH) git push https://$(PAT_GHOST)@github.com/Lightning-AI/tutorials.git $(PUB_BRANCH) - displayName: "Finish push" + displayName: "Finish and push" diff --git a/.azure/ipynb-validate.yml b/.azure/ipynb-validate.yml index d0dc2dd47..84bfe77e0 100644 --- a/.azure/ipynb-validate.yml +++ b/.azure/ipynb-validate.yml @@ -14,6 +14,7 @@ jobs: vmImage: "Ubuntu-20.04" steps: - bash: | + set -ex pip install -r .actions/requires.txt pip list displayName: "Install dependencies" @@ -40,6 +41,8 @@ jobs: - job: ipython dependsOn: check_diff + # run if the initial job succeeded and the strategy matrix is not empty + condition: and(succeeded(), ne(dependencies.check_diff.outputs['mtrx.dirs'], '')) strategy: matrix: $[ dependencies.check_diff.outputs['mtrx.dirs'] ] # how long to run the job before automatically cancelling @@ -58,8 +61,6 @@ jobs: PATH_DATASETS: "$(Build.Repository.LocalPath)/.datasets" DEVICES: $( python -c 'print("$(Agent.Name)".split("_")[-1])' ) - condition: ne(dependencies.check_diff.outputs['mtrx.dirs'], '') - steps: - bash: | echo "##vso[task.setvariable variable=CUDA_VISIBLE_DEVICES]$(DEVICES)" From 56a05745b7796fd598ad4695df843a562327b27d Mon Sep 17 00:00:00 2001 From: jirka Date: Fri, 19 Jul 2024 20:13:26 +0200 Subject: [PATCH 06/17] ci: git config pull.ff only --- .azure/ipynb-publish.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.azure/ipynb-publish.yml b/.azure/ipynb-publish.yml index 99c1d0893..2c9d35dfa 100644 --- a/.azure/ipynb-publish.yml +++ b/.azure/ipynb-publish.yml @@ -118,6 +118,7 @@ jobs: - bash: | git config --global user.email "pipelines@azure.com" git config --global user.name "Azure Pipelines" + git config --global pull.ff only printf "commit hash:\n $(COMMIT_HASH)\n" printf "commit message:\n $(COMMIT_MSG)\n" displayName: "Set Git user" From 637bee73524a1c9c4d328464a86c939facce4edf Mon Sep 17 00:00:00 2001 From: jirka Date: Fri, 19 Jul 2024 20:49:10 +0200 Subject: [PATCH 07/17] publish: fix when published folder is empty --- .actions/assistant.py | 4 ++-- .actions/git-diff-sync.sh | 6 +++--- .azure/ipynb-publish.yml | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.actions/assistant.py b/.actions/assistant.py index 3849982b5..b396e7727 100644 --- a/.actions/assistant.py +++ b/.actions/assistant.py @@ -526,8 +526,6 @@ def group_folders( with open(fpath_gitdiff) as fopen: changed = [ln.strip() for ln in fopen.readlines()] dirs = [os.path.dirname(ln) for ln in changed] - # not empty paths - dirs = [ln for ln in dirs if ln] if fpath_actual_dirs: assert isinstance(fpath_actual_dirs, list) @@ -535,6 +533,8 @@ def group_folders( dir_sets = [{ln.strip() for ln in open(fp).readlines()} for fp in fpath_actual_dirs] # get only different dirs += list(set.union(*dir_sets) - set.intersection(*dir_sets)) + # not empty paths + dirs = [ln for ln in dirs if ln] if root_path: dirs = [os.path.join(root_path, d) for d in dirs] diff --git a/.actions/git-diff-sync.sh b/.actions/git-diff-sync.sh index c461e6d98..ff4c296fb 100644 --- a/.actions/git-diff-sync.sh +++ b/.actions/git-diff-sync.sh @@ -1,6 +1,6 @@ #!/bin/bash -set -e +set -ex printf "Detect changes for: $1 >> $2\n\n" b1="${1//'/'/'_'}" @@ -30,8 +30,8 @@ printf "\n\n" git merge --ff -s resolve origin/$1 python _TEMP/.actions/assistant.py group-folders target-diff.txt --fpath_actual_dirs "['dirs-$b1.txt', 'dirs-$b2.txt']" -printf "\n\nChanged folders:\n" +printf "\n================\nChanged folders:\n----------------\n" cat changed-folders.txt -printf "\n\nDropped folders:\n" +printf "\n================\nDropped folders:\n----------------\n" cat dropped-folders.txt printf "\n" diff --git a/.azure/ipynb-publish.yml b/.azure/ipynb-publish.yml index 2c9d35dfa..4dc7f8b22 100644 --- a/.azure/ipynb-publish.yml +++ b/.azure/ipynb-publish.yml @@ -37,7 +37,7 @@ jobs: displayName: "Install dependencies" - bash: | current_branch=$(cut -d '/' -f3- <<< $(Build.SourceBranch)) - printf "$current_branch\n" + printf "Current branch: $current_branch\n" bash .actions/git-diff-sync.sh $current_branch $(PUB_BRANCH) displayName: "Compare changes & sync" From ac8f7bae68dcd973d3bff93904fb5b1bed85e8b7 Mon Sep 17 00:00:00 2001 From: jirka Date: Fri, 19 Jul 2024 21:13:55 +0200 Subject: [PATCH 08/17] publish: clean workspace --- .actions/git-diff-sync.sh | 14 +++++--------- .azure/ipynb-publish.yml | 4 ++++ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.actions/git-diff-sync.sh b/.actions/git-diff-sync.sh index ff4c296fb..feb7e1f81 100644 --- a/.actions/git-diff-sync.sh +++ b/.actions/git-diff-sync.sh @@ -4,10 +4,9 @@ set -ex printf "Detect changes for: $1 >> $2\n\n" b1="${1//'/'/'_'}" -printf "Branch alias: $b1\n" # list all dirs in source branch python .actions/assistant.py list_dirs > "dirs-$b1.txt" -cat "dirs-$b1.txt" +printf "Branch alias: $b1\n" && cat "dirs-$b1.txt" head=$(git rev-parse origin/$2) git diff --name-only $head --output=target-diff.txt @@ -19,19 +18,16 @@ cp -r .actions/ _TEMP/.actions/ git checkout $2 b2="${2//'/'/'_'}" -printf "Branch alias: $b2\n" # recover the original CLI #rm -rf .actions && mv _TEMP/.actions .actions # list all dirs in target branch python _TEMP/.actions/assistant.py list_dirs ".notebooks" --include_file_ext=".ipynb" > "dirs-$b2.txt" -cat "dirs-$b2.txt" +printf "Branch alias: $b2\n" && cat "dirs-$b2.txt" printf "\n\n" git merge --ff -s resolve origin/$1 python _TEMP/.actions/assistant.py group-folders target-diff.txt --fpath_actual_dirs "['dirs-$b1.txt', 'dirs-$b2.txt']" -printf "\n================\nChanged folders:\n----------------\n" -cat changed-folders.txt -printf "\n================\nDropped folders:\n----------------\n" -cat dropped-folders.txt -printf "\n" +printf "\n================\nChanged folders:\n----------------\n" && cat changed-folders.txt +printf "\n================\nDropped folders:\n----------------\n" && cat dropped-folders.txt + diff --git a/.azure/ipynb-publish.yml b/.azure/ipynb-publish.yml index 4dc7f8b22..9cafc03e8 100644 --- a/.azure/ipynb-publish.yml +++ b/.azure/ipynb-publish.yml @@ -12,6 +12,8 @@ jobs: - job: sync_pub pool: vmImage: "Ubuntu-20.04" + workspace: + clean: all variables: ACCELERATOR: CPU,GPU PUB_BRANCH: publication @@ -90,6 +92,8 @@ jobs: container: image: $(docker-image) options: "--gpus=all --shm-size=32g -v /usr/bin/docker:/tmp/docker:ro" + workspace: + clean: all variables: ACCELERATOR: CPU,GPU From fe4e7b7703bed8e5e691fe1cb3c230ab36b8ad51 Mon Sep 17 00:00:00 2001 From: Jirka Borovec <6035284+Borda@users.noreply.github.com> Date: Fri, 19 Jul 2024 23:28:00 +0200 Subject: [PATCH 09/17] ci: dont crash if testing matrix is empty (#326) --- .actions/git-diff-sync.sh | 1 - .actions/requires.txt | 2 +- .azure/ipynb-validate.yml | 1 + .github/workflows/ci_internal.yml | 16 ++-------------- 4 files changed, 4 insertions(+), 16 deletions(-) diff --git a/.actions/git-diff-sync.sh b/.actions/git-diff-sync.sh index feb7e1f81..851addfe3 100644 --- a/.actions/git-diff-sync.sh +++ b/.actions/git-diff-sync.sh @@ -30,4 +30,3 @@ git merge --ff -s resolve origin/$1 python _TEMP/.actions/assistant.py group-folders target-diff.txt --fpath_actual_dirs "['dirs-$b1.txt', 'dirs-$b2.txt']" printf "\n================\nChanged folders:\n----------------\n" && cat changed-folders.txt printf "\n================\nDropped folders:\n----------------\n" && cat dropped-folders.txt - diff --git a/.actions/requires.txt b/.actions/requires.txt index ff35f7026..e848955fd 100644 --- a/.actions/requires.txt +++ b/.actions/requires.txt @@ -1,6 +1,6 @@ Fire tqdm -PyYAML +PyYAML <5.4 # todo: racing issue with cython compile wcmatch requests pip diff --git a/.azure/ipynb-validate.yml b/.azure/ipynb-validate.yml index 84bfe77e0..bc4ea33f5 100644 --- a/.azure/ipynb-validate.yml +++ b/.azure/ipynb-validate.yml @@ -37,6 +37,7 @@ jobs: name: mtrx displayName: "Changed matrix" - bash: echo '$(mtrx.dirs)' | python -m json.tool + continueOnError: "true" # not crash if the matrix is empty displayName: "Show matrix" - job: ipython diff --git a/.github/workflows/ci_internal.yml b/.github/workflows/ci_internal.yml index 02ed1e4a6..752e1f51f 100644 --- a/.github/workflows/ci_internal.yml +++ b/.github/workflows/ci_internal.yml @@ -30,26 +30,14 @@ jobs: uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - - # Note: This uses an internal pip API and may not always work - # https://github.com/actions/cache/blob/master/examples.md#multiple-oss-in-a-workflow - - name: Get pip cache dir - id: pip-cache - run: echo "::set-output name=dir::$(pip cache dir)" - - - name: pip cache - uses: actions/cache@v3 - with: - path: ${{ steps.pip-cache.outputs.dir }} - key: ${{ runner.os }}-pip-py${{ matrix.python-version }}-${{ hashFiles('.actions/requires.txt') }}-${{ hashFiles('requirements/default.txt') }} - restore-keys: ${{ runner.os }}-pip-py${{ matrix.python-version }}- + cache: pip - name: Install requirements run: | - pip --version pip install -q -r .actions/requires.txt -r _requirements/test.txt # this is needed to be able to run package version parsing test pip install -q -r _requirements/default.txt --find-links https://download.pytorch.org/whl/cpu/torch_stable.html + pip list - name: Prepare dummy inputs run: | From 694b7fcb679c5bfdc08757116bf6271aeb3cbc05 Mon Sep 17 00:00:00 2001 From: Jirka Borovec <6035284+Borda@users.noreply.github.com> Date: Sat, 20 Jul 2024 00:37:51 +0200 Subject: [PATCH 10/17] ci: debug deploy docs (#325) --- .actions/assistant.py | 10 +++- .github/dependabot.yml | 2 +- .github/workflows/docker-build.yml | 2 +- .github/workflows/docs-deploy.yml | 48 ++++++++++++++----- _docs/source/conf.py | 6 +-- .../01-introduction-to-pytorch/notebook.py | 2 +- .../notebook.py | 2 +- 7 files changed, 49 insertions(+), 23 deletions(-) diff --git a/.actions/assistant.py b/.actions/assistant.py index b396e7727..84f7423e6 100644 --- a/.actions/assistant.py +++ b/.actions/assistant.py @@ -648,6 +648,7 @@ def copy_notebooks( path_docs_images: str = "_static/images", patterns: Sequence[str] = (".", "**"), ignore: Optional[Sequence[str]] = None, + strict: bool = True, ) -> None: """Copy all notebooks from a folder to doc folder. @@ -658,11 +659,13 @@ def copy_notebooks( path_docs_images: destination path to the images' location relative to ``docs_root`` patterns: patterns to use when glob-ing notebooks ignore: ignore some specific notebooks even when the given string is in path + strict: raise exception if copy fails """ all_ipynb = [] for pattern in patterns: all_ipynb += glob.glob(os.path.join(path_root, DIR_NOTEBOOKS, pattern, "*.ipynb")) + print(f"Copy following notebooks to docs folder: {all_ipynb}") os.makedirs(os.path.join(docs_root, path_docs_ipynb), exist_ok=True) if ignore and not isinstance(ignore, (list, set, tuple)): ignore = [ignore] @@ -683,8 +686,11 @@ def copy_notebooks( path_docs_images=path_docs_images, ) except Exception as ex: - warnings.warn(f"Failed to copy notebook: {path_ipynb}\n{ex}", ResourceWarning) - continue + msg = f"Failed to copy notebook: {path_ipynb}\n{ex}" + if not strict: + warnings.warn(msg, ResourceWarning) + continue + raise FileNotFoundError(msg) ipynb_content.append(os.path.join(path_docs_ipynb, path_ipynb_in_dir)) @staticmethod diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 8a799de17..2649cac51 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -4,7 +4,7 @@ updates: # Enable version updates for python - package-ecosystem: "pip" # Look for a `requirements` in the `root` directory - directory: "/_requirements" + directory: "/" # Check for updates once a week schedule: interval: "monthly" diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index f45984cb4..ac5b9fb1e 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -44,5 +44,5 @@ jobs: file: _dockers/ubuntu-cuda/Dockerfile push: ${{ env.PUSH_DOCKERHUB }} # todo: publish also tag YYYY.MM - tags: "pytorchlightning/tutorials" + tags: "pytorchlightning/tutorials:cuda" timeout-minutes: 55 diff --git a/.github/workflows/docs-deploy.yml b/.github/workflows/docs-deploy.yml index 265b66c73..9ec5dcd79 100644 --- a/.github/workflows/docs-deploy.yml +++ b/.github/workflows/docs-deploy.yml @@ -2,6 +2,10 @@ name: Deploy Docs on: push: branches: [publication] + pull_request: + branches: [main] + paths: + - ".github/workflows/docs-deploy.yml" workflow_dispatch: {} workflow_run: workflows: ["Publish notebook"] @@ -15,30 +19,43 @@ jobs: env: PATH_DATASETS: ${{ github.workspace }}/.datasets steps: - - name: Checkout 🛎️ + - name: Checkout 🛎️ Publication + if: ${{ github.event_name != 'pull_request' }} uses: actions/checkout@v4 with: ref: publication + - name: Checkout 🛎️ PR + if: ${{ github.event_name == 'pull_request' }} + uses: actions/checkout@v4 + with: + fetch-depth: 0 - uses: actions/setup-python@v5 with: - python-version: 3.8 + python-version: "3.9" + cache: pip + - run: pip install -q py-tree - - name: Cache pip - uses: actions/cache@v3 - with: - path: ~/.cache/pip - key: ${{ runner.os }}-pip-${{ hashFiles('requirements.txt') }}-${{ hashFiles('_requirements/docs.txt') }} - restore-keys: ${{ runner.os }}-pip- + - name: pull notebooks from Publication + if: ${{ github.event_name == 'pull_request' }} + run: | + git checkout publication + py-tree .notebooks/ + mkdir -p _notebooks + cp -r .notebooks/* _notebooks/ + git checkout ${{ github.head_ref }} + cp -r _notebooks/* .notebooks/ + + - name: List notebooks + run: py-tree .notebooks/ - name: Install dependencies run: | mkdir -p ${PATH_DATASETS} - # install Texlive, see https://linuxconfig.org/how-to-install-latex-on-ubuntu-20-04-focal-fossa-linux - sudo apt-get update + sudo apt-get update --fix-missing sudo apt-get install -y cmake pandoc + # install Texlive, see https://linuxconfig.org/how-to-install-latex-on-ubuntu-20-04-focal-fossa-linux sudo apt-get install -y texlive-latex-extra dvipng texlive-pictures - pip --version - pip install --quiet --requirement _requirements/docs.txt + pip install -q -r _requirements/docs.txt pip list shell: bash @@ -46,7 +63,13 @@ jobs: working-directory: ./_docs run: make html --jobs $(nproc) + - name: Copied notebooks [debugging] + if: failure() + working-directory: ./_docs/source/ + run: py-tree notebooks/ + - name: Deploy 🚀 + if: ${{ github.event_name != 'pull_request' }} uses: JamesIves/github-pages-deploy-action@v4.5.0 with: token: ${{ secrets.GITHUB_TOKEN }} @@ -55,4 +78,3 @@ jobs: clean: true # Automatically remove deleted files from the deploy branch target-folder: docs # If you'd like to push the contents of the deployment folder into a specific directory single-commit: true # you'd prefer to have a single commit on the deployment branch instead of full history - if: success() diff --git a/_docs/source/conf.py b/_docs/source/conf.py index 5d489f2bb..ce0d984c6 100644 --- a/_docs/source/conf.py +++ b/_docs/source/conf.py @@ -43,10 +43,8 @@ # -- Project documents ------------------------------------------------------- AssistantCLI.copy_notebooks( - _PATH_ROOT, - _PATH_HERE, - # ToDo: fix coping this specific notebooks, some JSON encode issue - ignore=["course_UvA-DL/13-contrastive-learning"], + path_root=_PATH_ROOT, + docs_root=_PATH_HERE, ) # with open(os.path.join(_PATH_HERE, 'ipynb_content.rst'), 'w') as fp: diff --git a/course_UvA-DL/01-introduction-to-pytorch/notebook.py b/course_UvA-DL/01-introduction-to-pytorch/notebook.py index 7ce6f11da..0aef3eb94 100644 --- a/course_UvA-DL/01-introduction-to-pytorch/notebook.py +++ b/course_UvA-DL/01-introduction-to-pytorch/notebook.py @@ -373,7 +373,7 @@ #
# # CPUs and GPUs have both different advantages and disadvantages, which is why many computers contain both components and use them for different tasks. -# In case you are not familiar with GPUs, you can read up more details in this [NVIDIA blog post](https://blogs.nvidia.com/blog/2009/12/16/whats-the-difference-between-a-cpu-and-a-gpu/) or [here](https://www.intel.com/content/www/us/en/products/docs/processors/what-is-a-gpu.html). +# In case you are not familiar with GPUs, you can read up more details in this [NVIDIA blog post](https://blogs.nvidia.com/blog/2009/12/16/whats-the-difference-between-a-cpu-and-a-gpu/) or [here](https://blogs.nvidia.com/blog/whats-the-difference-between-a-cpu-and-a-gpu/). # # GPUs can accelerate the training of your network up to a factor of $100$ which is essential for large neural networks. # PyTorch implements a lot of functionality for supporting GPUs (mostly those of NVIDIA due to the libraries [CUDA](https://developer.nvidia.com/cuda-zone) and [cuDNN](https://developer.nvidia.com/cudnn)). diff --git a/course_UvA-DL/07-deep-energy-based-generative-models/notebook.py b/course_UvA-DL/07-deep-energy-based-generative-models/notebook.py index 7df72cbbe..989f80b6c 100644 --- a/course_UvA-DL/07-deep-energy-based-generative-models/notebook.py +++ b/course_UvA-DL/07-deep-energy-based-generative-models/notebook.py @@ -227,7 +227,7 @@ # if the hyperparameters are not well tuned. # We will rely on training tricks proposed in the paper # [Implicit Generation and Generalization in Energy-Based Models](https://arxiv.org/abs/1903.08689) -# by Yilun Du and Igor Mordatch ([blog](https://openai.com/research/energy-based-models)). +# by Yilun Du and Igor Mordatch ([blog](https://openai.com/index/energy-based-models/)). # The important part of this notebook is however to see how the theory above can actually be used in a model. # # ### Dataset From c40681bf32f44aa7b4913046acfd23ba8e4f5c42 Mon Sep 17 00:00:00 2001 From: jirka Date: Sat, 20 Jul 2024 00:41:55 +0200 Subject: [PATCH 11/17] drop Flash examples [legacy] --- .../electricity_forecasting/.meta.yml | 24 -- .../electricity_forecasting/.thumb.svg | 1 - .../electricity_forecasting/diagram.png | Bin 98002 -> 0 bytes .../electricity_forecasting.py | 311 ------------------ .../image_classification/.meta.yml | 19 -- .../image_classification.py | 115 ------- .../tabular_classification/.meta.yml | 18 - .../tabular_classification.py | 98 ------ flash_tutorials/text_classification/.meta.yml | 19 -- .../text_classification.py | 110 ------- 10 files changed, 715 deletions(-) delete mode 100644 flash_tutorials/electricity_forecasting/.meta.yml delete mode 100644 flash_tutorials/electricity_forecasting/.thumb.svg delete mode 100644 flash_tutorials/electricity_forecasting/diagram.png delete mode 100644 flash_tutorials/electricity_forecasting/electricity_forecasting.py delete mode 100644 flash_tutorials/image_classification/.meta.yml delete mode 100644 flash_tutorials/image_classification/image_classification.py delete mode 100644 flash_tutorials/tabular_classification/.meta.yml delete mode 100644 flash_tutorials/tabular_classification/tabular_classification.py delete mode 100644 flash_tutorials/text_classification/.meta.yml delete mode 100644 flash_tutorials/text_classification/text_classification.py diff --git a/flash_tutorials/electricity_forecasting/.meta.yml b/flash_tutorials/electricity_forecasting/.meta.yml deleted file mode 100644 index 5c793fe6e..000000000 --- a/flash_tutorials/electricity_forecasting/.meta.yml +++ /dev/null @@ -1,24 +0,0 @@ -title: Electricity Price Forecasting with N-BEATS -author: Ethan Harris (ethan@pytorchlightning.ai) -created: 2021-11-23 -updated: 2021-12-16 -license: CC BY-SA -build: 3 -tags: - - Tabular - - Forecasting - - Timeseries -description: | - This tutorial covers using Lightning Flash and it's integration with PyTorch Forecasting to train an autoregressive - model (N-BEATS) on hourly electricity pricing data. We show how the built-in interpretability tools from PyTorch - Forecasting can be used with Flash to plot the trend and daily seasonality in our data discovered by the model. We - also cover how features from PyTorch Lightning such as the learning rate finder can be used easily with Flash. As a - bonus, we show hat we can resample daily observations from the data to discover weekly trends instead. -requirements: - - pandas==1.1.5 - - lightning-flash[tabular]>=0.6.0 - - pytorch-lightning==1.3.6 # todo: update to latest - - numpy<1.24 -accelerator: - - GPU - - CPU diff --git a/flash_tutorials/electricity_forecasting/.thumb.svg b/flash_tutorials/electricity_forecasting/.thumb.svg deleted file mode 100644 index 3f8037ed9..000000000 --- a/flash_tutorials/electricity_forecasting/.thumb.svg +++ /dev/null @@ -1 +0,0 @@ - diff --git a/flash_tutorials/electricity_forecasting/diagram.png b/flash_tutorials/electricity_forecasting/diagram.png deleted file mode 100644 index 47120db9c56a3a3a065f8e0e3ba7796597f8ba26..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 98002 zcmY&X?(EK7bLN~g*Y50w!9R*)fyh8eNJvW)ZAtn`0>$ZiEfE=Wj#8@P;;=;O)IQ#n#))1NfX7izi|*KU{FyEo4E z_V#XWZZ0n`_xJbx{QU0k?~jg-roO2h92^`T9$sBt-QC??Utd2wJe-}K-QM1woSdX0 zA@zm6?G0q>4dTSc#_kIjT3lT03*ixouYL9mO*-jYU*w08L>VYAJAd?7YD)6e0Y9hA9D z*7i28W9G?5*QZaP{#@UHs+#wDl4r_2s((yw&lKe5=C+r{4-XI1eQPJJX$_5tq^GBA zZfV}@jK%xfVp-fdH8nMpr&Bh)#m3IgT-SLxn6kdTGrxCSofY!!+c$QW7w6~a(b3Vf z4LZNKwks+s%DRS%hJQ`{Y!q(pH8V4lZR|?fhZT zJv6zt;rVso*T!aIVq((7tXz82)$t}`YVKq)|L4k@uCA_-u&|Ad&CU6ayn=#N-_PX! zMZWj%=er6LbH5lF83hba4Ai7{J2;GujdgW(X=-XtmpHk&x(*Bsyn6MjcxL|Vuf_WO z&kgfSVq#*_($a6=aU;GrU#>S478V++OMgyCI-hUesPkT`acZ31keA?}o11G}SYDrQ z`q`G(zr5<`=;&dm!`svymzH(5SfK|05Lfb*v!Qcu{9BlhMN*WT7NU7PQ~CE`#qECY z$yC8`w6M>Y@}$mjwXBAr9}PRLAsM}s1&O{{Sy_y)*__POWYx7jB4SikRo(N;f?N#k z{DOZ31Q^&k?fto!Xh^)6sT>IjNe^?#&5ZZ(@F*LY`nkRD?d`oYREUUjTj{in3A9~3 zKHXnz?);v0e|P!PH@|LduD&9)iv>?ZdKC7(KF@zDPeTYzWRiZJI zqZV^j$j8{IKzAf04FD4b60)a{qzFvOZEioaEh?{{C`9$U@nW^6jcn7r@1Y`YJtz8H zj$|kcBa{wiMuQYIN=HXmOB>2v6|^3-Rc~P)AH}z4+R&@Kbn-`NWH#Yjg0JqoiT--8 zs%swS-AQW7ztZx>OFv${`EKT`lj%4|&Ptn(%?G0Ityr_3ZUIoOSgGw)Ai7-*)Cy zVqDHkZ%~xNI#cqH4#-$fIH=0;gLfo}SwxVJVMl6=>2ErgzxUzQ(YJmIM7kI3axvTI z&oQXzKQ*Xknb{n4yzlyjYSt<5#pqP|%%`GmOcI$)7>x90fpSm<%;-wNPP*vumSPE@ z6b9IHK}^{M)zK1ILYd*nSNZPCNrb3sSVmP%e|d=09EZcp!=YJxB^SvJd62%E zAAgV@BiA&so51Ax^)O~os-&zIRk7y$s!Y#8i5gYdM(n@i7rcfyIhsF#DLvziX{env zxFJUR{gjO}AnmAj3op2z(eTx4T|KW?e@Fh`@y-&3eU?cRQ=4n0k<^JT^fzw$By(6R zNiRz4YX0_%yhNlN@PDY%UJa2bO(BBx+p7|a!lt}`M}*U{)jhU^o->AbvPyGks^R_R z&A;T~NupC98t4v;FS%8};a_@aGnme35{ibX?Ja)A;xyJO_(wzkP_Vrmg6^R=N^X&Q zQ;8XXF0wO~-Xs(P2^DVEJmOWv$9bY664F0>qzKMRbJRfKx(S(LH~ZZgr_Q{;w8NIi zom{LcaVDqzKdt{DP>E`0Hj12_!G>8-B(z$2AU>gx!2+v`>=xS2vwUFTpI0%lbC?wp5Khoe4Vzp}k=N z#4ML0`YS~L7{fohSmDOY_xrg%GB`@``%Zuk=pUZS{Sh4Q=6e)=O=py<@K>E z)hu5xlZwFlC2Y}ud%7ci63D-qBaY}k|uR)Ws)xI$vj?jQSBvf6{ zoNo6UY7AzaG5#X^Ptg8C@-cVXN5VZrY(~A8@JLNwO+)QYj4l-=2O1Gc;aQ zTZrfTzyoN)*?D^<$&=*!`=?{WnL|kvDN`4h;D6)z*NPUxgt>E!eWWs;Q%_xosYp@{ z1%3(x{Ax4!F4O%+o@N3gDM8uup8>S{F=J@1nv3w>2=TPeUv%Wka(Mx=u6S}08w;P| zQm1>v!TxBRJkGnp{AncRTsD*IFzpxlL1X9@tspLE4#l|8v7IkUV7G?lS}E`s?n139 zTwwdNHA7MxpG}((Lmx97@ve8`YsCPr6(*?0J#w1hI9B%NV6ttt78zIYoB0Vo6sIN( zc=pP(ciy!F+Y4T2t=T9VwdP+G#gL!!oMCP=dJw@ukS_Si@`F3ZBKb1M{FvRyu>npo z;t_`dxcqX0y!&B9z{cnnG+@!?{`eHxZ<}XbLbV4oe2_e^=CAH8ly(h7MNA(lH^<;X7s$R4s3l8JAf9gYs z$D8dl1*gdoc!Nj+heVdm`(iLMzLA7vamJ@;{zalt%Ra>eX~E8$MOofku4yx*_}#7? zFIlt-uxMD(?*`?db#o^F*VQ4Yy|KDIJh&Gqx@uw&r9 zHd)^!=OS_TSi2?gk*5Lq5sZo>dxKf)qC17K@HG~vV}0_(tvJyv0lB0gezwin3Jd_5ti{QgGqJm{$mtIV6Bar zclM!W2b3<~)uv{qW4=MIXbFRz>Shul#y@RrA!zRf+3#G-IDZJPam1efX2Ajg_^)&8F@iikHjzDh$hlQk!k3o_?==E*#Dn#Em$D#kIl|mi zAIWJ!jZ-%yzc2+p5mSgD|4k%TIz?fBK78HQrz&}d^TKuUy2Ij=?`Md_ic6BdipDKb zz6)g|cY^k<%LI!Et&|{MaD!GOtefS30{#;jU~2smycw~tr=3CdW}7iy77w^c7mwLH zVodm8V_QTf-U$acUU*JmdrAW7v7cwks?-0c>2XdFp89JDdB_AR-$8v9wk0#dkOhR- z(v!BBP1T@0QqTg%&)FP!hL%gS$yzRfD5YGt*ywV|*>4DlUALP1AKV4<;Pvxd&HFAm zR3+n7QD3LP@`$?Gq_xM|T*U;yQ}~LPfjkkB!cv}A2NMs#yne&{^KV zsjPnq5F9ZADU%Sr?-bO^ffzHo*M2+y6e=vSRAnVyxnxH*@$#fmRpM(u1YXK?kQ+)j za_Ov{C^9hkx^6+2d50lY*Hhs?lc?QFE{%~)Glo5u(P0TDfjcEoWHEa$L))TnV( zf$LK%E=q_6A3g}A1E2++zGu_>*RdLu{pN(qUR4IvA`tT5d9KSN}8c z?H@=N=?<(TcwfTV{l|IEkQKW5r>|d9rO=Bw!;a zcHw9wy;k^7*5K;{@>-PO>4`q-2E0JPo`%z&_a1a+;kSqKa$~Wz|o_a*Y53Srin-+H2vZdgapn#~IYb$`>Wa_SoMMmQkYTc2UCJA6*~Y{Kl`)Uqi%;=(Ll$(Ssch!*LYT=$h6TlP z=x<(S1%5`MP^|YC8y<6QoRB=nAUZo~CFDa&SJ<1*=!b@o0NF1=4!o9glRO+m3!Wt1 zREwgI!f=}RIkC_LgU6w|1exWmv<=0j?lD2JmFwFVI4nMJ~v`a%fLZPu#;Th*E9X}q&FA=rl zT}$#BX1HRe4s7)Q=*O;_fq?VW_3VIw9Y`KZ1eV#>W4XK!c;;rWvmaxEScSr3b02ns zu$h7veZ4bA-%8MoctG1in)%aX)BIc|zMBLYXZ)AJgOs3(f>!4ZdxZ`(Iv}v8o8q1Y z;^tS~KZrVh^$|57AhxcN{}VDkBh?QLXdUXyXTg(W)xA@^X{n-Y%f`^z)&Q7)j{#f{ zUaw}u7V{gbPbyz`M6fG=tXHD+QgW;%e1bCk6)LEOwAcLOgdtH!TF6`8+`YH5*dRxA zfl#4983+TJ=-c!o=$JaLeLy{PR{U-0{gfy^I6 zf#^~oNDLrB|6Ux+Ab)hNfq=ly9s&ja ztA?{G&7ZN{H-ru6eoc%XVMO6E7jW{4KMMHIKqNl$6VaCrF_=dDgQ`W_Q54{S54+aZ z=Gi!)0Zr4v{{xov+lX~to*c&b-qGi1K>A=dnPJvL&$^rnITKZKFql)LlNc*X*e@3v zo0~>-Q33*mxtGTv=S0qR}bZh&tT?OA~u@P6)Zw-KvnQfa1?jbO)EHaHtG3Z^{I zw^9RCKCXuFBA4@GnEt0aPd;Y5rWY&)@bvuuR3-ctN;^_j8XTcR`C)CcK!Cu6MhXsB z5DyF(GPfrVjretA7>AJ6M=WprjEiptFp+*3eo^|d+hr0_)t_z;mTXGY+CMSARtj$W z?`$JY?B$KFUn|gq&NlzC4kEpT{s%HtIGSRo0kFa(*93L{6Q9HZ3d%s>1b<)=!=+(v zY9blXh4!58GX=hq< z>-#DwoeuaEV*H;PPtk&6 z2!2Wb?gUd*zU+nOBXHkLojUp$6mRUTPQYmW84?!9Z~lnz4`R>e;Sq6Jf#72MRO2Cw=$8BG>MZ z_~V7CINED)BM}!6pl{eKR(zBu21n3HV|?!r^$#!~?4e`AB)4S6d%J^p@&Z2#TnG7( z5UwtcpQQ5Cc2A@$&3z(J#k?WBh##`vVA-@$-Ieg`SVZVP=5$)<+Xp#(5LSl zXYAQ}AFHrfL0}U#A3ZIhvlpK3Vk=zbbdVHyyZY-JE=j5+g&2A=xG`L>x&#Ac!~dl0IC8+R z&nR)nLK<*36ZHW#Y0m0XFYTu=_dH-761^I-)14wWcVUUJoP@QP`;VL7FWBM{j7KI^ z9+kLEkY%fqkJ5zdT)U!Dr1$#p_A=pN-n!sI5;Z4_otPNj*N~Lw$2dhk@{o8fblPHd z!Ka}XEvBZP8m0n^37Kt(lJM9rJziPezqN>dxUm-?JJ`yJLd26Zi3|;{UCp%J5Pf2j zF~Prnn@v7EhB7r!B`YfQ%6am#E-9YL-(SNMj;O;DqcU>wZ({qjBEuhkc|Z#D->yTd z;Xbdw$}@U%Wslty!zFHD5Zfyf-n$K>d?C5f^1+ zYYEjI`{6iQ$dw!K-7Td^mlSzX*x=#N8`lL8KJd?RVm}lR8Tc;6YQ*PjOH|dL_Ox~H zTXLxXAH4Y6$2}r}FSp@MtZOUJ2hOHj`=N6GWEm?FI(u><+O6bM%8sA!F`iYvX}fD! z^uWq-$!T4vcqIU^`pUnQ^nM?>x-ofTX)Jrj0VD|8JEPXI?obs%e+Qms?uDq6 zrS>Mt)qVe~lRwVnf2&5aX2;Y#u;upRE-6j6)(r;qcU`nS#O{#5FXn7i-B4$*95fs{ zef>#3x;K>XZtJ|8Eg7UDx;iE+v9wr8Tb-*#Amvmyz5S57gUy$O40$Rk zUB#D`e`}>>BsMwO%@*oEP_7awj9DzjiAOGq_StZB@> zmFlfVQ+7#gie~WpyHmG|d)>#m`V{P(k3^TXsx9tT3Iet`=@=r~d^HXzkMhs5llL*1 zEf6_vLBtRr$cgzPH#%leMh&n^ zjJ*?knQ(J~MQ}pfQ+#>Q-^b8ZUpPZ6aQfhZx)V$H^KRJbhD6&&qV1lum)Gq#uK%B} zIC-1Hu4#&-oLVm8HVr5*xNW^NbolGjE6`Df+uwO4fv;z0mT8C6gI+jJplooj=# z@;2#NrTt}LX{$2uZ8+dTf%;`}9)hbjsv7U&6<i~qFcqz(DiEgQ^Hx?6asBQQ+`u?t>1QrLxRHdQR>1@+u;B}bVvCu_63*uf^`34ht-&x)8ZArVEWG#Eu{A1? zt5ygPobl}z9Tc~dG}7+30M2AIZ@h8=po0Wtt&0pIw?~fp@)EvBjB8Y%llJ%r;JXtI zAC{r%(yS}HEOEg)nAAHBM|1|6f*m>6ZLwLwU@{d2<5}kcn(Ww4(#v}q2qH%o7ra^` z002sQnsix+11%LQ(OK~$5u={3bFCwXm#@G3SfF7Qz7Mer@B|;bw=+7`bK7kq4Z~5k- z9{r13QVh{6Q01~H&e6XeTfSm9c4rZsbgcW;JV~5PPp_~WKc{-F_mi#Uj71w2k#MWYltXf;Of#Ft9h zEulQMCF+8eh^q4(z+L+*#t79?eG z5eU?oFv0)j#S{#H!Q|CkBzfKZpu&b%&63>cATu)p)Zn)V1PJ092Vm>o+WMz@2ict> zn)q23Z`9x{OFXOdd^XQt1#3!^*|IVZ{g?U}wOq1$cQWlAtEgaKdQ>k5(U~0frO^1m zfW}WIO!~f1M+-AlGDgl;iC77EjK5oFhl3*ND@il!0e8H=?MKcUu-u2Y@DKMR0y|YFpeP`jXIID z+a~{r4SLt@CDYaxApRL47JG=6b5tKiCiNkScILbs?|MZZwJ8EF>`FEC@Dbu$Exi3;-&vr7fqDULH;iyaD?!W+wCZ!W z+J|wEzmWQpKp;c33&t0zWd28UPJ5oua5+Mq81?Xu{6GlKU>_&m=lKR*U8} z3PFTO03evyfU%4EYniaR1?Tc$j0hb1Aw5f6U)GV5i#urq{4L1ZKWoIOr@|YBCudN- zC7Xk&5$#EZVULVTFLSqSUyHK$%gCsT*h1bLui0+mx$0O+)tGA74)Hk@QU55N0ONpI z{5vF`y;X4Fm-gu88YE}zx?;cvBQnve#m-9%9L2Xp`Qc%I{X_s>^Z|>avrig-Y+_c> zY%NrN$YO1((+?f=5(Oyg-`S{e`K~}2*{H$|tL`~8EWEFIc6_ttc_SM>Hz|F#o0a_V z^#jH;pfP1Ylc_*t!LAm!?B~0E)tCh%iC>_XH+2v=+<>fMTqSSyeXO``uU&&jAMIbC z!~r9UGDImmUx5Q+CDg=PdseYU4Y|u18Rf`e`d9eRwTV3wv?3MYg!M1lGGdW`blrCB zX)+$wwwkFj?bx^de>z1u1?4g<;TNpx?slUD4y$!hK8PKQwnB`2yHaJ>P}4 zzy6gZJVE7y2PyjaF5NiRaPXzz7zT7DCIbJ@3Htld*i=u^7R~gCJP2PUkLWP8IoWir z7lE)LT-;KWMt>GeDn~CNDxYDd&FES+>XHN-$k^7II3gWLHI?jy!m`H7{n^Lq^-#^7%&@Vj6=l!ZUNas37};Xa3pK)gG$^nMmMv zT9#OyR7?x7LOZAM$5(L&$7bo4c1e7E32*-Nof9hJPg8JifCA8;0Y{Hx^^`gofHG0( z}dlkP8m=3>@AwuZ>YVf4y;cv0Lln-NLG|$mUeqGNM$;d3$rCv~}b5@NmDH z*H3@Ai+!qa@%#4fk=PCz^ZjeX$Gg{>l;S_<1!g|nDvaf&F7_0z33iDcn zLJbJLjwa7%7Qaf;&Y7YqSH2ncibsOnB|m=L7w{7nIDP4}2z49y;hLA-dxs6KM?;4p zAIMclRDZ8e-xPeq8uZHxW5TvIqf4Ay^Edxy(EaAh+Wd=s^oWgM=xtTKmRCJbv31gI z^Lwkva^-E}orgY`x@)(9_}7Of#b?3&^sE9XrzAH5SLv_+d>TJH$k=N*$WB1zh^HLk zv6b@RVy5)utqR8%a6f@H&s*0G4&M=Dk9wgDw*(4Y=4^7mC)%$*-DslP@;N@0hhGst z&NrjbmVIlXeBaM>9%gnJJ-n?uy0LNV7{#O?h48y5oHZ!R{C$runEmD{@aM6HzGwd4 z->=8mlL2Fgn?g4s5m-FAX9vRhnYvkTKAID!TaTv+6(C4$4i6r`FDz(<3TAF2YiRAG zGzw}6y}NIurV#rMMp)wCvR&me1L49e2`;pbawhu%WJY;1fM zkrr9hrws9gBGRq_g%$^XAKVLNY_SfVDGYbPxOQE+up6<8zdpr9> zk|=9$NX`L)2;645l})^i)5@Zv)f zXilxWoM7HhJbDse=Cb)(lY-O!4+~3{;EJ}-twC>|<_?Gnn8hS0{7mFB>c`~#+r@6F z-dCTg5>0UD`;Ev(g>BlFB5=>IXSF%kg#-zR3?9}ZDSdKFgD2ZYyXUQ zDgn!REJ%v}Sea-j;Z^_U5F(T;_5wDw0HMzkuA`kCuxP#=zALUp&fADkK|kg*;ET z!Gxr+ReuW)aOlU{O;0TMc4GJYag~CRhC2OX>=7MOv^(a;#8pVC}>JhI3X9+8^PFfwzp;JnJxHP2M%XE zsNFrkgFU@JMrtph8$!rd1410v)M#i?PG@5O3cpv3VD1`o)4w0^3&0>qM*$YQn);4v zu4}6?8v8hgCjPPe6s~00^5Bi|yJ4%}U2BBRa=DZA2vSZF9&5-gxCRP?4A-071xL&` z{J1Aq;{g7`wFJJ8$Rgo#&#rr>td{_gb!MmZeheKprJJ6+9cp=rKwUZDaL9= zM13~h6h+58_vLnUTbT&v(|7)2+SMN*y|>-Czf2WRaGvqeQafH*l7b=ueaS*O_c8zg z*u&a4s=+=}!BU?{Eje+`e#a38(K(+;ZsC8iIx0%DZ0!UUJCq61ej5*3-_#Zx|1jG5mEApF#_frrX~cAQFXhwgC3 z2mo%$w0p=54v1eh(ceX-^-;y>hYcTb1fOW=2jlrl-y1pywFECNJ--ylO+uR+(Xl~q zttQaqLvT;24O^0Pm;)cBiNal#FVw#hid^suSy!e?M;8K1?#tT3;QD$tBQP2G@yPXY zVBVuq>Uyw82V(ujcj@4V@&QgxkyL!3~WW-7O#M4yom9h2DmSh{@4c1v9YV^^QLq%y3+X$qvEV zO8YA(^k->5oK_j;_5$)S|5QS+rQm~md>bJoX2RcsT78wufA>m4VF$$ycnJZaX01kR>!V2Fs3GF01_X&i%<&YfT0Oj9iZK-Ro^0iG;-A$6r6 zv*s2I^QK-(y@aR%xGIQ@kfys%7j_6g-Ckp$`nTa=Js!`$`JKX%@$q81&m&+`I)A`X zv0sc4=Z%JDJ6?`ei5{O)2TKlS-oD{%;dLUutIw1-G`$ev=a%P!y|C_Ni3)S*hsh$gABJ^(W!?8otg!88~S~sMQ=yXo!IS_iC-QS`9V8X-(BdOh& zAF>;!Ts6I(L$s{Rt%T&py5yT`U`HEMM#5W)d#J!^v>)sq-<^Q8KWuJRhxl(%P?*s6 zXLLUz*xW>ZPtA&=zxBP*GD0SL8MQL!nqI3{q4%uh>+3lbuO%G@7K3v+gQ%L=#F;d)+c3NkL8;z+cp!4UTiXI7u_pFMV=Xs!f5%KFH zW72=XB@F&Jjh>;Wa4(?zIb7@a_B1(Ef15&=>qg;%r|*;qf5!l$p)N2WZoGFa&cmCU zYB5J$sLr_qN{x}cm@S8Zl7y0ut$*PUR71bv+qw@o?TM91`tcUCGhFQZRy9ZKX4fm! z*G&_JvR8tNk6&vtX{?@iF!~2fCCzN8-zkuF;8&lT%@o5JN!dwz!H19$GOlU4g->FL zWmIM{QM|h`aYsnab2=)w53ggk{{gobn)7)}hVuBZUW03E1}QjKiQHjjivzIB+U9_| z{~6H^J;(8k5Zh3resyjkGw7icK=eA2CuoNh{PVQ+blR7M{u$o6GYySd76TKNu!*G# z=a)om^Hcsp$+9mAxB-j#u}WqI?z!Povw{r9O2+OJF5a(ZsfB>RfX2`$u7=f{jU$)} zhgC}%Qx1j4+xPpX)FIFNih^3BpvV)1=+Sxv-0GDuF^%Kz4kgz$rD$N8lvxr;J7YYz z>;Or@Zg2f>4v1QLnb+#owJ^^*xrVw?JOXZn?MCx@@Wa;H16yhAWQ`T|ysM_U9qRqfIns2Glqo_l$bY`m-Ni(Nb}J)V93b=a#Oh7f)Pa!i z^~3#b^g1n{N?$d87rRL15YhIZ%$#56x41VB5&3k=}7869^PCj zO|sa&i)9qP!fiPZ_hu)YgChn=m6(rmMs+bvwK2~~>oEIH;F}fGblXg5#})9R1W z@0C07ttT~vW}4eQofTJLm2K&WMKG8kVOlC=TLh#pbo~lWZ;E3+{s@5~ccf5%34F*_ zjx6WOnh*bup{8k$*SS!-(QdeCY=OT9UxcDd?~Qxvk%li4FehBM{8Wi$G6(F+?8YND zk?3Xwu1m7PXcL7KoD{2&XXlWBU6Nk$cxgi{=t|{j?3h9udILQZ2W1;s*_|wi#fKMf$`8il^PDv7Oa({>VE3pL7YVZ)v zjl!gr0D9R9&l;3@S~@}st_0*Gh3tM$ETL?r^!e6ZtW{!`{|rd3JAfrhd^MQEXKJ<^8$uk@#OmwB5%)2o(!6{jt`|6V3F?E`wo0C_Kg`wvnx%gKc*EL z5`igGQ`VGuciDUREM;dfaeRPGU8$&Q5bpVTX3LhrUub3hJGpskS5xGgcxlvPdJCZG zK%h!{|Hhz+rD~@NZ16V#l++KCys4k;HKmQ^5Dw& z2lPNQ+ap*VW&=HFzF>s1c(wM*<$q+j;1pm=D1bxce`0g_D1i-gqr67EQ9-l3jMwK? zo|J=50b-cCk*4U-b{EP%CeP;(ul)nPzOy{7&pLX>5kSN-4i1@e&sPzgxK|g|Q^Ldf zF@7GM>8Yg+|AqR)w*!83=2yx^b>_ch8bL8p4>+Exm1{*`g9=g;o$u&1Yx>qsT7k&%EA4-u z_M^v0;Buhq(K4)uqA!mA^Kn2oPJj~u9Y@wiKymME@4Jgy*R6`8VHZ2a`wufSQ_8y4 zhmlZd&Yj$x8ix9j7jf`ps%41(;7x@29TEqoOvmd6KR=I5IjivLxgYUmcd&@K7k>ZH^RZ4bU5Jq>`X_S+>u~%q}*D~wP2mzCnL7$ODF8dmKdHo|1l$Q15aqa1> zfeyH<(~^#HDvtyh8sl#As;jh-vU^N+!-Et~@^~9eYw0h{>>CkDow!eb!%iB-j)5v& zY`!*AC>nG9Gu==@pC<#e`rPzQHHLC{cluFOcspC%1f7=cZ@E!7s(G{z*_#{1E~iUX zgm`qjq;lPeNq(u=>YN%yCytgAH%u38;!A;m>r_^d!84EB=oC83(ph<1eq#T!z9L=8 zU%w*?4^Zd1#0x;{9|>%u_PCDDoU%HCdCH-{H2D_M$k5{c^4C8vN05Uyo23okMk<6( zPsE*Y8uQ5!MO9>#eX*L2HTZx8!dPdmxuB0|czP1L?DDzv{ls=kTe#76xkY_iP3Tgk zFD!olxrdg=Yq(hfVI)VYpW~<&*TZmb(O`f5*?{Z8RBq_v+PC%1npDvsMg2&{U@Oa+ zx{xmFz<2H%1N-orRCOpK<7;RY<_|7s>DNjx8!CDztY}QBg_04{l=GkL^GG#^(y8;H z;^i*dLxwbG_M5Y4?kNZfAsso7%g7#{?^9Ai^tK-8JjYbElM%AV#VoAlXA&q)85dd2 zh?)}uMeU7WC@Ol+Cwf@OkRe5=AradvX4GUhbHA5L2lsn$`UlIGo$ast6r1DyNFaZd@#!n%Bg_P8e@|0s#DwaT!SYY0 zeU`CeQA(A&KW#8NHbtH625%Cdy^K}O&H(BW_A`N zgDGnf?nfm=2C-NDshN2gyS||@U&B_BNjZaHQ5_OYAC=y}d=Gve;QS@ACCPl;t=jnU z6q2n2t~_7du}^m9E;y}(9CbzRn$iT&;7>;syp*nRz7S@2a&q<+f||J1oSLGWBnaxU zJDQx@U%nSv)=@lqJTMNkHtfrKPqP%E{(Lh@!E8G(%|ZjMhg18A@Q{CEXu zChr%rtz`$`?>2jN%E_#qo^2r%$Aoc$q+k1jAf(-z8w8WoQrX8Zh3Ba#v9Jm?AA(`V zMI-a~K_1Z=WRX=Lzm>l#Y50D4lgKqp5Gj5t#(70w)K1o+FYAl8Y^^AqbFnxGV2>Zl ztn~E)h-W+W*-ue+G25uAm$xB7BE{N$Vke9i7w^PEm<}Gg3(xc#rky@0yS&6B>b?0< zwyBQOqMIH4CV~i3ds^3Dw=5syUC-MZI3$Zcdqx-VJ6s&Op4Kc+-oXNwUX+Wr za*#vDT^V`i&DTCvW6^*is_SVioxxvH#_}++3}sXm4RbP5&iW!?Zn9p6act59AH?)` zk{j&Bf6M@r@tH#N>X;N~%g`$)@z@Rw+8d`qAIV zl2+2rsOGaRi_L1?kA4u3o# z#y&JB#p4=X>p+wNKqX7V3*-e-6PuRB#g60>$qS$#t(z_j%~AzEA=C z2P`CrWKe^#))J6VX4YBw6Z8y}M}!hoXjuA0FkxjS+Ar=Cy_Iz5bXFE?O{M#I%=9(S zHT)d?-X?Wpz!4@2so3}J z8p%VT3M!Oi?=jm>()^ar6+>ZH3} zI}`v-H~!1_vwodRz=U{)_>9%^(23_H$L}VmJ~60_K;$1%l>CGTZmn-rGw@S~m#(6y zsoWF7M{Y!&vr6{Ho3xW_fS~f)({evy`@5E-O%lS&JZt_bDmxV5^Nk^EvUwL8!tP}v zp%i)18k5Nryl*T<37Qjw;;moFlw6E-73y6rq|oKhe(9^1R{JNE<)NZoQ$Woz2iaeW6RV_Y1Hh(ySiQz$V6 z-igM+`^R{~(^KTtw(tE$yQR$8jfXptG%vwBzGJHR^`9+pMY-Afe&!s(JTG?IXMD$m zZKLn+7Yd=yG%iaJezZQQzKYINh6)ama8;&hh(Za96=Bm9wUAFXAQ+QLE$7HK$+Uwe z2s6||(=jY~TtQ6^@!%Bl3hJNcuT!h}Opl|CY__p+k)0ybQNdzk-!${ZQRhtbhn;kw zrY2%kiupG>1UJ2tZZWx?29Y5p<8PiK06Hae`SB%{A_!^fv&-Cmgy(}5%Ko!v@*n4r zFY8DQ?lX0}cMs&0+VJD7^T>%;FV@E%3W)rjUA7f-^36_e3=yN+9=x+F-!H=X-j=m9 z_iR5J9)&xTJ7-m`izLOVg~bi_3~0jb78MqfM&auDp(25B1kzJ7PDuE+-&PMJ?9;RA zKek#0gEe=wLjeb~yhK(Zy0wZM8Gf$nHe2(lwynSR3Ho`(M6KHAB*T0U{5pxn+M!or zo>EZM%In`kId2hm{+oW&9Q0Xl#@p>B^Me!vMXgf4PfrH@NKI`Ihakr)@p44ui*~7? zfm7JYap7?Meu`V{9_0*!6HofDzj46mFh#4DKzf|Ummvc8913AK{)zj%yu^YvNgO1J z-i2e|32;JL+25r1@h#*a@RVY@L>6Ue?b^`@KYRSvMgbY`3Mb^V%*odycf0G%qf^+i znh}3y@I*oef)LR0=Ql4)Lh+TfkSKn(`3jRmJAk5+yffCE9CvPl*c5%vnOy9`)|z(` zwFcr$$!*x(qT(j8xSz3qWzqi=uHe&`1PvcHN$l1cjS-|Gk|;xb6WR#w$)2$@*@@}ge_yV1u5poZ4RpD%_(5||GA zB{C>ii0PaYZ6$E(*7-Lu4%Od?en5bk39qf%|BBsD~zVfn_eTM=XO2K*tq3J_HK$%DtGt;k82o)$( z@}O?S2ae@Ev@)N}tc9ffP^JL09*;GbgYZ#_3+c32oBhE~9y&?nF&&ENE7i!(noyd6 zBJ#Cwl?oIPKZcNv*9iDsa5~!io-{38z?Zoc&D$7=G&n?J<#zEE5Kgl}oqJ%XX`6~c zNOgiVo4NV+_*J z(W{P8i8uDbi;~F1JS+9A=u8of&)A|l^Zp-eZygk8)AS3&!XgU^wrFq&Ebea0;v~RA z5?q37kRZWbgF}Geiw4&OcM>Ex1osf!-QjHR`+3jz)cNn6`f94SYIb*eepmNQcTZn4 zJs6PzsO!11qn`ngI9n%K`V$*vaQQL2+n65?1bm9Rv6vj?Y!`ns`4)w54jn|<=jUgx zJ^rp-$7fbQE^0Bpl;MTA$`0EFYc9`!;9(s_(3xL-J!o?AHxZf~l~ei^Eu&rpNKnne zUPz3TW!2oQBRGur8Za`tq01l6)7h8g(71z>V8*q`9IsYqvVLK5_4%QgH7j`~j?egs zPs4nf8Un6D87Gn5SI$h_!k!Yh#EQTy2@eDvvv>s6ZuKyFe0m}cwZVbVPEH?_(n=4r zPehVN=38_}xsL_PTbxvvG7L13yIhfUOJF|vcaAHd{@N*)v!b$6EIm&dQ6G<8i!Cd_ zd`}4});-WK;q5x5UF*$AE*eX2>Br>zb zYQoopuEHF}^`>jQnUqf~2mz6e79sNDoZmV=$-QRIme+W?mjIEtag5L;UivWl&#>%7 z0S341DCT&4YR&|RdZjMkim!i9GL3&K2VzhsOt-MY%KfGt*?3^6#ef>J8|HUIH!Q#p zliqY~pJ@P2GZ=uY4xE`Itk_~w*S>KM_#zxjgywA06j8yP!Wt^ZK$wn_wK94bZjQJ= z1w_BW@#Y8mONL)xG1pJSpTRiIaUh@8g(42+3g21hLWr3LczbFX&FkXJL^cIfj&G=h z<@lbCR|zbW$4W1JLIqk3hN6M}_tYviX|xW^Nkn`wc?1i(?cu%{??6SYk_K^4I~I4T zDRcesE2tlyfbloKyQV8;JVVtGZ?#8vDlwBJ1T5PKpPaD2{KX$wWIC zYic8v6HAIP+lH>#hev1kc)faTCm5g^OF z_Tv)CRG2vh!qJI9D}W$_g9O9a?&GQn_c6LC!e6*(4PPdt8QN#ijcvh=n{fn$TiMlE z0v8zv(?1htXVV`A1!AQcA6m`q^wg%_w|O@2`}+NzNeD+_L>%msaK2;*1_n3IhJ=AG z1tF&E=bLvi3vo4HXkY|%tF9^@P%GJ}?0En%c)Z9WF&fL5?(|b4QyT#xfWbl0;V5y# z6&=LF_*Ir+u>W^jq;&9BGYU}FEJHwn^E|-8aJQnEm|YAX+2R=67nm>H!9C4l@d9 zQPb}?)03p{nrb|6Re_Q*#C~(SEqHtE*75V#7k*^$w->MW3QF(Wq@>#0ytf@DK2W&_ z|4QN1nv8&6WO+SVWlx*LoZZ?-aTaCBjmOweV(ZhfS~E|vRKkzlP#XX?In+b7EJs3i zb=Zs(1rD$G)_=LVe$)4uvO{D2JoNLS$-AM&#;@Me>(!ycSECdznVNyF=!Hl7`-=8R zIA@cmZ{;pMpQ!-VV6g9fB>-fQ^7u{;^qg;PpOI^UoZHW`=i(KkkJd@i$W)gu5xeWnZOOiUE$i#|LNjak%UXO!rA@s{%N9}7c++66h4JOL@KOSI&-iirW|8JP&B0o-n%A%%rH6IOGegzfOBmO^W$>LIon7Zn^Vf z^UY+S8Su>LHsiHiYRkqloMY7GCO<|xZ;K&o|IXuvDP7o!2-Gn<_k!pC+)?12Pxkh^ zlFChuGFcCbWP`<&#+)g?1412}7Y;l0?GBl4GvxWyqRVeR^P=)z_B2&ySQ!StFypWn zGVB7sC#>Bn>|G@PN9J{^{xA=#j2exnWIK!uXDP)crM6cqh{1U5%>9*pA-)g zg-AbXEiDxi_jKf@G?BiRX-tnQwN;p*B*k&jDeTZQ6l+X}$i5)mRR6i?8%vD$_V=)D zvE*!)PgL3G%{HSWJQrxGhIbIWv1XhzUdfeV8}LGkp2PhbH6H#4{f{!Rd9S55G4r=V!AfFMdv5PD-)Tx0C<+bx<6}dec6? z>v2&Lwqd*TSpoP(Uoy3=kIzNotS#)<@@}0m#4d{I^L=vZa8xi!3!yg{ioo^B&C3<%@f&HEUdzf>+>rd9y+lumM|24HOYJjhW0+68B{1a1>*^93pcKg2 zQhuqDZpug4IaiMIPv?OPLv02C8Vfi31SDU@4sH=0)<3N(N%syrj*pLI)PEXBW|4TX zOtVWUt6}k)XzTmS<;9idAVwE(#4Z+QQl#$hyMI~cVoH-;PNQN30qmX9BSLo7Vu?Zn zVQX}gw`nAvf;_6iM3B4nA8eP*hi;*+&XfT4cb%yO=oC1hb^E&)liMx$iV2w)6)_uF z1Fe0s#QqIak~D`Wkv2dp_2<|DX6L5MUaCbposQ3T^{y|!n;YYWAX?eb5pchTvX|eR z;nj)XrQSD_jz-f>L1~P5?LZ%+3hj{WkrNk;4eg{p2@C#XTq(4hxefPn zvuUncf$-iQ&tb1M7$}hPrZ56RSZui=zn07O!Mh(y_~cZ{o`T+6@-O|7W1VO&e1RS! zN4jF*e${C5iPPhCX)-^K~_p>rZA$havz*R-x%LG)64xs}&+oEv*3%b_g?wgCXsA8yYqQ&VxLa zi{tZg+9e+0h;`I+iZ;6}rMOPz55VGo3br4Br;R0$7v@>4kUw=B=gl>WP_3tr3b#tusbS3A z@C2Yj*lX~;IJZw(3@79Q0~1_zA&v4kFW>K=gREwvvwgBDYw6 zR#Exgv%uoGp(--qbVB4RCAnz0uIT%{ZM*A95uI;n43^clR*m0rXvSfNwSg&veCBmmN2Ho~1O{DE)b9OXW z_xhsz(_>e?@>-+JNBD}lY~*8EnD}ew*s^iyh$>@cS#$~mj`f$0nGRz5`!9RgF;`2r z>BYOdiheU^QGd1>_LIetx(}|s=;1*DO=E)2NduFLiw#FZ0%4&(jcEz=Q=M(gxguKj ze-%Qt9^Sg%=)g5mB)8Mds_Ez4wfnnZI|{YuR;)vb=d9dClWkeh{9h9!#PY*Y>T)1l z;uB&2-}*ACzk59qCxMXnXM6PtXFf>`{#^orU7uW$iei%bMd%2GCyN~{`U3^x56zQa zb$g{b6#y2Np2Sur-m5r@9hD00GzfrkGpAL1edc3-obQ%c+N_r+W`Wm)JU}eGD{`U| zhTli-QG8L1(F4WZ4{M5w9lA5kpUzC6y+{f+Dr0kO z`7k4Z3P&;>F8uB=pqBtXE2%l;N`mtr>ZcOqq2q zvkXhePv<&TK5oSdRIAslS)TH%BepVDj&ik`m6xD7 zR1_#Gx+p1xo$TjjV2j?Ht;2+DUTJxI=C3=1YM6ElrQBbKXlAdWOjUk1 zo1#Vx2;rK}s}z8EH|Oq3BOK*O&!VJPf2(@O>nbRf=hRsFMnMDa*pRxVi9#Z7r;d7q zTI)(m&3t2nUe~EIOI50Gf~bPJM&mb;M~46^#8ka6=xY5J-%O4LB&ITphT2|p_syHP z>3s=jW?6MCFq$PFkE_=LLRKEPjuK7RPW4T>10LFo(`uu;#0H~m7$E~r+x+;P{LPfS1FMl3jlq)PEh>-^4Of}tL)3)K+(TnF7l%p?B)O@|2hH@P^Xl+*K z2i#tbVER_pbbWhac)8uodD^6(n5D9v8&~|>9NP#q`k2HG(O*e@?421AKzF`$&kB-6 zZk6?i_BJiA7@(I^$dYbty9oWsZZTX=N%}!0hDZLU@vc7QzT!pJ!n>H|6TU3(vW}dz z+(v`$6aYHHk$|;T0`h0FF-so>I64{by?W-pYuvcdV4%Od-;X(~$vRhAB$$`{RA{|| z_vAs6A-l9NN6Vr4WZLpoP8Me8T*avJ+I?LY5QBKNt#fT-NHz)Oy{%E`4)$l5D%Hzv^jwcw8^qfLmIo+DM4ZVx6N6&-VBn7Ap= z+S(gpzKV*{&&LR4X%kPhkitgSYdi}6>iT|&HDqgZx5~FbRU_?X8(hnb8MD|=V z(&K#zT21VGdUR9Ytlmsljge9JgFJML9Pab)vs|q3rzh_Uo>KuCk;xT;`&epdapTNH zcS!~;q;4h9|Iyu!Cw*h$*ZMh^^hSSm-edI#$x3S%1Io=b+L=OZhlzaGSHWd9?^jY` zo#cIMOy89PvEs3fyUdIk`EZEY!mD2cs1#vC;YzdvSQtVNIJ0JN+!0^XEidml4l^6I zs9gDN1TyDN4fy#!_@;VMj94B>TxJ{x1%4@shj$zyobjU4TOM(KUa|59U5@zIio9x$2!w{rb_dhf1skRZ1b-%8(C-N zsHd+SQ9yr7d#o_e8XNG6s*7#<|!ibqh>AWy(S&WcQ%*EBPOh_ zP{PtIdS1Js7nJ@-(?q7p4izY=z(9$U7wyE&C^|tr^57};FK@ZOFIPZ5zH`U>3lIe? zoZ8eM$Aot#nfVhR9v&lgqGs0t6W5H@oe->n=}(r@S0=WMtTv?A`vVvzzN+WR@$u64 zqrOYdL;uHiTt)x27f3qd^qDyyo;#zd9BT>u*4$0}c(?ZbwVW2{xQG^AuL?hgw^FW{5sPZ$Gzt@5eaXEzg znHJTO4nD`TDc4NtbM%AA8;QW~nUpcCg2P#Buae4=%VwEA0Nq8L}rG>h*E6gc0e zt1Y_Sh>J`_#!!8c*7Dzi{+=2V{)TY9mx=nt8hdkpvxpuV3s$lu9IF!vd&_SKAACLL+W8ZEHMmdsFQ~8mA9kG?A;JgFsE{*YNaz0F zIRM{61x0Dos@7JB^Qy-E93|BFIic`yY;6$z1DRMZM+iz+o|vyxVRu`D*$A#h=dO{a zqtL2-6VIoqY}2pJK|u(S$4w~1h8ipG)47OHLd0p?T`wv2anC!R1PkuZO~YS3l%^{j zqvxDHAm0_9Ogsf8bMl?-D@jdmUDROy7YIs)SI=tt73lF;gCWSZ$ae3xNX**M!hpUI zr0)%{$L}4K&pkg!daPh{>83|ds>nhK%o+NsRdFO z0FxIglK(^iAR5Y+tWTqZ8@fDm1YwBm`LuUA4;AGEGV^yK3nA#;gQ4cmkad`&MDf<- zS7Jvpkxb{5nvc2=EAv@wqj~OJZXS4_AJ5tG9l$Hg6eUC&YP^|KaIWMVjsn9otwA16 zz>+=pe9iH@vQaBi5Yvs%485&l@1ogPNT!5$lCQaviq6`}Ly5P3>yJ{6naB=DqRRxQ zpnhy=>dynQW-s>-?KD*TZ7$yrYp-nFE{0C0U*)F03$EBmu?TJWLwQ}^Oa>11G(pet zX}(YiHsr5+?8@47F+(#JI+6GE3E=$lU7ipa(qK&i=Zy!iOg3+r}_jVMI3n=+mYQl4tia&>X)M z6>Y=dm|iokyAbaAdntQIF9|d_ib`a@`nO;sZ6l6Z%FoxT%Y8BmMz{KpG%U+xs=huP zBCwH^j$^|N#0|i~7rq3}*;+oDz}U8@sIHh4|G%EE+W;W=26f7xy>@xJz zSn;Z3CvQ6Ah!Xg^OE4K)?k@m2^~@=uDelH~E`p9 z{ZQ;APwn+l?|qjH5xDAw2=>{DCxQ32;6r0971l$4c{}s%%yEl?o~@AOliu3s>_$uU z$+fVoQM~ln!D0E6p3 z_NwElZw5iwAj{80_FE^4E^?5=k1YxngrqB}dJJ!!{JHqlR=7r${R07R;A`lYwQE9d zm*)F!wT19Q4&8GBY9BdDydO9_?yC563@MvKVdON&1=Zs34{k@Vqdc`)@rO zFE@J!iB)*BrAuz*@<70l!yV`ePp|V`QW#|TT9i0R(uA8Gv_E{{{+NULkbM3PiwuxY(PfPP(b@AKpo{; z)=<9m{tbj9LNF+c^Sf@kr!7*R3j<%hyk;B9=Xb}-n|@aK_}b%H?!1_0 zue0JN>@#bh9Fvs&Yxvbz5aR6to=jNk?YJFD?JPQ-Kre+wdw;Af*wC5_1C-YKwI763 zp#Cs1D5{t4hMsAv#cc3is4=BXkadP_ka-7c;j|JBgbpX*CqDjRAAsiq_X%DIa}Bkj#lM*P!2d9d0})qNV|72+3FU*s zPw`Urzym`5u)DxFL1{Yk#Ee;8V;?frexiWfA|O||4|($9p#=diB|cdj>P0NCuz7y< z(!`fnJg4*&HqY{edrLOSuf}ckdyHhi7q4{*$x>(Pt^E=5(wHSc2yGjm41RsL~$iHn&16>EU2Rp zwEHLdQ9+S^IzV_3yQdHX*LgShsc!ZY*10ksopcN9x31u}kBaQPaA=FY#;2QvGLi>s zV`@v12>bG!qAB$&d$O038}1jb?e!~{w`DPyxBOC4tGGFo}d(&d}A=i(CX^{soRX(yK%P&jCOuQ@T+};)b@ojkMOTBnb%| z($X_?`E((D%ICZ^zn_E0ov@*uf&@T~75^{&Nweki;~i{#OAP$XJn)s_?HJF`ubSWi z`Z2^9pn)B1hZ3!ler{ACl@O z-VGy%neDWlfT?JAT~|iTAFk%Ty8V`L*hS{=jS~Wjqfrn0=L6!61pz?-9g7u91oXuf zwlP6`uGB3hyY~$CXwspP+yjPyX!z?F!fRs%y^C|l5cdwHUCj6<90nEG?4NF%GvSbG zIYa?v3lIRpF;UaB;J*x8GEQ#ePyJz6mwWM(lH*g(=alTWB|~&DiHp}5Df z4{s)N?tBespU4;=JeF;pX@AL)v3?X5cHQwSq4DfgTuBaz24fT9t!T&0-IEwr)4tMVMBE;5(_U#uRpn5QNu2_Rlv*5+8sCAVB@E@C~&{Io64729I!2BcnU;1q1 zIvDV)X$)Pv8Ht3SU&JRzb@%nNOT^E|66Vbibe7~>W8qz=H)}v27M-R3^5l1Rd99K@ zOs-&DvGiL*?@nYu7X_>v2;-$PnIx~Ah~~qc-$wDFx2Q}F!+^)Fd%x@r?iifYyj-mF z0)WegbG-axrq#C$;8o4qodv5>jc<|RJ`@mA1yq~>&;ybN>y8h6I%S?{39`9Zp#!B? zO37!Hd!kFmed-&RP(nCK{)s^Xih7Yc!vN>l;~~1lDNw4miwI;_XYPy!LE5iqqC!)_ zxhBy6jsffxEwyUWKpc9;>PkHaE2Ey7J#QkeHyz5oyn+28Cr;*CS@a60{Pd=dLwhC zjL%lSPCcjrNIoN%S=476mD3Yk54_nMI-&7NVS3}d7wHacjWwd8#!tErv18gU!k&DI0JmQ7qMlLa-7z&wZ&YuI8#QLG$Jb9+L{G!KU7|7RA zBC>v55(&jWr#Ua=o|ZW2H0YbA;^#!|E0YOKspHqhCesI?D9vn5?TIPI2A?VZ^MT)Y zfjr;fchdCYGZ@2c@W~m((bZb)(G~y{(J@$_>7)5kV1NE!ajkm>P?x4fL}J(ZjVAAq z1vM;PukN&+36B3~KfvakceeGvB1Ss^cwF6dh5JK?>y4F98ng7w2x>ZVrH70I!fwG! zELeeucP*J>O5&v89qGRjNBq7q1TEWq3yG-_ty@)Bq^Qx1!v+m>)u=dXG{ZNa_RqT~ zAdz6uz5;5n>N$fDjJi8<(=ayHx1yO30bgotR|8*5Kg~SHfQX-pj{b_-MI_Ds4S}l$ z03IUSwPQ!2T`siz-6?E#^A0J`Q9`mXLeOCYqKk$2qCDarL3glCXpZ;CB32epGGb;Q z1E(LZ43Mw9Vn}b`mk1m`L1Cu?&&Q@(rr)<26%FEo17UxB0vDDos-Rh7{`~u?6CN}tV$O-`KNO^q6i8c=`KB!=PlG~k_z{m(%C1%*CWRSJ+II{@Ocw1*tUaAs-gkEg8#jVH~22VNZjXteglGwLN= z!Q`tmqRL++K%y(ZN`@u_fea4Cq*Eac+71MQ)m4#p1Fa9gvF^>6K>^MOz-F_XXT{bA zbaH>3Sow7tIpZ9_Q0tY9gJwk9YFoQA+6RYIx{bxDxo&J3SZrDCMWRMN9xN=XRB!u{ zH(Lhranio#!2n+XpMY~CbU6qFGSVQ6(~Tdp!8Qv_<%y4DVm@Vi|IV^K$l( zdrmCsdrk151FT2i?mRU`|M0Fmcxh0pk7+&t+BVW5{$;pWYas2rmH5u-=KIp0tRv1P z!%Fvln+3|Y=EO~45)9CiC%ng2^ou9H?jAabar{{j?98a$w?=VwZP_>FSWp2$OAd+R zy2@zdr;$BC`yRvZ|BNSs1Cuj#*e84S8EK)z7Zg?H6mQLyXel!c>YnM1Hun1Wz!jdI z9T|UHuRL7%SE#YKn8p9F9J6hZ8i< z;{^4ok$q>W4eZ31kgaqKYJl}B@=bGlgtG;_w`Ez6tTjCRrwj2&qdVxLv%Y4<`Y0# z;-Pw}KFT;tBT-)5G;W9;Xaed|obNyClTh5ZFufrWV`V<$&O{#`{&{Gpo3-g)0mi8TRN9 z@mlJr+|y#riO5VSasvbgL^jXw`sOfp+4pvlW_iFPodF&P0#j#9EsmbO;twzNC*}M$ z=GH78A7EN)StxCG{mxH(bu#6`)Ul!GDeFxww~Ly(aJ{tjpo%b`CFuqqf@Y5HmJ+PI zQO9TFPw-X$?U6emGwM~C-p5IgfHC%Y)>O-{@d*6acolhL*S~_|Oqk)c@yP&Cn3T`` z@kDRC?;}nCARusrmY2B>Y>#iPsCSz^I2&w*zpex?vPSSQ%-RdD4FIE~aq}tls-6{S zs=S;FTD2aPV|(oC?<&7_^^tpqS7A5O#5e7!ibE{pHVBpR+Vf&}JNN0va658`VH!kl z)EK4?8UGlaE$gQ{OzI=Ov#hE>9JP3kW}t_p?)*n=FHl3QRtKbfAFg+~ve#RN2q5eP zU?!&Ob^Y@8y4m9x4E0o`8tn1(F&!8JMWS+7>>GREdDru&Am%JX+r=j!oskMRq$Hj^YoEY@(3CUoJ1cuqA1%|u zvC1%s5ZtB`SZseMcLX#5YU~OPxMafghdq$Yr+duP?)h#-St+UVh4alH(|9C~r=uP2 z-cTG+1MwiVFx;Cu)fNOo#Or3gW5o%GAk~H9xpCe0TtLvL(4-)Px_Dh5|Bf?y38iF$Lvt|H*NKR7_h!Xvm1f;W!Y*4YDy^iPRxm>tjvjRr zy|#?!xFqK~*Xj{+zpTnQAI5>~;7-WGpd6yiEr zoDsM3%0;w{@JRybi#N_2kAccyHvS1K^`eul#O@ol*(b2TsUwRDhUUENV#ERIV+ zd~0)1Br`hDS9Hee{J%!u? zt-8;Dx#J!Y_rla`Gzm1g(2J)eve3!gB>RWOi5D zy}F$me1Qd4+|BJPD^ZA9UiGuh_q1zOWv(f$kVmcNw6uI77xZ>OL0@vx=AGSzLHPd9 zWDnWMA(g**4fM1oBkTw;-B-$iD{*wB| zVCy1z-lbwL0177$u62#-=}6vboTTz=9;zdU905!)2x31hNL(mf9joZtF{bZ91O+(~ zWP==gCyXIeoE}B5{M%}u>D}NRbdRYGqU$VBK86PX^hhu?`@Bw_r-c-hs!J0i)o`-P zC^3FDWZl~ps1wqVJ2r#-qybZTho5KRD=W6cA8dBC5fXOsDEa z3EKWr$HeNt~i2`xL^xZUkRb_4|=t3$hcMVQ$lXr!=~ve2irk%W(uXzvnTu{yyifgIU}}= zU6j=Mv6xN$#ZJ8-3)Uk~@izkE$DQ|CS8t<=YCu4iP-@M5>bq)!yTEY#Pqb_h_9WuJG!vpe zK@re2Jjj&}9Tu2UIgSjR+zUc_nuJR7%CvCWp(kMRGHz55wQ}4uaPs&6{)Grksd*F_ zm`#|E8Dd_A)BzE)3GsyRMQ{Xv?U_I85A9AM2=x=n-e%?>4W=#GrBDC6LwXL`8;8er z1YEtO)QA3F$csAZuT9`k7mH@}Al-hKWPxz}QvMK;u}_Hfm-3&CYMy$6*cg>+uvFgY zeG^yXnI2yLF&ZS=G^&~V8)&KVj8d*x0ZHK9$BRIn046@=JVB`KR|W1Nj(Yy`fR9Db~ zC#0pT0$mVQ`hSHGuXs|^MQwnT=j>$0Nm>9L4lNd#RW-Zx+gnt7!MaS-;UvkQ-ZWc~ zuKSMha}pbuBJCCNf?OOYKr3OTM`4sx4qrdY#Gco2wkK?CL((6ia39x?5nnHWqvRm34hCx%2Ln+2;j~~ZAPPzj6oQYu zOc)k`tfQa=1CVv(wckKw9eI5)2wDHz0$E4y?L__&*%0}kf7+q^e_#FI(?T}>2gARB z{+s;&JFWl6=lP!wv4H>Z;a^<;kF@@O;m!ZS=l@Ij_zMYFWJ`z}KGkR;{8w-tCQM)T z$3S^&_4gY|e>7DO!xv7Q0{;>JiWo$b@sq@KVp`^KWjhh5(t!7$?xyH{*v2bp#PmKy zm$8QU_|3h{zxn`M{q_Rmuc$h7`Q@6dot?ed-^)Y(!@iuHK6S@t)^P^#zwrG>%Nthr z&3cwkNhOH!Tu1h*xY8nKWgfgGA0J%YCi_!<=TvQbEWL;9xi1y2xu?WNcgAEr7#-KI zEf~c7eB*4y@iTo{SDSar!Pd)bmy1*KTdNNg;{El{%89XNRRpaxmZsFBtZR z%okoMHcaDevX<0d%4Ip2IsZzmS4n#LgMwH*ykX$u%M_Va?R=+SQ>W8sMRIC3w=&Ml zwFPv&s~z%o{qC{P;wOaeVd{)Dje2h~FAA6aY+hOy48Q1>Q;D~Qt(qDzf-YXJ8wz}8 z;??Kb9AJ|oAon?=QvKB}hz(qse|5W)o?Xo%^Jt?qy`PS}>^uGJHAjO5eD&ph(KvPK zBB$aR_n4gJAS;X+iaqfGQ@mbDa^~HMGReRDg{T?}fu1oan}FBg#+zeaXfBhfaHwVj}t+ zgrMqU@ocSy*Z8Z;=I4)GLPZOT9ASV>i1#i*2zgG`0bH0X989<*Un2bG@JXtV9pu+9 z%y#$VRIZ<=4&*RY?G!%euj=F*ha^&2m-J|S1#gGn?*ZotzbSFeFMe;_32r)l4H4hM ze#HmK@olwe3qJUmfsafl9o{w28yno!8`||l`wx>aqZSS9_Qdx!d?D?&Kk((CJ&dZB-3`VFe!p?~L9U4jyf+$T$%G_4 z9uD{mnk?dMgBg-M6n4NMlcNAZdHiTJq&k!z_)uuPy_Q+-2zV%RwMUQ!P9S;{NcQ)zhdtGkb`0({^=#x>eu( zqAN*egl}MW>{U$Uk3S!6jXwq1NE$6i2E)#js6Cdl+MG5$JTt@$PFfgqj%vE28sY^W zWfoO_!00YEw7;EXo(_za=`U*&fVNGlXhX&xRq^e zl=4%oTsV}{oM1~?c|5r#v0d;e|tb&NMJc6gSmpF z{Z^1WItCY*~fyj*I#j?}Y>_1Qgy+Pm$Qc;C8my=&IJKm2;O z!@m0%nuFg?A}^J%r><6PyR3Z7EK&b&mu!JE4Lj4W6sE5CKO-Ca!)x#B&7AXo-eDV{{Hy< z3KKWv;ycZ9i{CXX1UVyOrSk|HSBx#%^ujX+A#1*QIxanGA+S~5cZ5B9l}|$E-zG&g9y zSNG$`j#q5Wf67D^j0wHn^=|eP`7kML+5*qgC!cJ>BtlNI*6yh1D(NQ0T&@P3E<`s( z*g2&ozOX%d6`pqb%U5vFCY#Ykpe-q`pF zkK^u1t@c3~Iya7&V(xuSPeOh$&k~FLYR+@?TR%YB6a?(u?lxT8J2geKEm6R=*l2TS zB8QvXk~UROf-l>kz&xE&RhOfiFOTGAFW5|Fp>_Sa_|6^wVKgdIeGp8{6uMD7Q*U_c zS7-Wkr?8W0iROMEqc>4bh%DJx1Z+03uq?QoRe}cMqitKkyL}}@x+$tW~y`Q7XCN$I2V7p zpFsHFapB}>{E7`Xq#(t^dR?sCmrvZkNQw&7eGN5L`=a0VR7zw! zq_#ec`RdxYR)-Pn&t)vj zEKJ@Li|{VWs*Mo%)fK!U`*BTQ$tQ4i9Gw`HQbJ_^YIVw6^l`B-B+;SeKn}PPDuq63 zy+hR-WR@@n4>!O`%$E8Ry0peF(Vsw#fQL0UoBCnq=hjF=TDm%GbE?jGr5aB%7yD~- z;xe=I^ruW?u=&IhVab6U9RW+|X6M3zTyIzll2Z^Q8I21WYRpL7((hW?+%{`+B$GuR zCvM+f2<;qI7dEdny@hpJ&(eixs4i%bf%Mk6Q4DcI0^7pvSJEnJ4<9WL?KEF$&2taR z@aepp7@wP)i~#e#{{+;)*N0hZz%hXexhvlp95v-2!r=v?WB%Q$;<(72s5uH}t?kzi z$*z@6i$;gGRaz>h|BI@(j*IGh0*0w279=m-B`b||DJ(4rOLr*>NGc#94NG@PvxJ1C zv~&m}lF}$8ARW>m@m}=%Jiq7ti_db;oHJ*7&b?=5ep|{md+RFN>+`XXW_Z_kYf22X zdwEsBU~2+RtgbyK8RWrh9{mG(SwBCPCW`vZ({D>@KiQPB1mnF;#Lwm#2dG-Ontan9 zGebWl3emL4HV>*>bkM@mn>=aq_HkaxbMo*(=>%cW4_`Bhy_Z`KPcA=&W)+6PzluK0 zXCde&aAj&Vg*YYI$3M(8B;j4^Bd@W<_G)62NB3`C^pEA=T`1?=u|Da0`;^pxy|Nx4 z^6)Ym*LSHdOxmjY+|Tosvj+aTb~Q*|BZgMnrtNhE%pl2_SHe_v;<@@v{y=MK1&`KK z!1c?bQ&^Ys$p$`+_mPdgoJv{EsxZ99Bv&$?p@7U$bZ<3+=#j<57Z<($umzmtI2h<` z>vDX0f69DFw7RgVi(>v`lz8)EBi1zlR-lF7h_?3Nc0+VT56oYf4w$-*$>$bG(iAY) zWBxgYDMvTl8H8I?@pRhoth$=mT|qU`Ed0b6FZ}wo&pRG|{j%|hXtT$qPT(eTZF$V` zNb)LIU)Yw|&b3tcVp5q7<@1+i_Hg8xlQBHIAJmCmtJ|k$ZMV8=W;-) zsyW80;oR!Z3dQ%^4$C~F`Hq^@Tz`+g=jPWa`}W~>L-y%G2Pw7_2EP#nPvIlh1cls1 z;}HAGuq?Jpnp1^8SgP`Mgf7HkU5fiBA73(E8Odsfnf|ibKc6d@rF&lzL~W;YSyEqh z=}R%aH{l)@X;wLeqiZxZ&zxm|H)E!ewU^A3O%65zZ=(-z13C!hbL~fU3}=iAqNcIg zp$+u|Sls&4uS({&@@R0|=UOOJq{cUKeA2fPA5lPFLxny2mNSWw(<%o7uO>~bJ_vl3 z^1UJlb<7~U{A!Ys&^p(U2$ab?4;b^CGeP1bP z_)7Ljfi7ChT6(|v6UD~mD~h>Thz-nsB^SnBU{zwepfK{3z3fXuH{VRmCEb{{tn6)j zdw(8zW|^6@XEiiuPkzuDScT&_S_R#l^tG9}d@kHDeK(89AF9tfWzjPPFwHow*T;$E za|hT8+Fwx6jK}2>OWKVV@XT_2x>>p-Z>i1jiB`Dj7d<1B&BPoECeP=>ea0O!`KI+S zWi7ySH`BW@SkSzBrYsi!g@vp|JK1Y06GJ%Ow&Rzy<*5&b zNn|45{tuFVoY3ANuZJja*&vedi{V|RJsCNM6(fhtfArt8G)}ZoJ`j<8j(w9A%NQPL zux{RhdX?8Z)iac?FZ1j>e1ee>M|0Z$;73lDn1ESe9&ziyhecY`)<@crO0dIRnAO{+ zD9gLt(B5R;heeJU+Bt)AT6iJZW?8De&fhX2i@7q5ORbgtL5iFAoU-uX;h22YQiMXH%;d7BX>+Pz8J$!QTYoO>b z$NR-M&;1cyZT*waa@P;~a;5A=YR#>m|GwXnvFxkE^~<_d?+ML?@V`Gl6+hB6_`h=tnB0Rdl?#^D7$+Tf`7WOOOUtj#V^6*?G{?2bE{^-KYw~KdVA1Lqew4e9nIg3+qpS7!Q$^r?_b+GZ*w4--uaIu39 zHF_HDS4CZDLCO$As_gFS{HVV8XlZ2V$OXlbkG&th(oc*OGaHsG*jxI_dSNXR^D#zX z?8~585Z+&EypC4=Rd0UvGvozdc+@&|9&yoTWY^bhg`>kf1k2Wcn_{M~_T6iSNfFP< z0i-}|!Ue+#b6yT5}_-xz(M@zwW z)T{-2tbf?+Ppf-QrkMc3bd-SjU z2I<%096S;@z~1wu>z=p~#sbXJR4O>Lbs9*8OA(VZK}mj~ zKyFiy_Mum6YI>x^)Al@}n-ipZT1c6}O$YhML)9P}GYO88v$ zN`13fo$NHSK{G=Z>n}=$$>e_myQ^htU>O|`Ng*k0zO=nQW_q56q`7oAmiAv=0TQIe zjho1tyM-LUmYW?av(SA@av2?hsCbQ~O#ude}cP<`ZL*&h>wnoL(IMIy}Pq zGBmKP_q6qkJ9q5(v+?sr^yQIc+f`1bI&B|0f-(7}$KGarR-K7q_N?v(qcb|1{Mp60 znz-6`BD=m5vVx1ehXp^$ybHBoWB99;(}Hw--Bq>UCdr%M5$L;Nv$v+%izL5uG9!j_B7!SLEHt&X(WClT*|Ddq9&&{~PS z|HDP}_5L?7dNJ3=&0`8Xk_r=Q$q(x#jJpe6qACjSVrL&7ytO%+Xg4g^f_|v2ZLT>Z z?y>%Fscz3&9n*jBmR^ZEi4d4Lym07 zQV{a;*U`lNEcO_f_E`dL3+M@@gPzSH-|DF+e(UdZIufNxS|%fI;dos!1L?_M|#Fs0-O_?7l+(U=$!{- z`vk}2pJ|euiR^X9+u<*)O;U&4JE3RfAAF0%6>4t))L01Ia zR8h$ZQM=zV=J(sGw$zk_ryXod;P-?S86+TL8cu#|=R0QUif#1OxY*~=P38aGy^&KH z`EyiD%Chn)AVY*3svnv2^F;}aJ41en5>CMkC2ur2$qaohUwetozTx+c4T|&BzNwn6 zclT1@*4+0izbi^p_~Yl3HmC70t9L#Bu*v*IQ8Kxf$MfF>*F1adk9KE*DWi95+5@dM;Ls+wkSw04d$k5A0=ZkJ7t#dv|A~{$E;cUrhS>KWZ70E^D^Vk>sKIcm9|?X-g?&ix}Y70SvA$tJzdZkCBVsh0W@H zZD(gfNW>R$U-B{NT%d)TrqlY@fm$vHnPHm9hJKQ2^(_abKfr;b^f zr=mr#LudoDB%=qkVt=BZ$Tt+yg}}V|C)w0Mu7P-zUMxKZ zX)T1kc3Q&PS6$5NV6R*TR|uFPs=R7mgQLC&e$G*w2s7$fJ4AZY#qb?ty1f8vE4R+N zpZEV?%21C$Q%s|8`YX9h)Sy1alQJK( zRY}l!s9Sqw83kseq{vE_8(w3jd{pcsQoDh~v@^_jOE9lmb=IVpwB_w*44;a3FNK*5 zXe_@kBwri+k%*yE`SRVLw%=MGFL>SILsm|^5jn@!QlKdtl4q?R`*zPHZ0R!c*%MYSAM}5^)85P{0rl~nI`#{THZ>Baa!Yvz9 zJ|BU*-Q5CP@sMWtfvP~g|6?aT@uXqxaOd%%=-s)eA8REyUQv^cD)COaTxsxrG*P5E z8+zw6!$TFdcHe7u?kz(M$xlaHA$8GV(%0mL zd+vre0<$E3ox+7MzmM5=?_3@j;d;suDQ3;yIhvAof9vb(tYyB%yIU0s8>c{u$d7vm zRg}zpXq#y+%H*AT!kbj1%9Z@Y#P=CzqCOP+IoggaGa1%FPGbDbeal=A&b7eWF7SS$2zGx`v#RFPY)Q$fmrJ zim;Rg)E6pAMoI}bW@tyfnq4|mWS(^*rRt&@P|StJZc0AM>1U1<);>%6>1Z*(!IfG4 z-R#lH^whGKEnxs`oCF!8P+LN^Kn;tR4a2BnFCj4{yZaiF4T57Woo!<4BCnPrWTP&; z&3~QIW^hFhu1KWHtG`kGzQhc5lh1uh;_wJ4q>*a5lZV#GPMxn18a@?>rdcJH(ZqSi zHAxln;H9VHnATJ6k3;P7hdrSbvH%S+QeB}ojmrI)e8?jZhGv{_qWw7RcOjm4L?BaH ze_okaLNd$D;ObdRv8xrvtTXAxrZz7e00o*p!p7zPS=9%8W~jgFkF-?INZRGgXFBQ% zvSJK?g36rw%=54csm7TAclxZX*3l7UECXFCqa3gq0S{%Fm5<7*Kng~i;%cIqRI9qn ztIK1BVMs>V+Z0wkjPA5kV1gbi1UzMe`hPygGI|}S*r--?84*aFhFT4^j4}@cP1S9B z@zxcQ+%p}6rfE@8qDjH-R&HbCx1d)|g%eUBq$4IQp^Al2*+90~~qm9guK0uq9!R|2@ zTVV8mi|}F}L1_XNzy#gS2xNONd)RqCm-Ap*?CyW0h!>|w{FI8Oxws?+(bQWTBvF6H z-~S&alzJ;8z7fg?;Oi-$sDVSMW1!>_3e41d1QEN71HY+K%vEGyZ_o#bG z-=S)`Q!lTIt6%qmTJTL9mZH>ulkr>W7<$jm%99)BG_W{rvq0epogng{XjYoYf^RN6 zi|xVO12gHaVE%~qy)o#$*eIv0$M5T%kKfIZ1{4`#T`}CH$wUTuND|R5y_(_rqDBJ(*Wg3Ji_(@?%i{R7fvxhgxDi^b?8se>^n3CS5|d+T zTr)DwE^P9-I#lzd2c#f?-I;MUirCYoo{gSszAeBE{|QJ2l4o@XHS}8JK$nP#avuSE ze~q3oDv!8I0B@%qbN1*Bi)3a946WlEn`(0U9wMw|PmrMVkNwt^-JzY$^KCJ*ye89+ z<4M;)4}I!qF!R2Bxt7`BJW~^*bFU5eBRS9yoeK}m7xZckjlhV8`?(T=Xt28O)mwb+ zUU1qd1Td|Rz?PMkCk0YB4K2aOBcPE`dc zCUlE2!agDJ74o{^`G&*E!Sua_&C4L-%HLdEe$$g#@2d3J*V8H;|3vL~=E|6n=KWS* zQ4E9MyJ1_=70TelcqaHGRz3`4(Lah)b&$P4cs3JdUv4OF>zI5h#Cgdm|Ix5KYg7;a z`FT(9Cq$!m2}vG;7g`GlBoe759QN?sLY_Z2rk=8$St{${Sm*0kSW1(grC!^XZfp)E z6ITs$=NE_JhOUz-Ss*dN7jSqQTD8PyQ5>QwlmR* zu{;?QR2W=?YyEv(ABa~KtTdr<=7+Bw_9o%4*&upOqQEN6oi7M+E%XI~oNfuXp8M%& zn#HV88bL2~mao=IcjVEl9{!FHcAM3bS8ZRLR!%prFO^!i<}IIsUV}-F%>Y$6hhy|O zu4XX@lrXdnDIB-*vJyN)<{x%-;ej&ZZJ1iK+O}1huF5VjZB&x`T!krK>v?M%wju@W zlW)QJpfQbRoSSRTnxJB)a+LHxru$0uyIRWwRw_N6f55aI@y*vk!^ZJ zt%h>Z?IeOB{-jE2wfB*pIL}%|O_GSy_woa#vWU%k2kaV2mAB>HgkczCd5d9V2bw84 zWLcnRb1T&KT<$Ymr(yO{dpF3_tjiGmQc`*+fT1@vOp*y26&W~M+!1^4GPdXE|$v9h*uJU>x53j;0_*hc#TxCc*V(j2j3c~Wxc z=9>s218yXlO>Ht#+bKlN>Y z^z_ecFS*zQ#-Y8;1qeVqs(m`f`q?!Wyfbh~o*ObaT&Cw)1ptk17^paQz+U7o5?Z5- zj*#KLgX9Xn)Gm8@Sk}Wy^ziJN2!i-e(8sifT|EClU)Z#P60%Wc@*WyA>|O3Y1v6uW zu%oZThkiLdgpIcu4p>0gTc%S@3k@JvmFtx2!+8=79w(L^#DO;}V!elZc3itB1@!hM zgfO5pa}`CiIO*DLHI*x{GS)ZmtcqKvF4kVGE+TucD~Tgp;H8N=!&waTRpN)F=Y zp7f$6!yCe$E!Cy6xgBm{$;dJS_S>J0IIcsv{|0Th=pXI_zFDlm?6sM#$BS&zWpnTqEAkmL$|6?t zUYO|2R}m?B2ming6w1}5g4lNp^Y0paqzCGRHF#58b}!!F8Ez1b_8Z(x@9%$zkBvqBAMS`i)ISJoQv*4N25EJ}*$j5l!!M7o^wrWmyyjq$k?qjgZ;c~C^x;A6 zWuMwBY`eoy{%BB%!-e$sPEel2iTsIV2mda!g7{Mi)E};nu)wE*>@{P-nEV6Vxb^jJ z_E^F01N#qklkxq#7|0oZf-I>;R2}1+dxJcdku{$$ExS%cW+KjXbqC#KEQv_h+V|05 zD_D@%LmR0hYg~dPz)+n0W%uKUda)$0Ug8YXG0i)~GID$hdP#-w5C^W&VEg#UdgMyqauy6o0nPhSr$+U@p_+B*tEk4qSFvICMXUZR3Y_8|NbE2={8fcdzZbN0*f56OK={JMwMX z(oAJLRT70pCQeP8Q{rvl20{HnE&Xgt#HCRT6rSfG*4N%328yLUIw0tl(ZdGgy6P4e=vMhe7A2Cbc%T1Jf-W`#T zYa47%`eVFNXA#E09&`^j;1&Mk=>i4R$IL7FT8RZm7Z#;usBxdmImDAXCmr%~w`2Z`~rhU@9kxoABc-45gc*v}(Gm3GT66)`B|npaO-?_EO0c--%542KN*@923NU&$M;OYs%ZXj&O`?gm;6)q9oZz@ zJ12%-?I%PN8h3SInFmfU;WR+Yd&x(*)LOa3qYU@um8t&9h>WZfje^a*y|HWlK@}~+ zq7_I-hk)OOj`w^Ur&QrqMC2~2?7Xsn;({)U`_iUCmF5uu3%&|h3315fFQC^fx(fq- z)nSSz&0>Od1mzPxS+dv?+BJE*TXC7(oF-=oSmQMsgX>;jztaULAfb&ZZC8b`V;q+0 zM2nyC$~u25m#ZPk!hv$`&Mtc@Dl*Lv`fuc+RE37Il zs1pm=#8YvUS+~@_lV740j17VRn2!p|K=G&IAwRwU%+Gu}mh1da1FlgWtFI29w+$au zneLz`hnj#&_Byn{Fqc?QlGh;s&XYkfjSan4O(KrZqnBeAV121l^0ENM4;ID)@o-1r zo?VE0uR%W~z0|%1&qOipzEZ4#FC{g#GHZIJD$}(OpD4u?J;#zT-_m(RRMStjH!T3D z7e0rn7s^?Z{-y8`YVnfwEf0+-TZwVh%inUK=t7|O;>%#$5fZ?$_$$>!d(bC87PyGAk z91-i3bte{Duoe=X41rR_y6Wra*}C;S+i%JK&|BB`2k0A5@f)_t!HTUv)H9xPY0z_6 zacj=pS_qkdhHu^Hln)m0hrEO&HY(q0%3}9&6-#YH0Sa~%BTV27Yzoy);MSEUnutJ2 z?Q<@lbbj6341D15D7RB}C4>!#Uu8yT%tXy+hHCjeHYa``0o; z%qRpMr4G)s_m4?@%u@U1s&A2wK@iuUSWJefvK9Bq*ez`{ecH33EtSsC^0{aXjr&?B zO;}m|QNUBHSOC6qkR>a7H*rm3NYhY$dc2EhE?7;ZZa!B@7dHOwCg7YpY6+B?jxQ&D zwmHKMLt}Tr?>zXEL#_TYJeQcbHo8V7mx*Y%p-8quqm~vy zSgqptn$kG5JF0Ac`&BUl_HubU{&q@jMx^gNg^*rY^;*Cx{$Iw8C5-Q20S9SDp(%y4 z^;30E2}A2!E$zby=XP^fRB{MkBU`SObBq|*U)H`?IvukvD(4YfbapMF44;d89Fxci zs57`@Jv&#kD|)TuA@nnvpmzE*9pTPv8w|l)$?wqUes`7+({TDuu98rQq^7=!?(eRU z2{<-+j|q6FuXWB{JeoaG=IFJNK}Qf539_0qj?iN8XpX!fDwbLyL_!59?@jX$t%rb4 z8Vzc*RK_<{%+>>;p=tlCM@;K1paJ=(_?`Qo6pZpv;|vJTX$VYBRf_=pKk?tEW@%s> zBUPaAj{5lHdEpD;L0m{iYSv%l^A|S1Cgm$Ba(#l&6nm{t(lOuso7 zrM&uSqxY_MdAiMO23DIy97JHB*za-9-^M{`&|bJ@xRtzbOL`-pywme5h|?Iy;~lW; z9tVd~86mQ1t!<4ivs1&b`CX4)MfJCDtf8?~8P$6PzMZV0s;Xa2%+)gppy(>#9bQ}v zm=zubeyQCn)2Z6$P%4)jFYGb|_U^V8q~|+|7Ah(sQ*WR&VA|0)eOK``@>d{Sr|%Gs z4#h=TNCB6kbspo{+4E$IGE26q73O-5Gg{ zL*+7Z_pj8^fHM`=f9?!aqitMht7;h&ar#$(4BCU%S)tA(j`Jwplf|OVAWEw{xGFH1 z`dK6KU&V0DB|dxj#+~?^UiN2nZw8xm?X+$dmwsl?ebT5qzv8TZS*&-qdJ=EuHp7|c z@t7hR@RzkzU*|H{W8DoT=PB(59MY1~ne4;sH;ppu^FO>C=e#%fjPK7) z`)R`I!NC(!yM(sWwguIM({!xQgf|*DHw!nXHy2wAU?2%%tP>nK$o`3AoY`K1oF`Ms z8%dw21FbC-h^u^*8QK2`Xs|*oKJ@6uf{n*9rt6I3{&jY@dNz@BJR({l1w_PLNq#e~ z@B6w|+cC<1+Ku~LE(Z%40TL$3*7$_?c9nnR>7w)ln^}qW7!pmbFcJ6D?~P1g8*TypNt)pQWt`4}vuz_kEfd^Q=UmA~8!7W8LSDrVVvL3;)`WhShq|NxUt74O8iL{H zyOKfx?e1sCCV{%^Xt)uhEYh^q!6FR0D;I53&F2*V%;v52Y_oz%4OU7g1;H6Vj_^n& z1q38aE{ombcQ{Qv_r1oyVLJR$`@?C_gpQ;+ zeBJ(=?EjDw7lqi0g6HI={*Kx9ZD{1t68IKNNJJF5SLhvf0=S!_0q<1>y?Fl%kC^=7 zcMvGnvpeS}%9P#Hlab87Ms_+L=RP9Uo--1)g!-d=ycnS7hUv`S{`VbD|;6b~v!z^W%49+3{ zgK8V8VK`&WSwWz16z{vxvfdrL(AIAo*OX$2n4vK>XI!|q566bIjRgvkql4nPK@{|V zL=6MAuK@2nPmaSv2g|h!xdz<4uM_FJ1G&4$; zamq_z>9GHi4-GPLxoPT%`9CIrPhm9|Bd9&F5b-S+!Ax5QohS1@Wd6DH|JeKg{P;WS z|DE>lA~BD?AAm)Z|5Kp9f8HCuS)OaZO~L;6f0Rg0hp0=1U8LfFsDX8r0U^o!krTPr z3~@|odI&tw^4Bc}w>Ys<3i>J)IOoPZra^}c!Ywy{dw-G*!E<-Eg@RzWIPq^+gf4^u zzPGzl&YVie~XKAKEn=Al6z!!7NzU9_3xfBwe(%pnE=AA&`z zVN_($q6LrBtp=rP6@>D$lN7=l6`7^fF+3f--S7$Wizmfy)$Xg? z59S#HSOMV$HFHXzv!+ii~g-Q7rcMhpL^WUkW*NYmpzZVsdx-Mi2lO|4VoU>g`Tg>?YFk?p^Z5q7vNnd z_Xm;s%Ee@)vPFWPRjcuTLuo?07}{5e&HC8{-w0UZu`tg=^`w6?l0mO#1G+wKZQOS+ z?53kUDK#N_W%A=67%D|zje1fUk%|t~;v{~7^+@v{B&<)77Wm#$GqEC$-r`RKXMua{cptB3e`=dKWy6hjzL4kF}3 zaY@PN*1R;Vn4j<-=G7>4-n7d>x)n7+OM;8~3Q_MPQ(gc!@A9$$(i;DtFQH zhe8-G2K2pQqiAA=l^YdDx53ewi)k9&*%uvYO0z0Ibb#~u_>5;^H%xrFMe4actk$Gh zDY92lF5t3xUQ5feufW*2COkM)3jZ!?=((-GXYBBE7tg?%Cv4nJ8!P^-*}6d(=c%9M zjr0+Vua{j!}JjumK7cF$vNK5DPo^2xaC1E+(BNrZmfr4zI2zSbWj@h7e# ztecyj4`V`;50r{(4n5Atk4tLs&V@u9Tw2^EvKTX#gTc&j=T3YVsyA0gl{BJ3=B{9x z+WR`UHhxaU;A|X6+sy6Zz9#-7fY(Ki9f4ve9+3Ug&~)ycjtsOy-iyKa(V2 z?yn!1I>H|a4E*>ofA;e7vvyGw9x|oQfd$z-8v+M!SVt%$oGByd@IUN)iDZ7a5x@)q z&YB}v+a#|%5odchyP~{=g+ia8u5QeF3{bwW6cRKPP$DnG$rlaAZ`^OZhswGyBP~t5 zE?XyTYU#1kZ?elZ{c8o|zegw4l` zrGl>j%^hTI)p-ZG8Rt$XjO+Qp2n6TDIMd@%CbfG7_h?u!Xzsf%wy}p|>;-r0Ny`OX z7rnVR&MNh5tosO8QXHnxXm$Ujhm(qdbKk!|16F5GYz6htP7jpup}`8||sIX3!^Jw7*hc$S+1mS%@rH?dZKeb?1@51NUIV`&dl+ejR1}&6}A7+TbHe z`UFq#NR+ht*W(8{`Dn;~^${V!UI*_eNjWAGm!3xd%!5kZBVn)6l@UP&nhWc$l?j3Q{Kr=&Z6d^diGU` zUY5BMD_!jKon)@(CQD+cLMWyI4g|;crZ28AHZbrjQ~DF(cvVH*R(1Mc2M1yj93F`I zq5PF^yA>V~QXToBiYdbLr7ZlJHYO6Ds_BG6f%VmrnaX&jxN(rD5v_^XAzh;4B0!TZ zHgJ?++MJwKa#4x5pl^8sR!4_X(Lsyqh#R3hC;yZ;9;VM|H#R^Lf3Uj(0zb z`Z-pp2>E&Kwyt^m@!D&Z`+1H1**MQSy!!a#EguFn55h@c@=BPzxc1c>yIZ%-<|thi z*4rR=64YfAEsP$5feF0QbqJo5$e?E2-Ps|}?Kq38lvL^pLD(R=RW_p)qJ|3-BN#={ zAjjBCG|uMY7%<$oMT-oLKYsAzsK)r-2EUHS_8i0>hc!E>#Xm>(7|lL!?jVZIBs}y3 z1HIaEUvwlXF)j_r$eQ63AuEnhi5CRjIrTnC7e7t?F-nPe4QYs8$p#2B8f~+C3*QDe z_ZebaC~&dN>}T&lH$#HGC$yu`$I&45A_!ZSBvC;{*X7(=xY0-K$RB229@1NT+M?gKRgcK`WGmq zSb+SXmUB+a+-6wCc+i)R$m7&q;xmWI*ZuL6y%N3> zaQiIVzf8xy%DJuB`oiXEMeV5N{dnavJ*vF%=2^d8`c>0=l5$I)@dm43G^mM~Nn|>m ze=Q}*XW`7CoL8($*BGWrw=XHm9ybs%D0Vy zv|sHpnoX%Hf9-^Z+-!~FV3(k~#|T`{zY@!Di^+jH&Pfu&gcjZypaJ$!WNy}L7BC%o zBg0Hq-lF$a2oal}U^@@jv#RyGy^ z_^zV^pO9VH6YU1tJ=T#P2AIInDovv+7J`@xiup9FF-~!}1Z-puq;E=tDnOg#$^bfqoUZfGj15HOxsfe&QrCA69=WHGv z6UV<}Eizt!wey058yC*ug>K+WPr|8RV+2M-PvA=`?aVZT)>6D5?|{qu!;S0OtcYNr z2L0hfwL_Swk@HP-AUL=yeGCm*R^R^N!1eEO0kVG6*ZIjOLPQwx&n4fr$ka)Yab;Tm zfvd(uHI{PIP0U4_qO7;cAR3ZllRr{-k$iHR5GYJ`+B#pjfIkOOxJ_)&enzZo1GQ*t zOPv}bJpXoOG$-}$F2A&xqZ>@@0&l79+I&-be-D%;(3g#XU8LifJ3P%aLWTZDWDX8E zjQUT>>z{*1#+s{=kQy(dyFaSYk(lopMbRN7teISRL+wewRP{I6|Bm085D+eV`C}8Y zPjWC{HZKRBw?m-A^CHxvxUgvq=u45i)$IwDN7cP2|MKA3;`pm_ZE3wwcxoKo%^UNf z0+1%oGH(~vYe=H~19?y=Z$^@>SzJj~^q~pO=-Z?UtR`Lg4nJifziPm59$s>4(MwN@ z5KFztoe5Kv=_Eu<^E%2af36)5-q~~^9s*04f@BvdsYmgv^r7%XAD(1=xsPHZ07P;) zwHl!?Tx`hdGaBCA7e`J>YpS$_a5SivtfgY<0ssRRT5w^mWe=jtjme)tp%5V;VVF=&>*>}DjDVW2Sw-Mop@FZh&V!MWN)PZB|6OYJug;JCy{DOFgz+Q z>0T?2w&S#am+Zu_6GlLmwSS-t!a|8o>w+Vp*H|97g^P`%3~<3ltm{bQz85Th%Z-bC zCJ4`@A7ie@hT$gX?=OFsSrh<^!vb2t9-+q?_VV@kAB%^LeXK|kadxp??F5K3q7#^F zg&r>s5qH@KY1n|c=PARld5$OJa+g9)B6zScvc@C(wci3VvUD)}BQmXGaY$g?%D6Yn z#_6pdOTf7Mr#%-N(thuw<#$Pcl?^y%$dk$-Gz33uD82@`)G)yh9CP0khj({?VwfAp zzVcQr%N>v;6+32P1{l78u%4SsQ`((8TjemZ!lYrhbmOng?=Ji=653Kv^tK65Ot?V$ zJ(_l=obXu0{Z7q2!o``Lkm*i?lfN52YiZvz^4I&VOs;Hgx%CiH49P< z#RLTm(hr4YI_D96QIk1$&I=}N5&nT5tvH4fM0aJEx>D0S#?z{LPjVOgs*~zswt+BG zYx~^ijI0$Ns9$)Y8{F?ez5%ey^Pj4>mRVKKa-MkJ9{0662K2t1?9`jMcei+WHRZE?a2I#mmC*JPj0#t%_hRnw9eDGPN4K@_hqpbi1p*DI9z@7kU$;WJW2ZOf8KX4q} zN5!GV&yvs#UI2(S{; z#X2KFL=n*SL7UN9Ei|y3BwV5nAC;r%f-DhC9Mq2*hxI^hAJNJStNorW0Kb1p=#t&h z?DHOT|JfpsIBTjrCA5Zp+6)mbrKL}%bfrWwi2@O=IG%s!46@)~zxd(`6QkSs0TYiq zh`pdCND9+VhXC?{UDgc(wNPY5O)JMWb1a!-=6dVb89a@ow{`_fUO_mBIC^Mgf*S)j z2GZxaY~>2SP%dDuR=ce;2nF}k+HvwANalt!6MzQLT&tmNL-M5vUo_c;SNk#nM1-xP zv$+8;*FSI@;lXCO)_MO8P>WUg!R;^?b;4ArHB&sy2#xkiFT0+7gc5%^f!bFw4wRPs zVyCb~!0ap2If+{;jOwG zn~OpBn#JCei4!}~R54p>w|qPr7iImNGF2_iIvDjU`uaxuXVlM8raylMGfvx1-x9Uk zIbACORp-XOXEiyz(>uX!2v!`du)AO|651ttJ+G<3kckBugz^T$qe;hYgR8w4t3!ji zi^$&$#b&+5fK4ZV#;LclzXNsX!;yb#x;xCRdE}fWZ@k@r1%8fpHf!_hIz;=B=py;) zu)_oKre7~$+bdaLmCi%D%-D+)$p~l8OJHB0Weg2#fK9{fBWLqoUNNIORG$}wT&2yB zl!AT$VW2us@{o)^^P%z$DSg(*mD;UbetN>3%wLqyKS`)U@5MOy&$z3+0rfZ-h|gO= zHE8=08ZI|E98&j6e4IMfOx0IUMvu7_pZ(Tq8p%T+f3!P;cck;$n<(b zMkX5SNTkna*>0%g%mzrU<>>2hUjEYFMa7AE*jQ+%tae_R>-AcRXVrjByyQX0H1xw- zOAHg{951uXfR`A0;*Jd%4Hx$YyRg2EaeX5-r<~~N^0WRm^B*Nssl3JC+@@m^{9xf4 z7+nfWV%KEbZ>(Rzm>hft-;_IXVAE;&TgM@w*TVTlLPAbs9txl=z-IJowI6~brD4Zl zjnT&rCQH4IZ#V?2xqYY!Z)TV8!dA2fu+Ce9aKi1Sr41+*emaVv{E>g)GlYk6D}I|V9xL9jmFaR{H-zQjAY$M3<`2DT4bT`6c#c`w|#b!R5PUA3qX*qk)oXk$!XQyd}GEclPW&k}7`-Vawz~I5Mwk9yNdQ z^qI!Gw~vn0=uU1wZp45F@)r;3gdyP%+=Oz0hO0j{#(Dov2F-2b7AKAo0piU>h$<{- zep$BZ83nX?!}Cz0x!NuZ4N_MLUqwBS&Xf(98z=e&7QL8RRhDeJSE}*E#UD<4u+NNI z`~ieNiEc4!VMFuK;VI$bft9Ow^OS)0Ar!6tBzs-1Stuig+>0c#A|oVM$zGKR8Ie6JGyGn+cc1U~`}-%4csM7%Zr`BCB*G{MKJ8alw-#wNIQ?;{Ev4-wYDwS=!&AIR{k=4Ra0&Vm76(J>8!CEF6%+3xOpc?#{1 zdom~hu^4>?Q+z&CJiOHAy9SM_$ zB=CWS1TYPoYxDSHQSLeexK1a2txB92KA?A5_CiTu40_;jBlX)i2*G6ED*vA@;-@*s z(4Q&#Q3ui<&7Za4OP6Tj_6eQYOG}3sq)$$Fp<5&H2lBB74*#rlg+tG7jIx~*S>py!t1u# z1EIw--8Z&Bl$5dyCWb?I2y}+LFNR{kZ_71|u%WL#+eUS|o7ftiN%ysL@5?RFsc`IY z0`!Dd&)t&ngn80EW>PSh4&!eEM8_KV_zyPhuE!dKx5LBgUG8NI(%~|GzCk0y6>fAt ztq;r~StR%xwpb>>q&l`Fu7K>);dFE88?VY|rUz)m@gt-Z373I8%V+bd;@rMppKcCT~ z#|rJ75g;i6eRyPWy|RK`yuUg@o^C8Wy=+ANKSYS);WZVWRg~Le@l| z0!)3kb{p=Fa%eazTgxsTL!7*;?F;eOda(*Ac%2xQ<-hiGAT1I7+Ct{fh)3!)Q{-nI zqCSf+-=|GdSqcJ$3_3(uTv%c|!TUvb+}*_}K)0PTan3>`Vz~DZ$+6Sms@m)H$fOn` zSRGDEB4r8HRXIejQ`=sL+nzDAw!s{-u9nG^s=~pEVY25ns(y}6Ef`y?ydd_glo3J? z#w2`6P-Q=a+uGVBONlUdo?QV}WnXIG|Az2Mssn*|oCif|DFq9B)%%Tt?+{P8rcKVW zjr=&`vZ{{zeX|Rc$uR~PXAMom!lP}+Tf`_g-%mdV_?FfGFhXjW;HfvF7~-7-x!-)f zy<)NTg>-L#xLb$(-i)oN^KWxH_av-n}Pg4SK_~eIUFFUHAx2n2^Y2f^!mpw~YI z)2enToHqZ9>J%~N^FwQjQsZAH2;+st<=V6JCi-*N;dSeO0%M?v$KJcUZ=RZ;drtk= z$NbW7(q!#DR#sdjf{19yfkA|ZM9Ob6PIzfxB-HD|!0sb4Ip~aE14j#L_r~>dJol2q&+5$&ukptbY?=UxY-ULm|@Wx(aj$t^Fr5D%IY(3*M8 zQE7Nt-jZ2)5IO~QKls1w^8PPhR{_-&>@kathObVqPh3V(2GGI~2K6v-y(+9}sKgIC z0A*px%lAjvdakyT|Har2>vxV@&E!l;+vtWNEjZ3~N5PT|@W|1tlD(ip3V2;8xk8^v!+YM@8u*j|O!?eL~DnehFM)A;b1 zSG78D;+!`~F^U(Jq-x&UH9+b}fb8o!a^7fgHWo^$a1v`X~c1^YeQ3RN#({bq(hAt!rJ=D?qt%rzu0W zek1*Sul1O7#Qg&2eaEfl%RW9nH-!h<22_`Rgg;py7`S}tN}!rjd*GX;qhaU~a)-uW z4)@fv%eIq0XtJTC0#`J%@H501Z?9l#D&Nu#VuTMh_N^cwVw}l+HEP67uj_Z-y~33r zDEqF+w(wVPB!v+tF(jQR{e+89R$~l_87ir5Kjd2hDbP^a#;QI`K^^(SN+b3o2j(hn zxH9T(LAa^oOw#I`hEF9(OJCnDCai-lI9${9>N2!>4sdwRWP6e+bSdj)xpv;EsK(U? zU^=D&Yv?hl$(qq)Gp5XY>b}M5rJRy zSIa5mUkIW-PAFN0>_aHhLNR?=0{QUtwb-jr#+S*_;`Ai&(m3?u*OuTfM3@bC>7C5b zs_ki}Y9d$~E+brDZH0v|Ktw_LpbV&Tow>Zk`6s2&^?e~vA`ylOjmLi#rz@)|dvYo8 z3JLrN1co~#0T9O2$#wogEKU=p6qv(FiG+ zMF-4#vPry{z8Mw5a}A0~!NFVx_&9I#4RML6@YZU_piUa--W-%0rZ@e3>0RgGx+hwy zYDySkd_iUA93Zj}xrD9AxgO4GQL81_$cT+=nvT%($UsAj_~7LY08x+}J@CF6gXczg z!}JhI-MP@kHaTWZj!84J_FUT$|Mc>=q|Ljcs>zdd^$nxKck3G(o*1h!+0nk{xwEg) zZTU$0g7oAM{&&*DVKNi|T~I2|0Wqxf3n=!Y$MN2uPrOZ)P%xZZbFMs$x38y?*)1fn z<8z$Qj%6be+$D~rK*ep~u?r;`iRx-9@y_7fT~Xztk>xD@b>lPdcxGe1tM)5N-HpMR zg^E25Rv?XLxTTL6mV{1O&EN9&iQzXFh(kr10~#a z5z(HcqkiI(BkG_@i@6ir^+l=Ol%%}!kvo{r!7Q1!p|uk051wbrErkGi5y49n@E7IL z4_igR{_WLxj-thD&w`Lff7f{lh4O76^8#M#3LAN9k*tZ7S-#s(0@s1^ze&j1$`CUN@C)eyy z&Kc=i33~01lN?g(MehdJ&z86U2}EJ3xr%$kbShN%?pyEI42kdgpQptHtj5pM5u;{l z;HMw__dMw#yY)GaUq|kn3x7$sXIih;vj*BH%$#>DNZ>TOiiY16R2MH~`3gHD66X&E z>co|BQrbz%w`kvatkRY|lK_PxWPrhs*6#)+{=&@kWE*46Ex{LE5TTDjMtK6?Y7v-< z(lV7l`bjtXx(3bSr-%sbH}^^7iM?mY6Mumt8>JqHRvqh+VydWqyx{0QMN>e&V(1laS#^#WKv)#RON|+*Js|4N)<7td7shI*tiTu>|0qNK-{Hdgj~yJ)4TBO=(;I< zjvIxwa7&W2`~`a|AY_>-n8b)+1ISChixeah!RR0s{$_nk*3xl=CYwS7F~g0a=N&RE z^`LLH;GLh>+oQh9lN24ms?p(v!iF>IQm@w5nx`1DcN1f(G?0`%y13y7;lUWQHy6l2 zKPrs&_3zAgJn2;|A^clOAKFDZIJFZkWhXi#FI{BuccFAA1+GxQ9^=>hUt242!7+PY z-)=n}es%8IMQ071Fi+FMrXjLJPZm(4fVnH^kfCOo;1-?{bIhH`yz)dC(-;!wt0v*I z^}E{|5o>FY!n zOT?IOs&dutPU_gV!a(kz?fh#<1pMxVEs$+xCx#Am{)t2i5y7A?{%(CiV_G_CEu>b7 zdT^xg>SGf4z1!5z!@>Ml(`G?>bbA6~Aj5k=gWKo9IsJ_?Bqnn9x>(4J9-}-sz!GvPJ z8nj7C3UAfw2%2M-{h)Ds*PftF;?TAPWc`rg&~x-%?=xaZ@=#b!it`!XBp>c`FH#0u zHOPE=nrPxIRs89*hsP$?q^WbfLcl07LYWxy# zm)7p_?Cafai$8IX#(eGt5muT(ekVM<>oqKR&<_@D01K#C*B3`#(}>;7yCMKRZ-}kn zY+3B<>@m-f0X(=l(n;aA$hqWHgE>Xa3V| z6dCrd2q5xonPKCJf&Ii)(u~##3MIQ|4<|qOu})E3Lt$;$7rvcK0{}N#{KV7Tu4RTS zR7Q@Dq6f5cLwXDQwyd`cu21(MlQIhczHxDnu-66JhBj48w_1733r8_y9q=45_a@=A z)E>=NncC)&hmkW6>ra0eeUK`|1?Y>JF!Y)VM7zXi+KWuwc$i9kl01sJ-O#ym1?0}! zg=$j;GQFt=hcB6_y@2#86lCX^}}zM_y2rT;j;^fQ5X4Y5T67s8;2 zWw6{Q$IsM)?ZfrRne&N5xfZf+Bp|J#Uq91zyqth$a5p3sc(hXzkN4Rs-C{(|lEa(F zBq9P>+CHCsoRh(dLsy98P&#_o_d_>_9Y^$+-p+W5K;ml3W;mf_I430~W#=kyI*p*- z&i#pHmlJIZ)5HF6*}RL%>1J)W1a5UEQ^oGQlS8)G593+}=B_>ws#Rwwse=!)gu=8N zcW)fN+rRG5i<064=F@Nd4tnoZJv zA9T|5XmqfH2in3BKR2r88`U|tAGe?38cx?PYjh}r;MYz=J#QT!`-0!BCxYow(N=3^ zS32`3g4m~rR%IlK!CZ1U;Zk#9LhbXHsJaTLuQO?%nn-O3V!QaQ= zy=`7?ERDwE2fv#)pc~c1m`6*8deJ&OgN6S}W(WwM)0o3mu7*%4IW855Oa^n}^JZ3M zo9sW&*9}@MCm~bqFCm~GXm>2b+NUiCw?0zU zp>aZn{Qu_Y@B>@<-|>F&^s*OT_6K>!0{<5nv@Bw{T1IQhr!>~CO72^Gw{Xz)7dd5XEoKo{f4Y~2 zzsicV;6zv@MaQ)0lHE%MITb4zRr5GgZ-Dv8Y2L~dO_7JBl>qex3HW^rF#i;`eem<+F7O$>`h;?h60jCI^o45h;ujfVY=9TB)Y zH4aZ=(%xq@dy)f{DZGhChUe#q>h+a`(ZQVaT>j5z)Ny#lVpYA|%)3Ktd;Xc>E`_f2ri zqH6SCRT;E$0N7+!4OYf(&y<7)y}9lSm-Fn#+lbtQ_!$34psuH@?66v;;KDUK(U%W7Pv4r~ zG5s&C$#|VQbjllyxWmR5gI|orx69eMBeb+^Zh$otc3gQpx2zci$>X$t?*2Y6Kz#g> z%MnOg8b5h>tn;KHU)X;Wxc}tI;%E9bmrc!XLQ0hpk4`zKb+gl(9G67e=vh`}_stgy z*$rvMuQ?~ir$h)+WsDo%^6)%=D}tYs1c;!5$@kfKGbbn!uWI5liLsd{-c~}79im;6 zbO+oEU746niZ(5D4nuoGVpn1UNNb_Ol}rnXWN_unI}TE9NJ(no9Ko7!k3mlmFkicQ zUmRkuZ9(8o({!uVT?OrSQfS-oawEnrCb7Dm1~mj4`yd}i{m^vKRmtEjRPfyDSGZ0AQk(hu%Qo)CubIe_S zx@I(+X93l7L{MQ&isTQ=1oeipj!?^axfv{}2DTO#TV(_2FV9D^e-!}Qp6Aj))tI+M zbO(`pH(Lv?^twh}@~4H?T1Ja8Kp-81?ur zzm?$6b3&z|f0o`@>@vUuxhX#E!Ydqkhe7DXK+j-nN|@RLW~gG5DxjDz#*hpng{iy= z+i7W61HRUg#?>kD66LI&-z^aYp6WcUzaqLoaSsv5m>@8&{LZ)TK!4`qn!v?gNUV^Z zsO7of(egyp_yU#ePyY86EGdxm&MJW0c7~pt%8vI=qOC{?grd}WKT{ZFQWOFQZkjx5 z7)1s-I`1uPyFH$A%NCVan&pjyar3=J+}Tmam8;I+Sk zw`{bKhiZXwK@3T>ZNYWL+Y-0I?MCPZAHoN_-I4k9e!##Y96M_astCLmz>}_ZEjwf$ z9EH6I>ng03VN$4UCGON574yD+-jg)Zrn6Hz+`V(VrnYLgA$y5GjBrHKYQ>(-f1x%< zf%hho^9(0`(u`3)E64KNO_%$3REmIEkP_`MMuk)|MQ-D%BE;WwX>=5#a^=~iAAh&r z;KZTwb@CA(l$S6PeW801_1-EY<>+G{LlS{ zP(_8{rXB%mrCTmTNmbM>i50sm^~~^#1q}UJea<$togzayh~Gru0US=>`rOIsR<|=$ z8TybMzTu)hD+Y?m@FuCjWnaT!&MDVCCleftOh)u^me2I@bQtf^m-Cv>TvUE1f#eW! z_cRgacekn9Q81X=qTbz!NC~e3lX+jY9ACT7H2l|53`PeVFQA($!!!cJ7^4o}`mC}& zQABL#FUw+Qhm!E>ab8dVTnVanuK5eS9butFtid%!6!3ea0U?_o>WXFdgo9rjB@z*X?MCzwqJK#gBiCsqB ztJ~K;*T67=N;o~QBZf^vYCNB&W770dwQ2yK^LU*IJ8K4YuQ;%vUE)%xD9q*mUG+U# zd%EvW+vhNqtIc^3xA;kb?^3(Q#h>ZyAsz%k70RO09@e@XO)oU|r*xe4A9+P0H3Zy({l&ZnL4L^5p zVrU{mlvB@B+=BtdE_kU_>&b)vd2zXKyEiFPFeZy1&_NYaV_ZirwC`TMQH3 zYJ>7<)my6VLzc+wz(Z&Uq=8K%xxk%d{oBfu??Q3OgsPn3^ME3aypf^mw)1_=M^pON zN3vTL*fHB-{>SR0{vzyu!M{l^`-ndsnJo5|bcWyj5aPaUi;IQcr!2X|nU>x{58EU= z`m7~Pw%KxW2$>*g{AZ3BG~YS+PWjEQ)=2Z>TCsqS9J0B6PBL8+`ScjKwkN6QjP9`0 zkiacj;Nh(;OOa~|CzcY%-cz^0yQxlkx7}P_byQYo0=eLZQ&h)xmmmYUW$|BWGg1~c zOT_+4GTmwByLAs$^3}tp)8!6kH07HN3hO#GQh*P z&OUrQ!88Qf`xAW$%@BWla`aN&tHgw~(SgiBPSk-+AjSR))%DPB_Bbo{`2<|+fqQyn zU8*~ug_}ds%MK4GA#^`?P2271$7Ki+O}u0#rF?CV7XL%gp~ii=%kIvZ{(fQAAHCOS zS;wlz&mW9GSU-vg7(6Vn-+M840Z%~K=lMbl1#@FIwum!+>R6nchN z&7$?^ic%`J#xAQ=ppWa0GQ!U|WE2t$f>X7rh4|Apx2NwseG{qXkmFGp*80<5Sh60K z5rf;3otlFbjuxU)|1EGm zGP^s2zl`3Vg4R*d7yt3z3XUOMN%|8p2DkK>Z-TF)AWnJaxq25LZIFhhbt%@qS57Mi zy@f+%84wEzkQ{Sd*vOH2R4EVMWO-xl&3fzcN@wY>Nb89u`as(}ZV}*}y-D#${zrEk zKJ({@jD^LJ=#?^tviO|~$l1O`x|z#~ zv(fub$e+Y49#gF9%SD?WE5bs=-~lx}s9A=k@8YFKUiSo!&>gQk1b^rBzIX|l)26Xbz+owQT6MrFzC>}$^H$n&ft zx|i(A*KbfMvM?3}{e!!ObnFhLBZB)11Lz0?Wq^6+t6Us14a7~x@ozYlzpsxf^AG6q z)tu8%=o@}jB&~ZS%fM+MM?xd;Ep+KD(RGHAD1r+KX=hde6|<_s#C;h$YHzg(p5 zHPAu^u1nuf0#S;OnCX~ZKAa;bf=}YPSpJcd@Pki7vmRNeSoH0q+{@4fmoDo!v>4K$ z={kSe@0o2;W9JRyCXV_Y{GRoJ>NA zoZ8@m5r0gWm55Wx!7LGS4zn%sH&x-UqmM4c@|&~nmgJD!Mvbemk9?!O;lt;(0Nur2 ziu3z)GGb%;C029{9Y=|Irp@vHOveGd8=yQ#VZehLw`>f4XW$LqeY7X!1jc{dxN-Hz zVk%?v9qBJ?KW}rub&8-&`zTPV#9t39^)H|_ywM%UAY>`8a~GA7sjgrlqEOy}Vb(x& z4EaMHZ6`M~%9(FPY+?O5;3PSA#Bg~n#CeUm8w>uE-@n`y{}{?|<9^^DB4 z2{VIqi|%X-jjTo9fQunLG}~DB3EW>Xffg^ZRzgKuDBO|tWX!Q3c%z3ld9NVCO|&kl8|*l ziwW(jI+s4lYj)QeGTR8e{5oDP1FIg)FXQlz2v#ePWhKgT>{1%|joRfceXwZ?*3L6j zvKh9KHMkmqKTx+xuSC4Wt&iQ`?|e}Hes}s`R7Y|5@ydh84S5%W z$ZB8QrjvObFUglBCh9U9@=B4hVg<`jFD`$zR4}lkBLOcucNNMvNkl6Sgm65X`f*U6 zu0{V>$sIr~WZVmhEj_jae|yyB3JrW7HeI#%l(W%^Z_PMayQg>`#MBOi*)U|ytp&X) z`J+P%l$gfjULkI%x!>)L(R93uh3@;+fgc<#30Hgqr?oA35D) z{uH5tvcem-M%M(5^z}ebf0-)JPnx*iAUg_^P--kc(aHq^5Eu5mQli((`?>1hlTU)NwmUgddj4tn>nl`W*%u>TGG#3PD@mFb)&O6qIIQtIi~#1{ zx6WtXiw#|_C%6AA?%IX{=#DVKX`}I#BA*Jprb+?t{h!fVN3wIRa{e`}b4(j;@CXSZ zaqk!P6g;dc?{D64PGU$ry2bK-aF&7wj?b$l0TqcbVz(ZmOlxGL)OyDME0&7E?>NJv z+DFFi6?f64MjgHj7F|-}T7Gk#v z#|NaA!?gus&;lq1dNqgix`KNY4b)&03FhhYGA2EO48td(T?g$kcGirRO5URc0A3|Eaf zc$-=8rR@RgrKZswL5TRFYa~$(Wjc>SNQ_bp#oN8|fm-E4rx`{w z2ANhy3^ozNYF|L_U;grAIu$h;@6*qHQfSq%+eZxM-_C{l*Afmsbxp^3H+qyG6+FP% z@#f90yq)}{pUY->x#Kq){)j>_9@P9*n%kEEQ|+}FbRKHk$0@nnxSH+na7eEkx)`N5kdbdsyVX#m)p3F0PGkR3Bdq43+P`>X2W7O z-@hl#tU5R)0OioekcdFeeJxe^*yzB(Ux+ktAVu5rAMg)?>z|!u9_i>16xJUK4AY3{ zKb>=nY42hOm9JDWVRE&2Jo^Y8M0p2YX!=WzY)=F;i`iWW7bxmT#LYfPvWQtAkG>L( zA9a&CeBt1o67e|moKaRrbhgb7Ga%C2m;TD`^b!OigbB`t6dC_#Iw&&1_cMDI6j)t* zFUpYZE2wMY+#(LCu0eo(kvM)g`M<|KhRfGU=^s*%pv}+J~a8Q^`-lbQGO_`oI z29I}FrjJ2O$PYa^Zh_R< zHuB|@z^#*S>PBU4(VOd~Hpe-T8vRQqBpaxm*;6LaM)6FU-Yy2YlU0y-Ru`;}^LBD2 zdV8YD_VgL=-m>;>#s$Q6ks|c@Jt?IuG@W@EW$A2LA}3#o;byExhe*<>R>y5|0T+Q- z82WAozIPpI;L&prgQL8SijU*brj(|Gds&+q?0wpIak3E@S~h+W>Z|)RlG;F)joBjHR_`|GhE^Q74RDIA4*s}NtKBPn5!|@eExFqi@vN2{@E1D z8VFVAMPl4=Iv;*(;^Ba#kL}r>y@Wr$cuj#dT9_POO+<+##>L3ODPb2q-zOHf=oM_M z7Dm#yiqhlhie;))#RLGMbhbIa1L^SX{r6m7+CzTXu)R~5F%h0(xqbeVa_W&d0{C&x zf&uP>M$X9q7J`P<^l`5!HnS$X%4fuou^{}3nmM~H}HeQ!S0N> z7}Q%1SUlRAoN~EPNinOWlT=OhVUr~<``w#GxQr*exzzC_HrD$HZZ)NubD@FvgQpJI z_s_ad%2Q{kzV2uU5O0$~FxYE#5y>b&YQQPXraJn^oAZ||^1~!vH4Cq-YA;<3DF2H? z+Wy@3LBLGsdiz$Dz>jzGM z{}h-Uz1Z;`_b?lB;}y*bq_i*Ej?#2eA=Vdbx^U`Y(k>W}w|yslLbi4euqB0!MR~hD zK->x~UH`(Xy{^7Z->GlA;W$d#zzFwXbAqjW1|ssP8<7`6UNqc&IhE(<6k8jdmjbN?6MT9A>{}9rtaso&hcqi)@*BXZ6N; zhl>6+_yH$STB!Mf{A&KYW%T^0iDi8%?A+{ zJ>;+S`tr?w%fuL83+xKmh&V8aK3Jo~m|zn5$Oq9#a66SWrkeyJqu+(sc1OfvcY3u7 zgwmg~B0FeN4g>JB5b$#QEQ%alunG{n@@);`rh2yov_Ah)hKqqwkzuoFAKqZ1t5_j28xKzGD|wm>5?Bz>uk|} z8`QI^x{?)Yd?9~>a?$pVgG3n(U339$yO03TA7G#ey^IPrBIjr|cnM->UEsG~%=hH9 z!%TqdMX2qIJZ-2WDxd;i*B|Af`$_eQ>JSHrTMFrC*V#X1U><&l7!aZD6!xn%y~2od zP{aAXj6vmA&z%GEIeJWuHzsX|RvI-9CNl4zu*BnAwQ!uV`a?&yHwKxDZ0&gb49o;) z2FWi}C%yS>-g|!WrSaF>2j21M%dKZM$tVU!oftTQW@+(VTGhSr1*6-Z%?W!c?QdGp zs9aYt#AsrKwC+CgV+4Ydwv?FnPrtwcGuP_QjnFVYOO`_Dv!{)X((7 zfu*X%4Tvw3P(xXgi+J^?3jx$c5AGd!Qhg*FlF%0Uz+v|jK_!5-C!T{f+Yj-}$ zVSwjcR2<)U|F($T-qPW7a*X*y=snT~2Dp3-IxKx-qaO|@w{Tb)7LyI8e7=K0_rHO< zICFyRRyYlPzu8A#5e`f^nK0Gw?@)dZAfOFt-bZ!y_fUQefXs&@v5lYPf0UJ#Jq?u$ zdAU42J=>0=it{cTso-iS6|44_3tL8m6qe8a#Nw1v#-pfl{M=9%f4!K4-F~c;7iKTX ztDB)CoD0u;OsBQRsyX+7r3@oN+Ac}vEa)E@R*JGOaw)?BKA_0LL%cnQ99FkQ+Gm_^2WM*B#lP6GQx# z9xA^$HU_l8?O1dKczIu9@0aHpYlngL6BaOQ0C&f@elK+7c<~TxJh5(>;ns=6EKc!acK>)dYc;?H^CMJ(~=lh{(u5Mf4`IKNEe*e2x=Hu!ryENq<%$DbDwanh#0DH?1XE zK94_Q5}|3!8pSeD-90coIUOI(m1^F!Kiv)&&f)dC&6;J1W1s-}0d8UpZA{n2eQh3W z8fumbc9?UX$7kTuqRYLkZKJ&`DwE6n)skyxhWkU|<3IbgKwSgHY}F-0IU3wBZ*$Ji zOjg-)gM+C@sT>M337yscU*fv4-)4}YA~F0j-`o4C*bL%vP0+8vBt+t}v8siZra4Kf z3gSlrpJ=>Fo*`IfuYph7Z!!8W7@QDKBtL^*kA+lf0t>`$m_T;c*7o9iw(z zGYSkiGUnS)1~)?Cn=Zn)?8W#2?Jh$i^xts|gtBmbs*}kdWudpa25mK=|9CZ*oNnv$ z051iO#jldW62Up)@kb-ycKY#Ah6GELU6+45kL~0}`URb{YXs8cfhh zs7WCWYzc(E`3V)MF+!qEFgHX%cF~Sv9wBcg?m_R^qd)P z78K+Mym+Z%g#R~(3DCcF?_u)Xw}gP4jEGPpEJ{lK;yee(_~fwI6Be#sI3gv&9BRuE zn0PV_DNk5JU1*rF3V>7-s97ENIFZ>lUc18h4DzK2SZ`{u1Z3PBkfk#tR`kSm$7h z1bRajaRUC~k$^y`A!59>J9~u_A&^HqkjcE8a_P_B+wBSF@A9Tu8CAe7N6x=Q!t z8u#M>XNQAlU$?Vs3reRT1N4DE7mi=Wlt>!tqm4to{!}0NCOu(wPvM*W-9`+mh5^1` z#{7ouq2x!aqI9DP7NaCxDBY`*{(XIQ2UWrrCpf+%23jKLSTq=L4uRnLM0#}UK=1%U zAp*b2l*20)k1ux$b2?w0)$?QWyPfYWp%zJ>%Lb}N`s2oL6aHa(tfBD@nIrG{zD1V` zx0gW$Ki+%8zQ+we6ru&$Y9;qCHv!_XGX^gc#DVLpSWZBzv51;cg7Qua;B24r@O{d3 z{nUot>-u#h_0z3z7op@#Y3breJ^hb=K??cUT~mK6 zPsO9kZYG>$cKWMrfb{5t`Q(C1(>6{ULvoyP;J#HXT5unIlL>A^WJ3w7fUfwSFCZSO z=4PtYkYnEWxrjSmyDCZ~S64)4X)Ce*0u*6c(hN_&po`sBFJ$>U@F!6VVW4KV>bOd% zVwUPP`A`Rmu%u|JW(GKm_w$+v7g1L4C`F_Fi5T>&IEn`BFwr(S&LIdadr8qEOuxGB z$c1xQzd}XM)JLbP6%t#5l5Z?4%1H4{z-)v6=O1(ujZMa?tMLE;682xn|v?i@L`;C4G z(tlW1&+*d*uhnmm9`I1Y`m_*|;Wg2_8HK7|0uV3nS#>E!`?-AB&7_C{2-Cc8AY;%4 zQA9B3J*hq-AWB{ma`gBCnpXOHJAaqzAHwus>k^Al?4Ao-rxpZvh%o?dHNorUjNMTh;4l%!cJHnBF1DLjAmOhw9-#KaX;2^{D4qwVRyuYe!pMrP0)5H@{cUC16j#C z%YuxBtofG|@(&W@{^CRlxFj>DHR~dtSf}XT(v^G9&^sfxSF|>=EcN~;Z3V4}a0s0+ zy>(q85>-YTk`XRNHhftUIQ`Z0RghyM4*g*`3{t_QFB7dGUaYg~A>~4i^BW!5i82(m z^k9Bqxx|J;e4C=8wth-QqOPNYolpICu@}QZG7N);-lisnMsc0Xy4u@c^(og+9R1yiqn7rG83etJzDbN0 zdydQO9x*SPM~zEwyt^{#tHV=Hg9FqJRDP77^Gb`A6l(M|d8b@sL`O zp0h3yoyzg-UBJ@RiO74i=8N!RBF3GB9%;##T=^;Cb!Il{`pgf>VnBZok`BHVIrfb@ zMHBo(0!1}6sDP67j?TNx3+o;63&f(bE-uVYLLtX$mb0(hloU`=V@+N+t2H0n&K{Rp z&oprk2HnP?cCpgoASEUTd-qvK!Lay6Kt&2v|0r1-B!-w%@0 z2Xj-xL|>`KxKhFlu>8*SXO0wb)4duaZN0HEszm&Xy%4j%0o0;8WZC}79Oo`@8Q(8l`4Y2(Zb30#eHJ+K7 z;MTFAX|spAU7fg>7H)SaMeK@$N~nwsQ6ccR&V}x(qMDBVg#}Jhk<#C@gW}PzzLLC^ zq=a>1{aRe_EYzkA&_=1!E67aVv6-pSm^oEj=(wdD4SDwF(r~vjuT#GCE~9O}&DTFS z`D#KYN&TMLUf40ZtY-HYJMQLir|FOGJBqjnFn{e^sMNwh#4TG4nt1lTL*N7_3j4H{ z6t=tMK!tN)0KV!f7EIEzj~*t)-v9DiTJ!AG2%p7qq60R)TRgPMnlj*j{6mOm%Un$3 zf*_rAI-b-xh1E+1x-DIV#}R26fO5ZGWRf|ODT|4i4^jN+o$@EzAmG`f_WZuABpCg< z(bpGb(2+O?(8S`4Kjd$25dT_@9=@;lT59eU@ku0IFzxb|cLR<=B(gEi)9|>8l=V8_ zxY&)-kHeHy_?>>H45#}Gnjin@Lvi(VZ6H`If04MH9_#1vAVT+}vaztoj}*-g^$}Vr zD-3FQ_|vNGXfnyWrmg#1|jQI*KpEu>g_u{YlK_yup!sI zD5xMvEG(gP2qNGvDIuVAmm(qEh*HwsC<4E``o8b?_wdK^?6Y%c&diy)=gc|h^MSQu zF8z9#baB&e4J_Yoe#!-xk}6phsI-OVm}P_qP6=VYJ^wzfdDDH3TTTPvg`9W1^83v) z=T=3O3Fw!F)WWLRI#f%Z%DxvJY*}~FH z^_dOBN>KC2^MHyRxW6;=zN=SU^-&TlF@fZWeR>&l$=M>02+F(NV0UI*s`>|gA-TmO#|nfhf91s=EfMhN8GsO6^Jeq zlmJJoyl7yi5`E?`v~;Ncn5Ma<%(h5s`s;kj70CO107%CnN&-GsDePSw;A%P(LN7Dv z-^{b;OxK;@Z= z$}8#_+*k5Bu+DPh>A1S$Nc8}UcI5uIL9Y)TD3=xeL*QYn@3SkC&sd=cx&=NDLH|HO z(?x4{FJrzs!JYd<>%;WKgQLkflb2D6-Q!yRTe>h#nO61i=2uv(w(C z6AfMRCB@MAutK#lGiwHb=DH}tB>CHQx1XO--VUR-3!UFS#hu#Khe)z|y%7vS=S-KC zvnEpDQ1uOr>B@9)i>rz9*_2;s6Hv=D)uvswQ>MM#7=kb9xT2=V>W9pb-`O8Pbu<6U zP^8N2eX-{O*-rYP(v8lxi;tYR;a*`xUmh|Gu00a@@;L2yQ6rk7|By$q=OG>BpyBjH zs#xL3*>KNS#ou{OQf_gu*AeWI!5Sdm%4Q!3dQ5)qqfqXOqxOG#sk-DsfkXAvreYR}2-wnT4^-+5t`T0UX^E$p+1ln%^RKviDzk)-Ta&$gej-LPday3kFy=bGc?vEZ){aI- zgQJ*u>ff|Lzq@UCdd!cU#X9xbqwJAfESuMb#&0AYpjT-npIIhTO>+9YtE2H?{Ch>A zL4|oUUHEpMDw(?A(2c~718zFBFZS}qftpmZk-7ZK9 zJ=+b}Z}#MTZ51a@bXSFoE#|G%0Zq~L{x6v^RZMHzG(&XCqOixm^zU zr`IAPNPZXjne`B)d;<^D`<71P!U0Te;S&vx)jPyR_^80Y<&VFT$C~Ja+HOwy>@HM> zD?NmWnp49mForjHX&$@2ns$fK8==J`BgSAZnpO}WAE;l;tzmB6jUM~Ms9sW>h8Nw+ zSR9P4c<|BP2{J|70dirqT=h?2X#dPMZ9y1-6j5OA8-{6{_#&@3f1K*OA*wfxdh}|@+ zc)(vmxX-JdOGL*CJb5QbpmPbDuRv5R9GC@`D(W-`vVC9r3dq{O#Ce2eB}Ljv2TPCx z2%A4HSLY*~f6vQ2x65L{en>c*&ah56jlI+Sddf8P&Z*j`uJFUo#qfy_o+SnbL~mJ?OqNyaD4bBIcwUYUG~#MyoE z5p-s{0d{ViCbw>7)~6PCPDLJqe!t2%at(XeZVr-#MjR7!O1PhK>sJLoLYvoqqvzcRVlyleO-BT7tWu;ux8qmLHH@N z&HYEO{T)MNA?BRhOwm0A;xv&&l>K4?_qet%f3%JwLI$0il6Z%sW*7j;2zoz>LG()pL6%ji-LA}IEH9D*j*G}yFko2z`gwCS^hL#^#h z5&-9S4G&rr7iC)F!{*k5-u}c**`sn*X4#tGeF&4|e1||7Rdo|{&2tBf2PVW)HitBR zUw)vod5brC_1pLU_X2daJgHd&fi);&6_8BtDS-tS{8KOzTX)>7ob?q|YF+;kDr@%- zk;CQHu5pBDaJ=|P>K{)!O1X-g=fw7txYj5zSA34TqY%W&>sX|U9my;mqTXX;E|;T| z*WEr_(Uk)G=QqdW(2@bUh?axh35pOj?c-vSbjaDy)kzDI*$DHVeA8vC)}S&OwgGY3 zm)h13zSTWecntaSDc|7(@nM;NrCNt}jtZ{N*uS}ep}7#jb(vFZ|Kfr@Jxgc$)$!Ek z1js~T@x3SxvcKt{YxM5pD#LlSES>mhi-TTTC3kd^>I8KLF8At_!TfQtaux=Yaaof~ zYhL_ENqHm_l}1G1@FPf4+)iXcYHdpX1s&W*a6r5iB;^$(cUwcT!x) zr%Hsps)n^tjyQcptiOwv=LN$rk^VK=yP(Y#cyH6ruMF}hYO{MTqi=eOUsVa}^bDMz z%7qX$p_Z@s*hW1677br_uoju0LO=`Hy`veR#V7EMBtn)f@6ye2e2e3mkRp!_ z3|>EJ7lnavnfDkcAbMr{5LI3_P8&J+a!hDOwf*C#$q^7mBlbxeYK9#UGHKvgexqOK zrfE<{;7SG?!1lvRlO$IU=x$uwN`wPUQDY=8Spz1FO2W|Vh}fT2W{1t%yC{StNRb)l zMOfAbL#13Wo;r+-d+(O;JnXkd!e#h#X=3sPZ1bvyKt(GL1FrNvb0R0oxW_-vuIb}~ zi>5&#K^`5DiQsDK2yi)36sseBlZ`5MsQ0E=YA7SsjSp*G4@&-ti-)vP%$_o+>GQbC z(h&m;)#1XLd6kKRxoh^_#n5qAhTl;!e@yphUVM%$a7yPQ2?Ja2 zV#A1}L#(y-6DE;w=K=5N#Y&#_)Xtngb0kmHTHfAgzV5&cH}2~|Tc9910v_iysT3g_ z{mjhidsTD*8WCg{eik~{FkmdqF90uM6GK|{ts_!?;y8V0_C`$e(*u(g+_v=9kKXD)NTN+VkS0qAddZ zzuF$;!87i0wj!+DcuruqneRsS^nRD&D=;G%`z!GGPFZ^%&{*0c0=HeR2?|5n__K`E zB0lU3BlJ$V3>3ef5`2XV-9T~nte9)sXdUH6u5ld*y{!Gyo|KTE(kK+KlPOmL_Pfq+ z#$760k8IKOd9!QaPPGYTEr|Qf*)C!9WzcBD`6OoG+#ukQAq>XjOAGpRJfKPArD2o_ zmY@NQ(rRc<^10-)>caMg*vtj<_I}=VyH*@1ArpV!sUOa|fBfZllqy&3cAzi!sJeM? zM8vHP9qT6!7yB8=;@fAp&%I+h zf^Optu0CX{=o_i;<4A}0Qw4iTHp)Z6#AxLFUI8^g(aJ}ocG!Gt7v(4kzQl4`a=~Sb zOaf_<2yxJMJd5c>x#i=ICV+W$KY#W2-s&j0JQ0lQ431OSj*~Z-f{alp8Khcf`5QeB zf`CDO$4JMWhkO?hggafgxP{xk486kC3PRAY6vC6_mnVoXcdC4bOf47&GJ{)I!%U|j zqXYH%lZo}i`$7iK3%UI92TK<@rY{l(g$%6_)jtu?v7*7z_@l~rodC1vZqW!?D26eF z96ay~!9-R@0{1Fu;KCP65+PoCc;jP38cZD|km+Hw`fh*SR)G9Ph_)B~42rE6xDot; zLbhyhZ_=p^pusMg?kMVM$3(n?z3B?g8N z^^H)M2BB!Ffb~0=h`ID)uG%X7e@$^S&&~v|691h<>Msh4@nJ(r*MdLNz!m!`pxh2m z41qoVoWYeDYz5n_UwE$fw^==AuXrNZqcgZ|hw*Y}ThGGKH98C)G2J^&5gg>-I!qC}32>C(t zRX-QqjO5A4F>4ze-SH~3a?K(kXm1oJvCCrGL)Q&U*B{u)rdTu3SjwUkJM#=5P}{aR zU_QIT3g09)Phtn8*|?D{>k(y!&J;J&>S~%NoLxNYbxlEINvOg0&~GcqNJQ*IX(2{c zgqvLO3n4_E%g^bMI~_mkJ2obp! z{6L4zosV9nC4NT}hS3=_JQzFpXA0viAElER-vr6XVS{sC%vx_UjCtbNfV+`K{Ghp9 zR>7K%__Kd;EPJDi&2VIp8EF1cF1=28-93u($}`DDt`fl>MjAGtf}=zWsgV9PZOgyt7fAKh~HWfB?z<&$wJhy*n@aoyP>tOwhD-1J;5E9#UjP z@bsVQdF%{t^tpro-_6-|Xvud43AOEp{3d_ow`#oKM5>ZW$EuS!FV&Km-=J*X7Rb%zZ%rqlueh7CnRpuxd=i9{I{u(;mTC7WLF+N<`+}apo z%k{yobKH?W!7}XUj7m&;ob3ZXfz*h_m><4-rx{W$uND>98~VO(_4xixk|qGdX)s3q zr1(Tgfr-I`FO?7`6j8uyzS@741BcHbPD>h_P1r#V5g7jZVP@EWG8M_NGxtW{uHE%T z5CVS|$p%>dr6WSjlHSR_wP)5)rIB>{SrLReh)<_fC&xhfkU_p56bDMcKs2`I7TonO zatOj4G?%T$Q{~nROyv*&nf$kh|6(zqt06?pqH28eT?o5L1eU4tw`Vd6nCH#*{L6$M-Ox7R+(S+B*0 zm2ZbMoZxxBXG9WI$NuK&(!v2;a3rJ7l}%-{0NHTziWsmUNi zjrrv;f$cx=XF<`*Hz>ZZfMRRgH5zhZi;ENsI99 zvtw4=K1+kPg<0QRZ-kkDwR8DSKGyJSnvH{m4LI^T-66node#0hbr(!QX*f>CprwCh z=JKV6q6dDse|B;^j?6s>KlQd9R{+4!`Ro`^w;w{2}&Kv%E_sOOQ3_+geM#o&OEzz1ZN$;S9Zb6@pd zH?z$iZ0jKE)EHc$!WSb=p(a{@OxYE*P29kf%h4Z3s9XsBQRzv`YODyaB0R^9eg< zI}l<{dn>EUgxaPDoF?nr6`%YLJ~gk}A0t&9nespy0#j8-8(UgTj@N`IyP-qwYJ25F zbwX4#eQzL|CW0PQ%!OLv_JLIa97Z1@y za9QodW&VF(qJ@8{yKjkO`{(=)tMl>dr0Cgm{QKWAM~I5$3*eR`0?WPm=ZWtHG!lUw z@mJ^1{}q{{mAU*f@d`4fGP`VZuPL7Z`1q4+K=$v0(aN)|Hh7@ipp(J2=3o&RDwh8! z;vawuEivGT#}GglL3!{U2_~5*fkb&I4G10{i&~AN1O#42UZR95=+FcxC?4)3ac7?v z4;Hra2lYS+J97buz{Sj8&a*=!kF45Xylm{%rQIPX`nx1kJ#?8@m4mu(0H=u%pt~{` z-D~|NkdFbL%VP8N^!X~{6NIpepP{EiE7Q-Qs+2c5Wjmatps{VPWPYw_x52X%nY8}A zzX|kEWk~89)d3mhAa#%z4DF1h#>V1c6a12A$sZVcwEn{HUM?d7HIkiGF9i}^Luo9^ zkS-M*(LE|B2gk!&&!67A{-(E(dMb(I@HDUC^?m%C$`5nzN+p4ph%Y3{>39E68t={Y-KVnsP{?uTdXC>swT3A^hTtdPXmkDhE-3yxayR*SPLxd``*gE%%9H zCx#2-P-Bl5{1yh`_#U7lw=X*us6e7*R=(dPs2L(aQZ-`LCN5DX5Bw>V#e=mZ$J7=} z?n&Q_hfV=#F5etI4#O*?dG6Sk1E?hAWoz4F?{D(OQsykZ8Ui{Pyal_jTgE`%OP)C^1ZsLKjq6CoxqW53Fs=DPIh=##e>x$Ekdns74{a;vo z@v8VInS-SFESmrrt~ zsm;Kr8^}5!gv)jn>@1Oi9t4<-+Ax;KD%Rq9kQka+*ep2`p6lH>1ssloc% zwXPLxyfCV|ndP=nAo@LfesC)uBWzo4sl9uJZ}nY0+x=HmK$~h6go25sK#)jE`x7xu41=e9B!`E$}h;wLA1FksFcL4 zT~2m5$&^{Vt~~g)%fiQ)|3w8M@II`yeX}ymdH?q*_x(G2CEcmR&|@9lb9Kj(7D=v6 zB{E<+>ERzA8E~iuBYni)9bln4AKwM5$al;TqygYrUM9FpdvnY9p4FC*vAm!(oc>^A zLrdn{V?6iJyL^Q-2yE#^4YPp}Y6%=3%9Uql~81h7Nw#1E{IUA>eTTy2o&ounI* zk2k(MG_Daqm;G_WHYXF@exX?YxIy&-wjY~Xsw>^BHQ1{|`fj(h>M+)^ zaqR*cS$#6`EfAeB8Qh#i`T9@H>(GT1MP)UzQc58o8^0F0MW`=lXk zx0L7XO!KB-JwIPTbt%m&T^0(pvaAuqQ%e-W&3$Mo@S83sQCVd+Z5ZlVMHM>tNx^LH z^4DGJbBBj&y|98^)P3I$*%dh}Y^+RLY1CkF@MLgD5^2jKSfm7yUIbwILNuV&f`H*Mu;4B9WobyF!Ie@po$Pz(HX~)|2Nqurt_04**4b-P#pkOsLuIXX2|R z2R#Te*Sb;ID>M}qq)l&k9LV9dG?I7CKk3szG9n>>N&L;%oyD#;i2q{VLToLV)pdqH)K&K2^(w9v%y`qBht5{?>`+iUCKa zGtHDya#vSm>&%1}hyGQNJ-KO@0MsDC3`_?O3U4HZLr|81dP!l+lPV?jsD5); zAkwsgv_^>Lqw;JPUtp`ol$YB=i>dc6boTPWk(!1anNBoQLNvy$CURhiGro6rS>JEx zdsNC?1&1@Ax>S0vX4VCw6HzRM3h#7%SV|POr^g`tZff95PKmq*5$xh2I-U{WsUTIq z%kju!o|03UYp){A>L%8M%M3!@MJR%4uNrqbN#X0`6q=^sl$Q(CKlV2pmQw)EeWbwh z79_%0j^0fsf)9qY&bn8tlR}7SSEs*~u)$p>@GYQGMHk8Me|X3bv4=4Lag)JUlEMZi zRhXU(FH&GyV#~0lP0r5+kH7%iM0>7Ae;pA}{_y1VsgyDoy>GgykF1sfWZeK5!kjBK zSruUxVJckbjGih%!B~%tvBBM6e{7b`nW#-IOxaN6!1J%HgNnvZMB}BkS zBv-b<0+*g|M|&d^ly2Q3)niVIm>nDZWheG_;d0I&nwtEgqWb-dkFl}m!&^-o^vuBX z2j1$i(}W|gqivR|%k&#CxGe@*eTcoYV3mCsi|HaZ0z*<5GGyQF_Z!QdR|zQ8IDJ}8 zuv=I=DwNFfSXyZqlh(zR&JCMi+w!hJpKvYpp46SzNncu0FeCzPYY)YXXyHYZDur3N z8Pm=*kI@BAut-eIH@;A7-OtXy{RNH0J~(BZNe*BX@}@TD=D2%z-ool*jLp}d^qs`F zMZJY!{v2_%dIh_*#R|4|>aPB%(#l9-vsJ8pyK~Oo`BWA){L$OQq(f`+rF9jnKKZ8) zb$8@{>@c3I)V_Q%B{m^~(TO+%Kn)7a^v2zSUCR-UkGeiHq_8oQAt^CuYILh_&R*Y@ zpCME3-Ca#2@Q*YY{msm|;0L)vv`Uw}^E*zHdeVWd?ZyrLhhP!KVdZAGWqCj8YrneD zVoQ6mag#=MNgs52Ie*!&yCn<&ko=kfnF?hr(L>gWtjes&(}z1|LyQp+GZ*Q`r79uW zlB#SOctOl6Ycx-Lj#iVMGiamL;n(OK8DPtE-(M0)4wG^CYpi6GVth6;gNKqLl$aR% zDjsVFojkjJMTIMI$135o=*1sp(vq$;xaZNG$u^SnmWqE3-W7yZOr7_p&Z@H2?|kkf z;6~WS&=hmL9y8FgjUB#hY_ZZ~$=>zzjiDQbaQU5&1#-l9mgBjnqsSLt@<9~^%3SFCkiMKEb+OQJBE&YsfZ2y-4|BI`&TtcQ1D0xOC`Mymzs8q!3>|-s zSA;Zm(l!KKh7( z$_Ob?m=iju$7wL4PderFGeem3O#Cc=jWTK`x-Y;#lb+y8_p5NUP)8bsn2S26XQaOZ z(Q>F|PC1B*g9i2_sNr5gB=L&2V5#4?oUW>eyMi#N$yceKLTJr6iQ)c-C$9eyh$E9P z5WRtVH+oROoYUoPnpB+kBZqeQp!T}n!VQ(TQuiS$l52}0BPqHBaNEGvlcPUp*QAsY zmq2r#f(}9mcmbJQd`E*^@O77z#@B?1mr zf~e9&JM}~%Q4-ywhYzkyD9$Z;H3*uaj9pU~m;U4EL3LKg=w(@~)TxE?jByGXC<(mIKfZL`vux=l=(}B%B;$6jA)Q&^Y!U zxJ~x20VXjsWLf;LF8tjnTCBgSt`P$OvW0W?@5R{P$6^HMKTMyCVzKP-V!B_C2>$sC z(1%q)=^^?VB?yvY`o_-#{yy^~43l&l(ntgo=G8^Bh`&cyEI z0#yxsd$UU9SS-I|ul>DMvjrzEYfy>b=e$8gRd~nc1J(nx?5m8h1&1qtg)d_a`s&K?qmEr>1s(g%@FDoZ!`xaE&x^^PVG&f zXb|IDJFC7W1guqXA|rx99tQLqqrAbbsd0jdtx}35pE*9tNoMph)Tmo6#5%flk;3QH z#N`TXGFGi(LgM|~Xiwf}T%ch;>FC|Q9)f!2ecF-?=_NOOd|^>q8Q}sB=gINWMxPm& z!9S8>SV>&9j`rk`jcDEg%ndk6x0a^^iWqt#>~lzoo%eda=;7Yy>}ij{)4N7Q8J1#u zpX3J(-P4@>8St(rwET;_I$>C#YeV2*;pW2wn zlvmBylQLCs*o;;EzNao6Is6mZH7Zssw>&FIKwe5%0okL#)|-bPPjr>;lU=T1R)8)D zYEjkQXP1benVThc*0g+LGw->RF(v!K#QpBm+Yx#GJ>!hOaNB;Lb{7u^NQpA|lbcZ; zh2MW5hfU0G=-(E+FH~RgeVx0s=1I*fgTX((aPt~AtT-#2B-E@eAXF@krViKm#UGTn z7Q0yTXg{qg+E+B(UQHQc1+oj6Qh?!97a&kTkl*`pHlLT5+tq)n#<$b=wj#I+!mh$O zK7?(Ap(|WEQz!~97G+hgf4T>i-FKm6)gbO{S~kEUGqm?NQ{h{uxk*I#nn2{6U(Vw< znd_o|v|F&hMji((x4pPmN6FSS=24et^Pv<4?mwG&>RC5-aj8aimnvDH{`-3O_qBWQ zgTV{&ByiPGq+8$h!)BguINEYHfiIIR&u>)<$T9(tH<$z!v|;La2r=2#Tqn75de|OE zo=L_Nt@mHTUG=}Zb6p`fnm217b+Eo)rTFAw<9lu=kcx#bDHDkpy32Z=casa}r=I!Z zUBH~d=fqpY0JP(#ggttnfRr&0Lr%YIR4TOOqn7KANU&H@I5h+7^FVYz>X=IcqZsx4 z!Dp%63SS50RqLK{%wvWVv+9&4S1)dB(t%_0eSLLYDT<7?;$MbK zlZw94tM>y%6bu7n3E+j>I(_Mteq|zO>2;sh(`zArMSm)) zPN3K4>!AaJ1ChI^(&B@Q85rb&k4-k;XNL&ZAyU7rr-qe^6qtUuHgUE&5nTD3hAZ-6 zw9gWezGxMkQ3BeW;yj6 z7i$Hdz06_0vaF(uYi`V+_hR5p*&2^|$j_InnR*#sOfAw)43EdO9zTCMfMPLO;6!4A zc7A+$uUlnkkw%7z9oh0ImiuA#zN(CXPCi-F7cGhJzJfJ})DG9b)T#Ee^?dwBUBApn zq3E^v51hAbI+Ek-DRFz2`=t)L#%B2!ultC%(6X}ix5-4eP+w4BE)&x=vb+%@JC1ID z!nTBqmT|#KWb@H^UEW==oUb6~dv#69IY}z9qo}PmYWnx+^`qV|={ixi&i2;dDY#Mx z9PFt%R{m_(Ed8M&#yGVT#2W&il}2^7$M0R+VtTDS0gFxuGxX!Q*^%LC8r04}ZJSb0 zh~Xi#(|&TJ%!sXf5fK5X)sX<|0jx0rUzBm0DDWm+T_=d>4v(^qQuc@YM(j8E2|p=a z`jq*?;VuOb!l1?XdUn{vxH7Q&OQ=Z+=|exN2COUzKt%xrD1+>l*coS32dC;!f=#Dq zGLfuPf~wPHTwM<{?W5Q9oGz3&TvS_+!hW*4k8Xv9h`!+kWuZ~bx$H9AlT2LPU<3c~ zbHtFcf(E33cj(&z3ef`-sQj-ttPOSoq}!n*x+)r2A%PC?a<+* zgsa>SxkL_ZO-`P57jUq#z;o$p`;=S8u(U5axq=AgF4E2InW|5vF$PTB=`b^nnqi6N ziGAKf_vvzW%zZV+6G3lcAdBp&r$n*3><6RThb@_6RSg4Bk^gdh2YGNP&l^0XY zuU`_`)YyG(1u#3}zR?(eIPVapM=Pb_-Rrl5kOJ~@2}~jxf#UX_cI;tb*=C%$|H$a* zp?Y)>vJKs;vQxJm8-y-F%+}J=Yw*m%6*id4Jr}>xIyPvy2hFpqFITVMs#x25`q^>R zLJ5te$DP_+k*FVD8MpkA{OyNBGQ_9^jb(uaG!UDWcXt3_Vy)Ms23yTuUuIg)QhJB& ze_c8o!kK2xFOnngB>`dk%|8D|!%mE8hhY*0q-iMO>+M0C0y?68#coGkXo4l_3`JK5 zI83s*aDSK;r!b|V0rQGxwRtl%*@T#dhj#vnJnYy5_216kn}f{nSW-KjcHxT0jOLV1s(I&!GT4&r=^ z`eOHK;gPvlJ`meTHRm^IzCf3D|0$?ct0M-cKhGE5e?$pq4GTx%U4e}q>6GU| zUnqB=hpT<<>+8k_UgvT(*^}kDx4a^PJB%et@UX|01}j@>{#hl3s1-=zhk=3ppE`CF zebf>#1u%MgZ}bJtEN_Ap=H1yWy2p|)mEM!VIl2cc;#RM3K7#@LxtjB4We$P*zMa*e z@!>+H^yDR!cLm!y1GI6vY}?JxF9|!xf&*BTUOC&yIAIqn5tTv4oVfo?kzMj4pI<{n zpRSEYP!JI@CY=~?k5wmxkpN6QJQQS)Z~fRa(>VOHbMTEwhxssyr-lj1rao(AcrSHa zs6tgU!TS@lqtu5lwiB1r4*|HAmZ3hpQJj5q*y6K9VdE4uBZ*ntr+kp2X?H~urKgrE z+g)}Tj+eVd++n4U;>1PV`ZgMOK|;=%Mo=7e9D=Y~wrQ*om>P5y&I5@$Xk!wR=iG-!Ha9o*&3)OZES6W_C@1X8SF`$WE=M-e z_Wm|nn)2S0lCo24u(4UPMn?>hJX|VQ$6DGKBvuD%WfDej@YxKTinfe8@XxNhqnd|K zS*QTX-?su%LnuhJvq;LED%*-iZnimjhE1vflyF6;-a1lm_ny9r83}C1C-sSkF~|Mw zEdQrMKN=PVL9Ga{Q!4z}TCZAc&kh5Pwrk^7wg@fG%i3vgjA%)o=Y&C633f*>8WiTT8+0|}r=ib>-n45S3y4|;I? zw_Yyi$zZn$J%X!H8iB$cDf<=yJzUUAy}bg~!=)z5T8SU`ibekpLbBZ7eC@WyUqhk8 z1JHcB=#4-8E~5fP%QFjKyX*Iq?)u(wMG!L1TJf4@^#VAm&$DQxIa7v@i9xFA<=!z1CkQ>N@r1m+(cmz*Z6o6?Ba4 zR$>@-n|bHANDQ>O$l+LXRKbo?q4vtPNV{wXsZAc-QLI$E<(^B<#YIO)6(dB;@GxrH zRYPV*X&T+I(eQ;U*t1+FYmEwczyi9^!M*d#vM7OTFgiifiA_V>*!;&-OO~F=45(#DNwl;TC~CI=C)p!qOJn_s*!fNN!!qQSr3Id-72Ar+1Xv7$Y3{ z;O_j+TUOA85H8oYs~JoLPzS-KmBU)QG(D_K7Emv&$A&iCiUU>Pdn03$JtP~vi?#hBmvHdFfZwR(w}Dz z8Uz?zelZr`BKY13m)^&|jUSY&4%vtCg!dD5k=CuB@=c?Pg+Qr*g>>%uUIJKA67LiQ zAj@!pzAOeVNai2{$UW?QsT-uae<>3p#g zq?pgpRXL3eRoUrWm-*_%)NegQtoQ|i`??I=+bKwp#J${`2)I;YQ@zSV3+*@v2f5vw z*UNJ%t}scKP~gas{Q=7Wn@R(ji6+BuBKTKrs&u8xlLZYYb<`eQ(0g@x$y?sS`ZkRr z8tH>zThK~brO0^_bAfwiI^Mgres$!whZmbo`}>gfQ$bX7PE7kcad0Nj=>2CuVzC~( z&)+0BVC4y5?=IArOn8d0MZH)q{7Bf7x|?Vp#Aa&e&?r412Z&uCDhF1eKH4$ODe4|MNf$PR=Z&^?H3jw z^{xl6N}kjc!j^K~vKknpiUq(#)jP+Rz9?JW*2clbiqS~X2v~FX-rfS)`)sM;o3w*= zgBfyz`NS~p5&5hpTfO^tP0Tc`1OkzJb(NLpf5xGT<}*1fe$&!y#lW#r1{(bs-jYy4 z89~O-?N|=D$3~8$y)}R)zsdKEtK=E)E6&#*LlANNvG)tVy;h+h{I)?8jXl4)pGN$h zafp_yTS;xJe;FPvO96(&S};%CM0vQZ9!{7|EX$!=lM8o_je02Il_Y>p*nw+qaLZt} zBac(&k?^_05$ORomiRM9Rn&pji~dolM0kM;aDxzz#2mR^GFJ&A(wft4MA5-t9CvDI zEoHH=gxrh*RH}0rb-s{4dWYQ}jAtk%m`5cjm*^GQsB*f?l3>nzPJM;u^Tv+$-tUg( zF70GNqlN5C+(oi`Kem2aP`BQHlo|JGW5}0C|aNkd~8j5UD%?HN2Op6FGJ#(mmO9c`+GSpxjxYJ}D2em&=#`!%b z6o;TOUeWq4<12LW-k0#!9PdAjkj25l!z|qJJmU_vJ=4Bv-e z$~fVsh1udLh1$j^M`^@Wej{BbpPOd{nyh)r2Cy7)D!^FYap?C)-TLw7p+ZjQ<0!`} z1ee6=Z;R&3!9nN=*{|O;Xx7@_xfv>IlM6>Mtgl;;E-e=?Tdt7+BPq46hj(RaGp++A znvj91#tOZ)wNVa}GlpRk36sL9Xryj}71_#5-W|GuE0ntV$FD?0*EoymyjGytm@5iNf z)wqm@1bx}TE*m{hdWYz;+_uJh2m!Zd_LnSqyDe-CuP&72;)+-ntY?BxE!m4jM19rr zr)^5!yj1l&cweyv+w|k+*kjb@UaHYv;ph?x(3w-9-^d2tkwABrh-zsaGj^aQ34W$Mk`m*whNJ{F!hDX6ykB2+e9H8|zh2`?HX5MN4`1nF&6F zAoc?6A=9uv1cF|AnXMTc(i&1eJD@uuVMZ6)iu|dJu|s)$P5N~GWzDs5fOT?VhALqm zH+=meRC5LAet5INNXOh*r$4SaeN2M$(ZX^3BYt`L>pUn8MZDN1fKX&1S_b z36nfnPE&g;r9Zd=fns7b*u&YBQH552=83hhjWcF`XAR&mSj5I^7OX543Ob(RN=HBM zCEzKFpw7~yI)vXraVeyN3m40Cy}m3WU{3MN4=X;txh-9?JI`rvTzY<%q^AD+XBHLQall@lkbwCxZ%i#nPxhwFyQYhSu|dx;Fl z&q=-8Xqqs}3YZ7agxp|(^eZk$sJ6ASr~~gv9#5x?Iyc@*9|!`{+;0%i%AAQxf0k zWz_6x0m0lF@chYw9_wTCULk`!4fV>Qa0Bb|=!c+E2P=-Ejr*kAcaO%|ijt}MF z^)O0|RhacHoX%=cxH7`eqR34hs&w8Cs})IqkSNIq%@KlbBU3qM)RlJAVQlkRJ~3M! z^Y~?yYhi-cPtv!wYs*szt`P(MUL)VZ-^d^#Z`bg+xIiLo?AO#*>d68WnamZvMvORP z?|wv;|AT5H#1vXwRGKH`Ivopzv?j^VIW&*4&~{Ac<4`g<+LtlC`xRL`;|(5U@W=@_ zjR8J!SZ;KTr0lpI{)@~4a*R>EoxR&nqNuBR1mvz@ z#n+=Buf0X3m7-W(>e-{eCPGPC1ek@FyHVN|71~C*yc$p^XbOX z!qp6wPtl4sUUa|jS_DLrLPx;As1x)degVF_+s;h-!jG>pv-tBn`}XG6LFDny1 z{YqiXiwk8cHn#^aeOJ0RIx_N6HX}LsUt;6mu$3}bT^UC>@PCi%fHK>eAEt5MLCpD3}5xhxQ`hOmLR#^z4_+{k^=aqsl*Kc<`uK{P^KKTRdF66R+R}!SI0hy ztLud#1=`m6gzo!_w5{K|4D_HG)Etx%4YzRhU!|3-quw#6iKpm5aM)4TpfDN%T$mig z5o+~>K50ifbmF_y?W~QUQq$YMe?(X(vCZF^(=3NK!_yF4O~vJYX0Hu1>-U?CBG9dq zOB^u+|LvV~lw2vmi3LTx=%lSGj5ggsLXR(a9P~0cpTchpMGkT^65xhTaRqX zd7Vd)rgCc}{$B+BuZr5T0hOZ&eirR-Q5UswRpX9BT&8`g0{8#F=gGodD)TQ=l0SUX zm8-M9!p}4P=@d#b{ks97k60K;X0R;vyP>R3UyMICZY%1Jh$}V%+b`dTkKdZ0WPkl( z`uk&&5Og*7v}A>PmF(QXljWoDhgt{YUNNJBm-aUHpNTDSPhUV)_wA8f62<2j3x&v1 z6#S$8R6!!fmT_QM4|2!-#iOOU$MY&U`}n19g1;sTi#OAMnPxhn*muib0u`u00PsQ; zVP#M>_OB#bBp9EJrM%MDbSO@8AT$H^7g8VE5u)IqyB@6%y`ToF5WP>_1H7{1WlI47 zss#CfR(vWqFJku9ObF}$I7mB);CHom6OQmD5YXc!Ir=cTf%k>He|up11nMTvP|LWo)a{5A_G9yl1xh_ifPa>#_W)P>9wF8C_N%K!jO zVFKPVy5IB9)!Tcl%%!i(3d6i52ILSyzM6xELQ|rm#K7HXV^7&HtrNy%Lrlukhj5$Rf5q(dZ^5b2f%38hmSq`L&A`>y`}_kOzf z!+qWlyYF+(%$YoMp52)>t9kV+royNX9hYfsVu&cRvq68D@`om5k`--Ufc%@Vt$g&W+7z4dc>hP$LEk*uWH+ zCvq{l_vuS7m7yb|u2pH|h@MH4u7dn9n|4B$Ew3J}eRIZ#h!G%Yg{y#C-uMg0snV&_ z&ueN<*m9@(gS${mh;l$jbMZMQ%Rvh|PTiEME3k!r)2^~)?VNlvbCSczl-p8RW;|4tw?g;G{c ze=tIp{Ns@scd|Q`X+PKbP{h$|(diQ~$#Nw=z*Oq zlj@hSm--BU{B1R$&b*SD_8mNP_mfLCUQAub;BpW@vy#G4pC(G01_Is4|FRy(1~$&< zUi+U8@#IAbN3Hsr+nG_LDjc7T(A!$t2#oH*MEYsJKrVaIQ}yoVj7`zuyNhXgL;wM3?6pp-<8{3Z`4j9A8m5HWuwIkybo}mQq;QL( zc|)6#lhy%+!jaU6a-o24a-s9?(m|Fq&GnUqBsu&uey zb*^Pw4q;9ldVrATBr8x4`Ou|N!ECM6^SR&l1#MxDva7h_ZJuB#lwg>?;4eH=Hu!xX zC-xM&0(~?6iY(z-v>4e?067X(Xe^*>8Q1Z}K~q5VN!%R+p9Y!@q)>o%_Li{oUd2J7 zxVg!49a~{cu<+sACxs>f9TIbkVhYNpL;>*BSb_sI!Zi*R4T`zf-RAK7cY?!6&CuHO zt{7aPcm}_CG#Wq*VZalQ(!Sq-1JuW7dZyt?)*5`nWaK>;e1wi=W9^oL65I2y(!S%( z5D;_RW`V)LHfwErLMUt~g680T-!Se1%sf6eMSGz$6o;uXG8VWAGY^XO{RLaDpHv@% z=6{B!1Motq)7slMFhH2X?~f!sxiTvZkoe#|@pY+YSFt}sszX7jP}@!t?2raen@C9G z?jZxB#4$MaH4YwdIf;(=y#@<304rpBKcM->Xo&&B#n%2{SL(^KWc^Aoo%j$T;X|gv z7{8MqUG&oJ*_E{(*)MW0GCFZG9BeBD_?FpyHdR{|VOMrgK~FH$irr3_Dfa!UCiBu8 z{}Fh5LGDKcjqld`RhPoEoc;+K1iA^mK_gYKaLh2`8Gr}s@Qs4hc9TP713J!2x(KHN zJz9=V0y=iG~4pha+b6-p@{nJ33%p09a zp$*0XX++svK`7@Y)_x=jl^z&E0}T27aY$AtIjat51$Q}`HF`WLvTn-TK@>k&SF^x?a(6w z2d1er#P*3J9R0adKr9!T0L89>t5gx1us!FYc=8xNXOK)V>`4 zkwX5_zgoMi96BGB8nwWGtNc0#{_H{Z=7a#I%y9VR{>Lg59?{^~=PY|n{(;Qpq731- z-LGfz62dC6s^w^N)RJ_e?Ml4Kf~~%}IhXt>BR!|6RsIk&gsMv9imYWQA7Te@?{n^9 zLK>BL30gd7(8SUPNwPtr8?b=Jytl&D+}gmjd{cWgcunaKuMom+DRjwozxe}io%GLW#v1zymHUQ79B6YyJBjygWApbCpxA)&BM`Jfhl;v2VEt1 zt*76^TUfH1c^EdZrNbEZs*<>?)hF z!(~kmY&o0->-mY%Kk3zOUBeYqjtLYd)$3jI03keI0 zL-0!Vo>$5TXvuJpSo51?>}XruEX0VkbuzusUmO6V#BUjm%+sm&#AHlg5ktYQdf_^5 zGzILHR!-ldNBr~V)#C~K*nRIUo)qdg2<3ITQrNgR9J6KAexa(!B(Jwy=ovg@Qb5Qe z0O&+!n;rkI({&0GY+we};|5EZuG~cCr!!q)3)hjMAG_*vsXE~&_6IZ8ab2YHy^EL_ zwarr?>ioxC8*}9ZjLq_yOn(>zh6Te4*w{T`cwW{V6IyvApDP=f51rKGG(+2yo2uyQ zHqC)aPyEdnM05-$`~{MET}KBzG20w~>Is9n6bpXRdmObafB-oFs{>eC7Y)>)9{24O+t|Vw`d(pKtP7 zb=bZxa|;QNXsoK_)@`vk8LucR6B{W>F}iYYv5%Va{zU2;w`SG-{aLX6N4#z7B<@w) zn@4nhronDLF%?Hx70Dk(8Ne7q1M(oGkkQ5P?#21o7$G@hbKW#=M0#?FK($p%+nV6%3?d9|Gx3&p0$*hdx6i>>N1h1pVNDX_PIPj!i_}rug35 zex-Q0Hzo+;Sp6VJjRw1j<&8IOJU=*lpQtt^uKCzft8Yu1gnEhIbU(W~fS)Q)!|=dE z7Go0uxgVmQay*x`<4vdEVE%0-kY}V5BgXpehfpBZnv{itrVt*&zB()HL2tcwmXpgK%Mx>B8DrgnG2TLU}-dYpJ zYuE|N`2KNeN2-G3WPBCaTvmH{u~>3{%{XWEJu6S`J*W>lSb%7)__maj1Es(J)MPy% z>u7Evs+5dT%pBBVLh;YS{?Kul{5ILew7%xZ{)yG#qTp^t(cV_Tj}&)5a`l`4j=HDg z3}-0_#qT3zv@YkSf1L z>{8(SXfqAM8h!!8PhZTLf(e1u+D6R%ZTj7#|;4*B$J>LMWZh}zCzb3|5m=c%Ht6fl3 zV_$XkLUlEG1Uo*kQZW9lq)tuNm@z{8aKD;kFCI7hgyTu894893xnbMbIx3re-8Afo zY8AY0_KEXn%-dY?ax!l#*Zt-9mDR>52}0pO%OU?*R9j=jFuYS4FNyZYljXSOYG6H4 zpV|>HeU!jz=m!LC^mDQwdcE_*s!0ab%71-NF>C1R zY?fRPA>RCvSCVUIVk$ns9rn@o`<7&g<4^GSnyq6l#v9U75UB3GOZ6%{`hdt^*u;rU zsby7t1jdmam zw|wXyeetA?;cQ;y`E^^>zHC3GhIz*1J=_r9hSvTv*qIDh1l_hwl7vJUNNXrhE#sz=ZrqUi3A)$yj1ry>jM)YG#+SU6Pzwy2)q5Fp zoyxh{z9oe|LfpMdp5%~hnd)11PKh-M#vZ#MG+`4AQ>r(vd)Qd175M#42ZvqYl7D;(wJZ{;4}E2VIO`9jPQKkjBEvOb733P-zc{rvt+y+~InBn^3=`J1{4 zcw2~{rhq1xH6;0v$0s3P{T3z5VUOQjf2g#o_$hD>F2d8e)p6E3PvH3Tf(JK}(fCVz zQ&VKWF~NtXhRyVgxQ|auwjIX#k8V(AHUdu{9j%FF9r+G@q)fj>ov&YZsE1I~jCrK;LX6e>?K>=^1=^4d!*YDArkyg z0xdgwKQh)@LK`U{?=p)drya;JV9K|hOC!ol7YBflrtGf={1WD8#BmPo#=nn&h-Fgq zj!3e7amaiU+N@AY*^q1vM|B?tThF!@pCcg!V(cr&YsHy z08o^gUh)2;_kNCB`JPtX1qV5(4F~rO9sXxO+-%_TO(MKDvl7blgdm%J*pL@9Wy%+P zYe1>DgRt8Wf{Cy33}3cA>C)Oa+#G&b;gWMQV1!r%E(CH&mynk%3Gl)`2bU#`z0E3G z)j;q$kQFHj!0<2ySaKmJj1P*z({7ZU07(6Z!~QExED;g&<6dBGsrw|Vahz`KF}MVe z9E+x zCxRvK_L=AOt~WBipn0NJ7dTpIH0JUXc8cq07CclZLhH(l@L1otMP+y~OB;hgUi+}t z%{W%{6hq=wPuNsW`NLrjPG&x-nS?{{rbbh#5FtTH7+1*B7 zkwMDqm@K%wkXM_$I>e%$Ped1CI$RHd9M1;rd?ZFsFl@wu z%GJwd%3@v{&@9(GH(l=f6A}U?x%vno6sXNG?c0O;+5yBlHGJweqgk>v^9`)7l-lwk zL*xRXL>&q~s-1+Nd&(Ck=@E5bkLJnM{-I=#@SA=MG~b%;Z%)3(+qL5#7giQ)5AJEd zLa1GNiCDUP%}!GYPS_b3cNl%fzo~~+LVF%9)=V!?mR9O zV(q)^!n|I?O;1TV2>F~7{N8jsK^;2}O(7oo{+jO0j>?2aj+=G6I8&aUg%I)}zor9x9 z1alRbJH%_{x7c40Q38QYZx ze7*Vgyrt64R6818egQh&pK-j)!VbYSRKvost&bm~&~lKbNnW*1g7Ba$eS!I*_l6N_ z75=hzt3%HQ29AGxbMmst$81&3G}OY~K$sJ^`mrw;Z_=yij*6n9WD8S6-J2j_#GyK8 zKNGVPUH7r?C_J2EU-6njDRh0^Hwx>u>hj@Rp5h?5hOQeFefQ!nPdB91sY{d)5x^J# zSCtLaH+fpeF3U8O%sb^)dH_V&sUi#to>|<;R)3!U&chLE=(Z$^t+B{CNf)Bhb4x3I zIoCh3$%l9%@<_+-Bue+Gf4aq*8L`cKN4p1tSG3Rp${fK1TBe@~1|sY>C4Y?sJwl5x zyBHSY&)6s&dANT|3LQTck(^ygZ^VV{j8pEvkq!FV?mw7ba%X-IJu-=o&7(hInVVkp z@rj5PeVMW>`z{vlB|=ZOt1SEA^|l>PM&5OW6AI3zW;JI_0)-X?zvNO7V-H?+U03Ma z-{YVJA>)gyFK`AA$6!pVVod9*-?Zf$0g$b${4Ui`0=_c{-hZMm#R}oZ{K+&hmdU#N zHJFJ10rZ=$ut{CrtzEIJ{TDl9To8`JWSaKI$!L0!(Y*};Ba?kjo~NwMNlxdwLgz&k zs>hXiRT|HAJ}ufQ>#~z~?*lBy0pt?O{&K^1Oo_rh%uwaijga+KN-e=L z*Z4O&LAqKA9vQ<)u9t&@Wsf?!yxqHBTn>L*4GLcs!2zm-OW^T`p1nfz(M&WP`h{D0 zbelMBhG=CYmeCbCU@`9TK$J+2OO8n1ldg!%aFIjbFYNYcS^qkv%h2ycvP`rSt7Qa# zExdyYW}e*x;T;jt+&7;~-{4TyHS+h2(pMKopf6r9&6$-S*@e08yyFMqTWkAqKc%jb zPLt1b%+!i#9v*(7t%NVLPr8AjlS#hzLOHBJjTHAi(#{lt-bt4*TB&~LiV`f7P`zMf z!v%r-4AZ1ugrnfF7l$iCc>3>6*EG~VbY#|bm|5}Pjo^X0gSQHpl6e8Z=(eQe^Bmm& zAul_BKV>w2c3`Xoo`?2I0hX=I`<sCW@F3WDb5F>t%u4)f| zqM!B6Hd1_CRUv_F2=7cbwDLpcsiWS}{%GaHNr=U6Z4dlmKzZr`F6XLpI!e`7zhDPRExMK z9(d=H^4SJ{{A}{&7+LXL+qL0`3@+6lR)TfR7`f0uR^qJ;nXIy6=~bUQ{a`ubAGFG0 z8Z|tC2J0KF%eG7-?fBRWB!*RjUdC)gOUC^=-XaW^(1@I${*wLR7UF+I%QdPiu3j*CxnmUJf zY3Rc?Vxz#}%8^BLOnwO_cqa#9@5;&)3u+pIQr%D;?WxQEXMNldPgisxV_vw0dSN^X zghD;NgwGuvCs%;kIL5#HsO~etFv?qeTtPGCk)0bK25_M7jaPJ zisrioOhS)%%tn#e=U;Ci1K%rHTaL}dLGee0AF@jPMe2GTI}(A=8g}TpC>N6u-K5c0 z2BwLvNNI%YE4s}bTfwo>Y|FGsOn8629st2Ej5Q`#qXz0{!$thC-F30y)JEe`_Zs)Q zVwSte5L@hx!_Q16-u`M5l+KX8TkJ|nNN0^2yjT}^+Oed=$NiNA#YM*Xy^7$O{^;I@KL=qyW6l(C}UENrL+F*)bR6x93mF>eqj2LO;?zR6iK zu(0cT;$gqFw5{U?H2cFL8s3HE3oYU_=MnzuF8%rvOTj!naigpRrom8{koVhSs#bHc z(O|N2pZE8}w?O21UQU}{&U}>aiPeM-^qq@ki4vV#2qWqL1(!oRN|ZwpBeoo^F!+ER}SmQ^J9RC@sqi>C+bLUIl zg}q;Xxuz|8I;5bH3*N`403r(_In89K#W_6l7XGl8r>wP?#|c|Fne0WW3(vSZ8huLL zIx~yY7vo-Wzn+l~tn6FQo6E@i9P+wL{i5mZ4sQl0lGkPJ%HHVFW@UA1=FR13l#9a; zFUgp%re}}ldmkT6Q5Q!%NYvt_h28K|l}emqin#rp@U(ww0T@9C;E>~y_kSv@$R(*x zGITOr;wrKXm_hiuCvOwg2M;nL)2n{tr$g)+hdg;uEs^Dh$a?a5+SyJc$>fDyYR;nQ z7xzL}tTk%Aw=A^^G#`T=f^p7B0O1`EseofmQ?T*DC5~qkItF9r97u5B+p*ryj(ZtV z>0L~-F|44H+)J3z@D7lhDvz6cf)8A67oIW2n_T)PrZ3h0A&2dUV5lqv09bOO3fX^& zr#W1<(}J#axO%zMwb0P>0+Ms1oEX&cgJh_V&G>n%=6udK<#BdG-G{j0 z*Z_c!05X=hKh42}x@5do<5c-@2hr}p2u0XGrwN_F*H7Z1PNH2gz9&hDcl(^M#1W`p`@ZKks} zRsa+LIDG_?G767bcglKj97RXw=dv@A4#gGfRtNrFX8Hq^Zbs-fO};&4l;zjszjU@` z%9|G^+}%T(aTd)l)nfl;VNU@!{ zp&!TFU;T$P5-OSC*&}0=tt3QdP3SY}KpE%joiYEKYN2}U!$*aQcIuX$JW-*-T7vPw z4!-NMHJ0a#ewv#!E@Cy|0|*_rC(Eb;+E8kX8(lj}!~G70HSE&+GC@XlapXWzV&mVQ zN|}LIpuZBUAVU^tL8T{I$;;2FUwu*2r~7+}}@q~qTwJgT1>NwKh{#DrZ27~ZB6 zw#{BhH}WxU**c^@$T`V>$=#m*@zzmPJBM$XTI(K>bfF#<0Dw{E5?LfXL3u}Zi8SI- z+3+;sS+M6wvzJJOo*}ZL=y;@oYdri{2#wG=++#cPyfY|V<%v%W=k0~^_-$$*e+Qe< z`JhYWfrzz|W1!PUqxI-kK=b9B%O0s5uYtOz0i|&zMAFwisHM3Rw@VbCyHZu+@7sf*=5x^J7I$GsN!k@Xb&M3Cn6Ok0Iz74F~qxA{M_3lkcR z2>oW|bKbR0KLW8ayQW?;M50y`R}VvZhh>R?-rH~UpfuNjNUdh}yU@}5|62|wKhF@< zyN)F|?GLTTMG1dg05s>&#;b~&&k7Zne7(t-}gj<(|%#K0Kgi14q_nJPA(F<_jdBV2<-gc@jMd`9OaE zSH-`Q#%iGB+zpOMyQj!{^&%k)q`@M)FtZlK=cmCx7nR`!G3bgigIsRDek}KTGOP$0 zh}z>ZE%pLK(lW%(u=xpGSwY&&yPrHetiGz4X2PU%`1o3)yCr24c@JtR%O1sQEOtE;VLS4w0y1f>l5wZbyF?&XJf(nFB!Hlqmp^kw*Xhj5fY z^xi*vHf}y;Kld)foHmGE1w zFBV*0R)vjWa;?X!RLJ^Asr3B^ERpxbtEEky*l>7swXX0h4J7-Df^gY{^U5tkbgaQVVQLCAPz%Lp)C!E&g zA`b~FN=`*OqW|0S4OSNA&zRJa#@xsl!Dlj-Gh>n?bjc9ocDyd1NfAHcF4zSt7 z>0$pHWEh|NW`W^0Lz3`>hl_pj42zO|RgZ##kQSrMh^~4Wp*3ZgN9wk$dx3K=oD%pS z5Ans`_4AFRZa*eQ?u+Uwz{H?j^Gny%q%JNOf1YRkQj2ySr#la6Z#!YJ^JDGVPF=VEe9OV5!g92`3 z8@;DdWu(MWJG_7l>hjnGh8JC|*WK#!2n)6ic_ zlR@YS?Kfl4yb>R_wb*mt>M_@g<%e!Mnt$rdrQ980PykN&;)k%^GYZ8M=-1XeRctV* zP19&ju2tFO%l36gN0xY)_C^;90SFzLW!NG6K*i#fQ^D0tO9;oJ>9O>lmLQge0)~qi z`f&mf0TGX{xX1u-tVfTH7y#LTJfS%`X;(-90$?IM7qlpV1IO9|bCY4-wHz@kxPmd? z7Lt-`TstGF_OW={A>UQ8Q}kSM*c1~f_Z2_$?$d!a}oeNRnU+xgPWoL7t9`= AU;qFB diff --git a/flash_tutorials/electricity_forecasting/electricity_forecasting.py b/flash_tutorials/electricity_forecasting/electricity_forecasting.py deleted file mode 100644 index 621cd3095..000000000 --- a/flash_tutorials/electricity_forecasting/electricity_forecasting.py +++ /dev/null @@ -1,311 +0,0 @@ -# %% [markdown] -# In this tutorial we'll look at using [Lightning Flash](https://github.com/Lightning-AI/lightning-flash) and it's -# integration with [PyTorch Forecasting](https://github.com/jdb78/pytorch-forecasting) for autoregressive modelling of -# electricity prices using [the N-BEATS model](https://arxiv.org/abs/1905.10437). -# We'll start by using N-BEATS to uncover daily patterns (seasonality) from hourly observations and then show how we can -# resample daily averages to uncover weekly patterns too. -# -# Along the way, we'll see how the built-in tools from PyTorch Lightning, like the learning rate finder, can be used -# seamlessly with Flash to help make the process of putting a model together as smooth as possible. - -# %% - -import os -from typing import Any, Dict - -import flash -import matplotlib.pyplot as plt -import pandas as pd -import torch -from flash.core.data.utils import download_data -from flash.core.integrations.pytorch_forecasting import convert_predictions -from flash.tabular.forecasting import TabularForecaster, TabularForecastingData - -DATASET_PATH = os.environ.get("PATH_DATASETS", "data/") - -# %% [markdown] -# ## Loading the data -# -# We'll use the Spanish hourly energy demand generation and weather data set from Kaggle: -# https://www.kaggle.com/nicholasjhana/energy-consumption-generation-prices-and-weather -# -# First, download the data: - -# %% -download_data("https://pl-flash-data.s3.amazonaws.com/kaggle_electricity.zip", DATASET_PATH) - -# %% [markdown] -# ## Data loading -# -# To load the data, we start by loading the CSV file into a pandas DataFrame: - -# %% -df_energy_hourly = pd.read_csv(f"{DATASET_PATH}/energy_dataset.csv", parse_dates=["time"]) - -# %% [markdown] -# Before we can load the data into Flash, there are a few preprocessing steps we need to take. -# The first preprocessing step is to set the `time` field as the index (formatted as a datetime). -# The second step is to resample the data to the desired frequency in case it is different from the desired observation -# frequency. -# Since we are performing autoregressive modelling, we can remove all columns except for `"price actual"`. -# -# For the third preprocessing step, we need to create a "time_idx" column. -# The "time_idx" column should contain integers corresponding to the observation index (e.g. in our case the difference -# between two "time_idx" values is the number of hours between the observations). -# To do this we convert the datetime to an index by taking the nanoseconds value and dividing by the number of -# nanoseconds in a single unit of our chosen frequency. -# We then subtract the minimum value so it starts at zero (although it would still work without this step). -# -# The Flash `TabularForecastingData` (which uses the `TimeSeriesDataSet` from PyTorch Forecasting internally) also -# supports loading data from multiple time series (e.g. you may have electricity data from multiple countries). -# To indicate that our data is all from the same series, we add a `constant` column with a constant value of zero. -# -# Here's the full preprocessing function: - -# %% - - -def preprocess(df: pd.DataFrame, frequency: str = "1H") -> pd.DataFrame: - df["time"] = pd.to_datetime(df["time"], utc=True, infer_datetime_format=True) - df.set_index("time", inplace=True) - - df = df.resample(frequency).mean() - - df = df.filter(["price actual"]) - - df["time_idx"] = (df.index.view(int) / pd.Timedelta(frequency).value).astype(int) - df["time_idx"] -= df["time_idx"].min() - - df["constant"] = 0 - - return df - - -df_energy_hourly = preprocess(df_energy_hourly) - -# %% [markdown] -# ## Creating the Flash DataModule -# -# Now, we can create a `TabularForecastingData`. -# The role of the `TabularForecastingData` is to split up our time series into windows which include a region to encode -# (of size `max_encoder_length`) and a region to predict (of size `max_prediction_length`) which will be used to compute -# the loss. -# The size of the prediction window should be chosen depending on the kinds of trends we would like our model to -# uncover. -# In our case, we are interested in how electricity prices change throughout the day, so a one day prediction window -# (`max_prediction_length = 24`) makes sense here. -# The size of the encoding window can vary, however, in the [N-BEATS paper](https://arxiv.org/abs/1905.10437) the -# authors suggest using an encoder length of between two and ten times the prediction length. -# We therefore choose two days (`max_encoder_length = 48`) as the encoder length. - -# %% -max_prediction_length = 24 -max_encoder_length = 24 * 2 - -training_cutoff = df_energy_hourly["time_idx"].max() - max_prediction_length - -datamodule = TabularForecastingData.from_data_frame( - time_idx="time_idx", - target="price actual", - group_ids=["constant"], - max_encoder_length=max_encoder_length, - max_prediction_length=max_prediction_length, - time_varying_unknown_reals=["price actual"], - train_data_frame=df_energy_hourly[df_energy_hourly["time_idx"] <= training_cutoff], - val_data_frame=df_energy_hourly, - batch_size=256, -) - -# %% [markdown] -# ## Creating the Flash Task -# -# Now, we're ready to create a `TabularForecaster`. -# The N-BEATS model has two primary hyper-parameters:`"widths"`, and `"backcast_loss_ratio"`. -# In the [PyTorch Forecasting Documentation](https://pytorch-forecasting.readthedocs.io/en/latest/api/pytorch_forecasting.models.nbeats.NBeats.html), -# the authors recommend using `"widths"` of `[32, 512]`. -# In order to prevent overfitting with smaller datasets, a good rule of thumb is to limit the number of parameters of -# your model. -# For this reason, we use `"widths"` of `[16, 256]`. -# -# To understand the `"backcast_loss_ratio"`, let's take a look at this diagram of the model taken from -# [the arXiv paper](https://arxiv.org/abs/1905.10437): -# -# ![N-BEATS diagram](diagram.png) -# -# Each 'block' within the N-BEATS architecture includes a forecast output and a backcast which can each yield their own -# loss. -# The `"backcast_loss_ratio"` is the ratio of the backcast loss to the forecast loss. -# A value of `1.0` means that the loss function is simply the sum of the forecast and backcast losses. - -# %% -model = TabularForecaster( - datamodule.parameters, backbone="n_beats", backbone_kwargs={"widths": [16, 256], "backcast_loss_ratio": 1.0} -) - -# %% [markdown] -# ## Finding the learning rate -# -# Tabular models can be particularly sensitive to the choice of learning rate. -# Helpfully, PyTorch Lightning provides a built-in learning rate finder that suggests a suitable learning rate -# automatically. -# To use it, we first create our Trainer. -# We apply gradient clipping (a common technique for tabular tasks) with ``gradient_clip_val=0.01`` in order to help -# prevent our model from over-fitting. -# Here's how to find the learning rate: - -# %% -trainer = flash.Trainer( - max_epochs=3, - gpus=int(torch.cuda.is_available()), - gradient_clip_val=0.01, -) - -res = trainer.tuner.lr_find(model, datamodule=datamodule, min_lr=1e-5) -print(f"Suggested learning rate: {res.suggestion()}") -res.plot(show=True, suggest=True).show() - -# %% [markdown] -# Once the suggest learning rate has been found, we can update our model with it: - -# %% -model.learning_rate = res.suggestion() - -# %% [markdown] -# ## Training the model -# Now all we have to do is train the model! - -# %% -trainer.fit(model, datamodule=datamodule) - -# %% [markdown] -# ## Plot the interpretation -# -# An important feature of the N-BEATS model is that it can be configured to produce an interpretable prediction that is -# split into both a low frequency (trend) component and a high frequency (seasonality) component. -# For hourly observations, we might expect the trend component to show us how electricity prices are changing from one -# day to the next (for example, whether prices were generally higher or lower than yesterday). -# In contrast, the seasonality component would be expected to show us the general pattern in prices through the day -# (for example, if there is typically a peak in price around lunch time or a drop at night). -# -# It is often useful to visualize this decomposition and the `TabularForecaster` makes it simple. -# First, we load the best model from our training run and generate some predictions. -# Next, we convert the predictions to the format expected by PyTorch Forecasting using the `convert_predictions` utility -# function. -# Finally, we plot the interpretation using the `pytorch_forecasting_model` attribute. -# Here's the full function: - -# %% - - -def plot_interpretation(model_path: str, predict_df: pd.DataFrame, parameters: Dict[str, Any]): - model = TabularForecaster.load_from_checkpoint(model_path) - datamodule = TabularForecastingData.from_data_frame( - parameters=parameters, - predict_data_frame=predict_df, - batch_size=256, - ) - trainer = flash.Trainer(gpus=int(torch.cuda.is_available())) - predictions = trainer.predict(model, datamodule=datamodule) - predictions, inputs = convert_predictions(predictions) - model.pytorch_forecasting_model.plot_interpretation(inputs, predictions, idx=0) - plt.show() - - -# %% [markdown] -# And now we run the function to plot the trend and seasonality curves: - -# %% -# Todo: Make sure to uncomment the line below if you want to run predictions and visualize the graph -# plot_interpretation(trainer.checkpoint_callback.best_model_path, df_energy_hourly, datamodule.parameters) - -# %% [markdown] -# It worked! The plot shows that the `TabularForecaster` does a reasonable job of modelling the time series and also -# breaks it down into a trend component and a seasonality component (in this case showing daily fluctuations in -# electricity prices). -# -# ## Bonus: Weekly trends -# -# The type of seasonality that the model learns to detect is dictated by the frequency of observations and the length of -# the encoding / prediction window. -# We might imagine that our pipeline could be changed to instead uncover weekly trends if we resample daily -# observations from our data instead of hourly. -# -# We can use our preprocessing function to do this. -# First, we load the data as before then preprocess it (this time setting `frequency = "1D"`). - -# %% -df_energy_daily = pd.read_csv(f"{DATASET_PATH}/energy_dataset.csv", parse_dates=["time"]) -df_energy_daily = preprocess(df_energy_daily, frequency="1D") - -# %% [markdown] -# Now let's create our `TabularForecastingData` as before, this time with a four week encoding window and a one week -# prediction window. - -# %% -max_prediction_length = 1 * 7 -max_encoder_length = 4 * 7 - -training_cutoff = df_energy_daily["time_idx"].max() - max_prediction_length - -datamodule = TabularForecastingData.from_data_frame( - time_idx="time_idx", - target="price actual", - group_ids=["constant"], - max_encoder_length=max_encoder_length, - max_prediction_length=max_prediction_length, - time_varying_unknown_reals=["price actual"], - train_data_frame=df_energy_daily[df_energy_daily["time_idx"] <= training_cutoff], - val_data_frame=df_energy_daily, - batch_size=256, -) - -# %% [markdown] -# Now it's time to create a new model and trainer. -# We run for 24 times the number of epochs this time as we now have around 1/24th of the number of observations. -# This time, instead of using the learning rate finder we just set the learning rate manually: - -# %% -model = TabularForecaster( - datamodule.parameters, - backbone="n_beats", - backbone_kwargs={"widths": [16, 256], "backcast_loss_ratio": 1.0}, - learning_rate=5e-4, -) - -trainer = flash.Trainer( - max_epochs=3 * 24, - check_val_every_n_epoch=24, - gpus=int(torch.cuda.is_available()), - gradient_clip_val=0.01, -) - -# %% [markdown] -# Finally, we train the new model: - -# %% -trainer.fit(model, datamodule=datamodule) - -# %% [markdown] -# Now let's look at what it learned: - -# %% -# Todo: Make sure to uncomment the line below if you want to run predictions and visualize the graph -# plot_interpretation(trainer.checkpoint_callback.best_model_path, df_energy_daily, datamodule.parameters) - -# %% [markdown] -# Success! We can now also see weekly trends / seasonality uncovered by our new model. -# -# ## Closing thoughts and next steps! -# -# This tutorial has shown how Flash and PyTorch Forecasting can be used to train state-of-the-art auto-regressive -# forecasting models (such as N-BEATS). -# We've seen how we can influence the kinds of trends and patterns uncovered by the model by resampling the data and -# changing the hyper-parameters. -# -# There are plenty of ways you could take this tutorial further. -# For example, you could try a more complex model, such as the -# [temporal fusion transformer](https://pytorch-forecasting.readthedocs.io/en/latest/api/pytorch_forecasting.models.temporal_fusion_transformer.TemporalFusionTransformer.html), -# which can handle additional inputs (the kaggle data set we used also includes weather data). -# -# Alternatively, if you want to be a bit more adventurous, you could look at -# [some of the other problems that can solved with Lightning Flash](https://lightning-flash.readthedocs.io/en/stable/?badge=stable). diff --git a/flash_tutorials/image_classification/.meta.yml b/flash_tutorials/image_classification/.meta.yml deleted file mode 100644 index 3629f192e..000000000 --- a/flash_tutorials/image_classification/.meta.yml +++ /dev/null @@ -1,19 +0,0 @@ -title: Image Classification on Hymenoptera Dataset -author: Ethan Harris (ethan@pytorchlightning.ai) -created: 2021-11-23 -updated: 2022-08-26 -license: CC BY-SA -build: 3 -tags: - - Image Classification - - Image -description: | - In this tutorial, we'll go over the basics of lightning Flash by finetuning/predictin with an ImageClassifier on [Hymenoptera Dataset](https://www.kaggle.com/ajayrana/hymenoptera-data) containing ants and bees images. -requirements: - - pytorch-lightning==1.6.* - - lightning-flash[image]>=0.7.0 - - torchmetrics<0.11 # todo: task argument is missing - - numpy<1.24 -accelerator: - - GPU - - CPU diff --git a/flash_tutorials/image_classification/image_classification.py b/flash_tutorials/image_classification/image_classification.py deleted file mode 100644 index f656f09dc..000000000 --- a/flash_tutorials/image_classification/image_classification.py +++ /dev/null @@ -1,115 +0,0 @@ -# %% [markdown] -# In this tutorial, we'll go over the basics of lightning Flash by finetuning/prediction with an ImageClassifier on [Hymenoptera Dataset](https://www.kaggle.com/ajayrana/hymenoptera-data) containing ants and bees images. -# -# # Finetuning -# -# Finetuning consists of four steps: -# -# - 1. Train a source neural network model on a source dataset. For computer vision, it is traditionally the [ImageNet dataset](http://www.image-net.org). As training is costly, library such as [Torchvision](https://pytorch.org/vision/stable/index.html) library supports popular pre-trainer model architectures . In this notebook, we will be using their [resnet-18](https://pytorch.org/hub/pytorch_vision_resnet/). -# -# - 2. Create a new neural network called the target model. Its architecture replicates the source model and parameters, expect the latest layer which is removed. This model without its latest layer is traditionally called a backbone -# -# - 3. Add new layers after the backbone where the latest output size is the number of target dataset categories. Those new layers, traditionally called head will be randomly initialized while backbone will conserve its pre-trained weights from ImageNet. -# -# - 4. Train the target model on a target dataset, such as Hymenoptera Dataset with ants and bees. However, freezing some layers at training start such as the backbone tends to be more stable. In Flash, it can easily be done with `trainer.finetune(..., strategy="freeze")`. It is also common to `freeze/unfreeze` the backbone. In `Flash`, it can be done with `trainer.finetune(..., strategy="freeze_unfreeze")`. If one wants more control on the unfreeze flow, Flash supports `trainer.finetune(..., strategy=MyFinetuningStrategy())` where `MyFinetuningStrategy` is subclassing `pytorch_lightning.callbacks.BaseFinetuning`. - -# %% - -import flash -from flash.core.data.utils import download_data -from flash.image import ImageClassificationData, ImageClassifier - -# %% [markdown] -# ## Download data -# The data are downloaded from a URL, and save in a 'data' directory. - -# %% -download_data("https://pl-flash-data.s3.amazonaws.com/hymenoptera_data.zip", "data/") - - -# %% [markdown] -# ## Load the data -# -# Flash Tasks have built-in DataModules that you can use to organize your data. Pass in a train, validation and test folders and Flash will take care of the rest. -# Creates a ImageClassificationData object from folders of images arranged in this way: -# -# train/dog/xxx.png -# train/dog/xxy.png -# train/dog/xxz.png -# train/cat/123.png -# train/cat/nsdf3.png -# train/cat/asd932.png - -# %% -datamodule = ImageClassificationData.from_folders( - train_folder="data/hymenoptera_data/train/", - val_folder="data/hymenoptera_data/val/", - test_folder="data/hymenoptera_data/test/", - batch_size=1, -) - - -# %% [markdown] -# ## Build the model -# Create the ImageClassifier task. By default, the ImageClassifier task uses a [resnet-18](https://pytorch.org/hub/pytorch_vision_resnet/) backbone to train or finetune your model. -# For [Hymenoptera Dataset](https://www.kaggle.com/ajayrana/hymenoptera-data) containing ants and bees images, ``datamodule.num_classes`` will be 2. -# Backbone can easily be changed with `ImageClassifier(backbone="resnet50")` or you could provide your own `ImageClassifier(backbone=my_backbone)` - -# %% -model = ImageClassifier(num_classes=datamodule.num_classes) - - -# %% [markdown] -# ## Create the trainer. Run once on data -# The trainer object can be used for training or fine-tuning tasks on new sets of data. -# You can pass in parameters to control the training routine- limit the number of epochs, run on GPUs or TPUs, etc. -# For more details, read the [Trainer Documentation](https://pytorch-lightning.readthedocs.io/en/stable/api/pytorch_lightning.trainer.trainer.Trainer.html?highlight=Trainer). -# In this demo, we will limit the fine-tuning to run just one epoch using max_epochs=2. - -# %% -trainer = flash.Trainer(max_epochs=1) - - -# %% [markdown] -# ## Finetune the model - -# %% -trainer.finetune(model, datamodule=datamodule, strategy="freeze") - - -# %% [markdown] -# ## Test the model - -# %% -trainer.test(model, datamodule=datamodule) - - -# %% [markdown] -# ## Save it! - -# %% -trainer.save_checkpoint("image_classification_model.pt") - -# %% [markdown] -# ## Predicting -# **Load the model from a checkpoint** - -# %% -model = ImageClassifier.load_from_checkpoint( - "https://flash-weights.s3.amazonaws.com/0.7.0/image_classification_model.pt" -) - -# %% [markdown] -# **Predict what's on a few images! ants or bees?** - -# %% -datamodule = ImageClassificationData.from_files( - predict_files=[ - "data/hymenoptera_data/val/bees/65038344_52a45d090d.jpg", - "data/hymenoptera_data/val/bees/590318879_68cf112861.jpg", - "data/hymenoptera_data/val/ants/540543309_ddbb193ee5.jpg", - ], - batch_size=1, -) -predictions = trainer.predict(model, datamodule=datamodule) -print(predictions) diff --git a/flash_tutorials/tabular_classification/.meta.yml b/flash_tutorials/tabular_classification/.meta.yml deleted file mode 100644 index 66ad2b265..000000000 --- a/flash_tutorials/tabular_classification/.meta.yml +++ /dev/null @@ -1,18 +0,0 @@ -title: Tabular Classification on Titanic Dataset -author: Ethan Harris (ethan@pytorchlightning.ai) -created: 2021-11-23 -updated: 2022-08-26 -license: CC BY-SA -build: 3 -tags: - - Tabular Classification - - Tabular -description: | - In this notebook, we'll go over the basics of lightning Flash by training a TabularClassifier on [Titanic Dataset](https://www.kaggle.com/c/titanic). -requirements: - - lightning-flash[tabular]>=0.6.0 - - pytorch-lightning==1.3.6 # todo: update to latest - - numpy<1.24 -accelerator: - - GPU - - CPU diff --git a/flash_tutorials/tabular_classification/tabular_classification.py b/flash_tutorials/tabular_classification/tabular_classification.py deleted file mode 100644 index ab19eceb8..000000000 --- a/flash_tutorials/tabular_classification/tabular_classification.py +++ /dev/null @@ -1,98 +0,0 @@ -# %% [markdown] -# In this notebook, we'll go over the basics of lightning Flash by training a TabularClassifier on [Titanic Dataset](https://www.kaggle.com/c/titanic). - -# # Training - -# %% - -import flash -from flash.core.data.utils import download_data -from flash.tabular import TabularClassificationData, TabularClassifier - -# %% [markdown] -# ## Download the data -# The data are downloaded from a URL, and save in a 'data' directory. - -# %% -download_data("https://pl-flash-data.s3.amazonaws.com/titanic.zip", "data/") - - -# %% [markdown] -# ## Load the data -# Flash Tasks have built-in DataModules that you can use to organize your data. Pass in a train, validation and test folders and Flash will take care of the rest. -# -# Creates a TabularData relies on [Pandas DataFrame](https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.DataFrame.html). - -# %% -datamodule = TabularClassificationData.from_csv( - ["Sex", "Age", "SibSp", "Parch", "Ticket", "Cabin", "Embarked"], - ["Fare"], - target_fields="Survived", - train_file="./data/titanic/titanic.csv", - test_file="./data/titanic/test.csv", - val_split=0.25, - batch_size=8, -) - - -# %% [markdown] -# ## Build the model -# -# Note: Categorical columns will be mapped to the embedding space. Embedding space is set of tensors to be trained associated to each categorical column. - -# %% -model = TabularClassifier.from_data(datamodule) - - -# %% [markdown] -# ## Create the trainer. Run 10 times on data - -# %% -trainer = flash.Trainer(max_epochs=10) - - -# %% [markdown] -# ## Train the model - -# %% -trainer.fit(model, datamodule=datamodule) - -# %% [markdown] -# ## Test model - -# %% -trainer.test(model, datamodule=datamodule) - - -# %% [markdown] -# ## Save it! - -# %% -trainer.save_checkpoint("tabular_classification_model.pt") - - -# %% [markdown] -# # Predicting -# ## Load the model from a checkpoint -# -# `TabularClassifier.load_from_checkpoint` supports both url or local_path to a checkpoint. If provided with an url, the checkpoint will first be downloaded and loaded to re-create the model. - -# %% -model = TabularClassifier.load_from_checkpoint( - "https://flash-weights.s3.amazonaws.com/0.7.0/tabular_classification_model.pt" -) - - -# %% [markdown] -# ## Generate predictions from a sheet file! Who would survive? -# -# `TabularClassifier.predict` support both DataFrame and path to `.csv` file. - -# %% -datamodule = TabularClassificationData.from_csv( - predict_file="data/titanic/titanic.csv", - parameters=datamodule.parameters, - batch_size=8, -) -predictions = trainer.predict(model, datamodule=datamodule) -print(predictions) diff --git a/flash_tutorials/text_classification/.meta.yml b/flash_tutorials/text_classification/.meta.yml deleted file mode 100644 index 7fefa6aac..000000000 --- a/flash_tutorials/text_classification/.meta.yml +++ /dev/null @@ -1,19 +0,0 @@ -title: Finetuning a Text Classifier on IMDB Dataset -author: Ethan Harris (ethan@pytorchlightning.ai) -created: 2021-11-23 -updated: 2022-08-26 -license: CC BY-SA -build: 3 -tags: - - Text Classification - - Text -description: | - In this notebook, we'll go over the basics of lightning Flash by finetunig a TextClassifier on IMDB Dataset. -requirements: - - pytorch-lightning==1.6.* - - lightning-flash[text]>=0.7.0 - - torchmetrics<0.11 # todo: update to use task=... - - numpy<1.24 -accelerator: - - GPU - - CPU diff --git a/flash_tutorials/text_classification/text_classification.py b/flash_tutorials/text_classification/text_classification.py deleted file mode 100644 index 7299576bd..000000000 --- a/flash_tutorials/text_classification/text_classification.py +++ /dev/null @@ -1,110 +0,0 @@ -# %% [markdown] -# In this notebook, we'll go over the basics of lightning Flash by finetunig a TextClassifier on [IMDB Dataset](https://paperswithcode.com/dataset/imdb-movie-reviews). -# -# # Finetuning -# -# Finetuning consists of four steps: -# -# - 1. Train a source neural network model on a source dataset. For text classification, it is traditionally a transformer model such as BERT [Bidirectional Encoder Representations from Transformers](https://arxiv.org/abs/1810.04805) trained on wikipedia. -# As those model are costly to train, [Transformers](https://github.com/huggingface/transformers) or [FairSeq](https://github.com/pytorch/fairseq) libraries provides popular pre-trained model architectures for NLP. In this notebook, we will be using [tiny-bert](https://huggingface.co/prajjwal1/bert-tiny). -# -# - 2. Create a new neural network the target model. Its architecture replicates all model designs and their parameters on the source model, expect the latest layer which is removed. This model without its latest layers is traditionally called a backbone -# -# - 3. Add new layers after the backbone where the latest output size is the number of target dataset categories. Those new layers, traditionally called head, will be randomly initialized while backbone will conserve its pre-trained weights from ImageNet. -# -# - 4. Train the target model on a target dataset, such as Hymenoptera Dataset with ants and bees. However, freezing some layers at training start such as the backbone tends to be more stable. In Flash, it can easily be done with `trainer.finetune(..., strategy="freeze")`. It is also common to `freeze/unfreeze` the backbone. In `Flash`, it can be done with `trainer.finetune(..., strategy="freeze_unfreeze")`. If a one wants more control on the unfreeze flow, Flash supports `trainer.finetune(..., strategy=MyFinetuningStrategy())` where `MyFinetuningStrategy` is subclassing `pytorch_lightning.callbacks.BaseFinetuning`. - -# %% - -import flash -from flash.core.data.utils import download_data -from flash.text import TextClassificationData, TextClassifier - -# %% [markdown] -# ## Download the data -# The data are downloaded from a URL, and save in a 'data' directory. - -# %% -download_data("https://pl-flash-data.s3.amazonaws.com/imdb.zip", "data/") - - -# %% [markdown] -# ## Load the data -# -# Flash Tasks have built-in DataModules that you can use to organize your data. Pass in a train, validation and test folders and Flash will take care of the rest. -# Creates a TextClassificationData object from csv file. - -# %% -datamodule = TextClassificationData.from_csv( - "review", - "sentiment", - train_file="data/imdb/train.csv", - val_file="data/imdb/valid.csv", - test_file="data/imdb/test.csv", - batch_size=512, # just increased for the example to run fast -) - - -# %% [markdown] -# ## Build the model -# -# Create the TextClassifier task. By default, the TextClassifier task uses a [tiny-bert](https://huggingface.co/prajjwal1/bert-tiny) backbone to train or finetune your model demo. You could use any models from [transformers - Text Classification](https://huggingface.co/models?filter=text-classification,pytorch) -# -# Backbone can easily be changed with such as `TextClassifier(backbone='bert-tiny-mnli')` - -# %% -model = TextClassifier(num_classes=datamodule.num_classes, backbone="prajjwal1/bert-tiny") - - -# %% [markdown] -# ## Create the trainer. Run once on data - -# %% -trainer = flash.Trainer(max_epochs=1) - - -# %% [markdown] -# ## Fine-tune the model -# -# The backbone won't be freezed and the entire model will be finetuned on the imdb dataset - -# %% -trainer.finetune(model, datamodule=datamodule, strategy="freeze") - - -# %% [markdown] -# ## Test model - -# %% -trainer.test(model, datamodule=datamodule) - - -# %% [markdown] -# ## Save it! - -# %% -trainer.save_checkpoint("text_classification_model.pt") - - -# %% [markdown] -# ## Predicting -# **Load the model from a checkpoint** - -# %% -model = TextClassifier.load_from_checkpoint("text_classification_model.pt") - - -# %% [markdown] -# **Classify a few sentences! How was the movie?** - -# %% -datamodule = TextClassificationData.from_lists( - predict_data=[ - "Turgid dialogue, feeble characterization - Harvey Keitel a judge?.", - "The worst movie in the history of cinema.", - "I come from Bulgaria where it 's almost impossible to have a tornado.", - ], - batch_size=4, -) -predictions = trainer.predict(model, datamodule=datamodule) -print(predictions) From ba4a2acac1497e4508c956c60971817cc2243e70 Mon Sep 17 00:00:00 2001 From: jirka Date: Sat, 20 Jul 2024 01:06:01 +0200 Subject: [PATCH 12/17] drop todo notes =) --- .actions/README.md | 11 ----------- .github/workflows/docs-deploy.yml | 1 + 2 files changed, 1 insertion(+), 11 deletions(-) delete mode 100644 .actions/README.md diff --git a/.actions/README.md b/.actions/README.md deleted file mode 100644 index 7eb322687..000000000 --- a/.actions/README.md +++ /dev/null @@ -1,11 +0,0 @@ -scripts for generating notebooks - -**GHA here** - -- generate notebooks -- flow to ban any added notebook in PR (fail if changes in .notebooks) - -**PL side** - -- git submodule with these examples -- gha cron to update submodule head diff --git a/.github/workflows/docs-deploy.yml b/.github/workflows/docs-deploy.yml index 9ec5dcd79..6c3daf4a8 100644 --- a/.github/workflows/docs-deploy.yml +++ b/.github/workflows/docs-deploy.yml @@ -5,6 +5,7 @@ on: pull_request: branches: [main] paths: + - ".actions/assistant.py" - ".github/workflows/docs-deploy.yml" workflow_dispatch: {} workflow_run: From 02ac16da3704f2427ee824e7f9c39765ed1f9457 Mon Sep 17 00:00:00 2001 From: Jirka Borovec <6035284+Borda@users.noreply.github.com> Date: Sat, 20 Jul 2024 01:07:52 +0200 Subject: [PATCH 13/17] debug json ecoding failure (#289) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .actions/assistant.py | 10 ++++++---- .github/workflows/ci_docs.yml | 25 ++++++++++--------------- .github/workflows/docs-deploy.yml | 3 ++- 3 files changed, 18 insertions(+), 20 deletions(-) diff --git a/.actions/assistant.py b/.actions/assistant.py index 84f7423e6..6d0725c7c 100644 --- a/.actions/assistant.py +++ b/.actions/assistant.py @@ -662,11 +662,13 @@ def copy_notebooks( strict: raise exception if copy fails """ - all_ipynb = [] - for pattern in patterns: - all_ipynb += glob.glob(os.path.join(path_root, DIR_NOTEBOOKS, pattern, "*.ipynb")) - print(f"Copy following notebooks to docs folder: {all_ipynb}") os.makedirs(os.path.join(docs_root, path_docs_ipynb), exist_ok=True) + all_ipynb = [ + os.path.realpath(ipynb) + for pattern in patterns + for ipynb in glob.glob(os.path.join(path_root, DIR_NOTEBOOKS, pattern, "*.ipynb")) + ] + print(f"Copy following notebooks to docs folder: {all_ipynb}") if ignore and not isinstance(ignore, (list, set, tuple)): ignore = [ignore] elif not ignore: diff --git a/.github/workflows/ci_docs.yml b/.github/workflows/ci_docs.yml index 3ec1d24dd..1378c20c6 100644 --- a/.github/workflows/ci_docs.yml +++ b/.github/workflows/ci_docs.yml @@ -1,10 +1,10 @@ -name: validate Docs +name: Docs validation on: # Trigger the workflow on push or pull request # push: # branches: [main] pull_request: {} - #workflow_dispatch: {} + workflow_dispatch: {} concurrency: group: ${{ github.workflow }}-${{ github.head_ref }} @@ -20,10 +20,11 @@ jobs: strategy: fail-fast: false matrix: - check: ["html", "linkcheck"] + target: ["html", "linkcheck"] env: PUB_BRANCH: publication PATH_DATASETS: ${{ github.workspace }}/.datasets + TORCH_URL: "https://download.pytorch.org/whl/cpu/torch_stable.html" timeout-minutes: 20 steps: - name: Checkout 🛎️ @@ -49,8 +50,8 @@ jobs: - name: Install dependencies run: | - pip --version - pip install -q -r requirements.txt -r _requirements/docs.txt + set -ex + pip install -q -r requirements.txt -r _requirements/docs.txt -f ${TORCH_URL} pip list - name: Process folders @@ -97,23 +98,17 @@ jobs: tree changed-notebooks - uses: actions/upload-artifact@v3 - if: ${{ matrix.check == 'html' && env.NB_DIRS != 0 }} + if: ${{ matrix.target == 'html' && env.NB_DIRS != 0 }} with: name: notebooks-${{ github.sha }} path: changed-notebooks/ - - name: Link check + - name: Make ${{ matrix.target }} working-directory: ./_docs - if: ${{ matrix.check == 'linkcheck' }} - run: make linkcheck --jobs $(nproc) --debug SPHINXOPTS="--keep-going" - - - name: Make Documentation - working-directory: ./_docs - if: ${{ matrix.check == 'html' }} - run: make html --jobs $(nproc) --debug SPHINXOPTS="-W --keep-going" + run: make ${{ matrix.target }} --jobs $(nproc) --debug SPHINXOPTS="-W --keep-going" - name: Upload built docs - if: ${{ matrix.check == 'html' }} + if: ${{ matrix.target == 'html' }} uses: actions/upload-artifact@v3 with: name: docs-html-${{ github.sha }} diff --git a/.github/workflows/docs-deploy.yml b/.github/workflows/docs-deploy.yml index 6c3daf4a8..d124acffc 100644 --- a/.github/workflows/docs-deploy.yml +++ b/.github/workflows/docs-deploy.yml @@ -19,6 +19,7 @@ jobs: runs-on: ubuntu-20.04 env: PATH_DATASETS: ${{ github.workspace }}/.datasets + TORCH_URL: "https://download.pytorch.org/whl/cpu/torch_stable.html" steps: - name: Checkout 🛎️ Publication if: ${{ github.event_name != 'pull_request' }} @@ -56,7 +57,7 @@ jobs: sudo apt-get install -y cmake pandoc # install Texlive, see https://linuxconfig.org/how-to-install-latex-on-ubuntu-20-04-focal-fossa-linux sudo apt-get install -y texlive-latex-extra dvipng texlive-pictures - pip install -q -r _requirements/docs.txt + pip install -q -r _requirements/docs.txt -f ${TORCH_URL} pip list shell: bash From e8a7308eb5cf0813435518926adcf30434d5132f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 20 Jul 2024 01:09:31 +0200 Subject: [PATCH 14/17] build(deps): bump docker/build-push-action from 5 to 6 (#327) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/docker-build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index ac5b9fb1e..649574c50 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -34,7 +34,7 @@ jobs: password: ${{ secrets.DOCKER_PASSWORD }} - name: Build (and Push) image - uses: docker/build-push-action@v5 + uses: docker/build-push-action@v6 with: #build-args: | # UBUNTU_VERSION=${{ matrix.ubuntu }} From da0a1e80dcf77dec37a2dfbe1aad4f713549e30b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 20 Jul 2024 09:34:00 +0200 Subject: [PATCH 15/17] build(deps): bump actions/cache from 3 to 4 (#311) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/ci_docs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci_docs.yml b/.github/workflows/ci_docs.yml index 1378c20c6..34219ac61 100644 --- a/.github/workflows/ci_docs.yml +++ b/.github/workflows/ci_docs.yml @@ -36,7 +36,7 @@ jobs: python-version: 3.8 - name: Cache pip - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: ~/.cache/pip key: pip-${{ hashFiles('requirements.txt') }}-${{ hashFiles('_requirements/docs.txt') }} From 30b6397409e6524becac90d650a195f5fb89f99c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 20 Jul 2024 09:36:06 +0200 Subject: [PATCH 16/17] build(deps): bump codecov/codecov-action from 3 to 4 (#312) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/ci_internal.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci_internal.yml b/.github/workflows/ci_internal.yml index 752e1f51f..b24deb781 100644 --- a/.github/workflows/ci_internal.yml +++ b/.github/workflows/ci_internal.yml @@ -59,7 +59,7 @@ jobs: coverage xml - name: Upload coverage to Codecov - uses: codecov/codecov-action@v3 + uses: codecov/codecov-action@v4 if: always() # see: https://github.com/actions/toolkit/issues/399 continue-on-error: true From cbbea35feeee2ac8133cef0b00f239e3456fdb43 Mon Sep 17 00:00:00 2001 From: Jirka Borovec <6035284+Borda@users.noreply.github.com> Date: Sat, 20 Jul 2024 10:22:39 +0200 Subject: [PATCH 17/17] docs: switch links from PDF to abstract (#305) --- .github/workflows/ci_docs.yml | 10 ++-------- _docs/source/conf.py | 5 ++++- .../Initialization_and_Optimization.py | 6 +++--- .../Inception_ResNet_DenseNet.py | 6 +++--- .../Transformers_MHAttention.py | 2 +- .../06-graph-neural-networks/GNN_overview.py | 2 +- .../09-normalizing-flows/NF_image_modeling.py | 2 +- .../Autoregressive_Image_Modeling.py | 18 +++++++++--------- .../Vision_Transformer.py | 2 +- .../12-meta-learning/Meta_Learning.py | 6 +++--- .../finetuning-scheduler.py | 12 ++++++------ 11 files changed, 34 insertions(+), 37 deletions(-) diff --git a/.github/workflows/ci_docs.yml b/.github/workflows/ci_docs.yml index 34219ac61..d93052c04 100644 --- a/.github/workflows/ci_docs.yml +++ b/.github/workflows/ci_docs.yml @@ -34,13 +34,7 @@ jobs: - uses: actions/setup-python@v5 with: python-version: 3.8 - - - name: Cache pip - uses: actions/cache@v4 - with: - path: ~/.cache/pip - key: pip-${{ hashFiles('requirements.txt') }}-${{ hashFiles('_requirements/docs.txt') }} - restore-keys: pip- + cache: pip - name: Install Texlive & tree run: | @@ -60,7 +54,7 @@ jobs: head=$(git rev-parse origin/"${{ github.base_ref }}") git diff --name-only $head --output=master-diff.txt python .actions/assistant.py group-folders master-diff.txt - printf "Changed folders:\n" + printf "Changed folders:\n----------------\n" cat changed-folders.txt - name: Count changed notebooks diff --git a/_docs/source/conf.py b/_docs/source/conf.py index ce0d984c6..45a0a3161 100644 --- a/_docs/source/conf.py +++ b/_docs/source/conf.py @@ -240,4 +240,7 @@ linkcheck_exclude_documents = [] # ignore the following relative links (false positive errors during linkcheck) -linkcheck_ignore = [] +linkcheck_ignore = [ + # Implicit generation and generalization methods for energy-based models + "https://openai.com/index/energy-based-models/", +] diff --git a/course_UvA-DL/03-initialization-and-optimization/Initialization_and_Optimization.py b/course_UvA-DL/03-initialization-and-optimization/Initialization_and_Optimization.py index fef7cae2c..4625ca66e 100644 --- a/course_UvA-DL/03-initialization-and-optimization/Initialization_and_Optimization.py +++ b/course_UvA-DL/03-initialization-and-optimization/Initialization_and_Optimization.py @@ -524,7 +524,7 @@ def xavier_init(model): # # Thus, we see that we have an additional factor of 1/2 in the equation, so that our desired weight variance becomes $2/d_x$. # This gives us the Kaiming initialization (see [He, K. et al. -# (2015)](https://arxiv.org/pdf/1502.01852.pdf)). +# (2015)](https://arxiv.org/abs/1502.01852)). # Note that the Kaiming initialization does not use the harmonic mean between input and output size. # In their paper (Section 2.2, Backward Propagation, last paragraph), they argue that using $d_x$ or $d_y$ both lead to stable gradients throughout the network, and only depend on the overall input and output size of the network. # Hence, we can use here only the input $d_x$: @@ -1098,7 +1098,7 @@ def comb_func(w1, w2): # The short answer: no. # There are many papers saying that in certain situations, SGD (with momentum) generalizes better where Adam often tends to overfit [5,6]. # This is related to the idea of finding wider optima. -# For instance, see the illustration of different optima below (credit: [Keskar et al., 2017](https://arxiv.org/pdf/1609.04836.pdf)): +# For instance, see the illustration of different optima below (credit: [Keskar et al., 2017](https://arxiv.org/abs/1609.04836)): # #
# @@ -1128,7 +1128,7 @@ def comb_func(w1, w2): # "Understanding the difficulty of training deep feedforward neural networks." # Proceedings of the thirteenth international conference on artificial intelligence and statistics. # 2010. -# [link](http://proceedings.mlr.press/v9/glorot10a/glorot10a.pdf) +# [link](https://proceedings.mlr.press/v9/glorot10a) # # [2] He, Kaiming, et al. # "Delving deep into rectifiers: Surpassing human-level performance on imagenet classification." diff --git a/course_UvA-DL/04-inception-resnet-densenet/Inception_ResNet_DenseNet.py b/course_UvA-DL/04-inception-resnet-densenet/Inception_ResNet_DenseNet.py index 6e2a2b2b5..704d13515 100644 --- a/course_UvA-DL/04-inception-resnet-densenet/Inception_ResNet_DenseNet.py +++ b/course_UvA-DL/04-inception-resnet-densenet/Inception_ResNet_DenseNet.py @@ -244,7 +244,7 @@ def configure_optimizers(self): # We will support Adam or SGD as optimizers. if self.hparams.optimizer_name == "Adam": # AdamW is Adam with a correct implementation of weight decay (see here - # for details: https://arxiv.org/pdf/1711.05101.pdf) + # for details: https://arxiv.org/abs/1711.05101) optimizer = optim.AdamW(self.parameters(), **self.hparams.optimizer_hparams) elif self.hparams.optimizer_name == "SGD": optimizer = optim.SGD(self.parameters(), **self.hparams.optimizer_hparams) @@ -875,8 +875,8 @@ def forward(self, x): # One difference to the GoogleNet training is that we explicitly use SGD with Momentum as optimizer instead of Adam. # Adam often leads to a slightly worse accuracy on plain, shallow ResNets. # It is not 100% clear why Adam performs worse in this context, but one possible explanation is related to ResNet's loss surface. -# ResNet has been shown to produce smoother loss surfaces than networks without skip connection (see [Li et al., 2018](https://arxiv.org/pdf/1712.09913.pdf) for details). -# A possible visualization of the loss surface with/out skip connections is below (figure credit - [Li et al. ](https://arxiv.org/pdf/1712.09913.pdf)): +# ResNet has been shown to produce smoother loss surfaces than networks without skip connection (see [Li et al., 2018](https://arxiv.org/abs/1712.09913) for details). +# A possible visualization of the loss surface with/out skip connections is below (figure credit - [Li et al. ](https://arxiv.org/abs/1712.09913)): # #
# diff --git a/course_UvA-DL/05-transformers-and-MH-attention/Transformers_MHAttention.py b/course_UvA-DL/05-transformers-and-MH-attention/Transformers_MHAttention.py index f3ba2d528..6c84c8c9b 100644 --- a/course_UvA-DL/05-transformers-and-MH-attention/Transformers_MHAttention.py +++ b/course_UvA-DL/05-transformers-and-MH-attention/Transformers_MHAttention.py @@ -660,7 +660,7 @@ def forward(self, x): # In fact, training a deep Transformer without learning rate warm-up can make the model diverge # and achieve a much worse performance on training and testing. # Take for instance the following plot by [Liu et al. -# (2019)](https://arxiv.org/pdf/1908.03265.pdf) comparing Adam-vanilla (i.e. Adam without warm-up) +# (2019)](https://arxiv.org/abs/1908.03265) comparing Adam-vanilla (i.e. Adam without warm-up) # vs Adam with a warm-up: # #
diff --git a/course_UvA-DL/06-graph-neural-networks/GNN_overview.py b/course_UvA-DL/06-graph-neural-networks/GNN_overview.py index 5b53866f0..ab818f1df 100644 --- a/course_UvA-DL/06-graph-neural-networks/GNN_overview.py +++ b/course_UvA-DL/06-graph-neural-networks/GNN_overview.py @@ -750,7 +750,7 @@ def print_results(result_dict): # Tutorials and papers for this topic include: # # * [PyTorch Geometric example](https://github.com/rusty1s/pytorch_geometric/blob/master/examples/link_pred.py) -# * [Graph Neural Networks: A Review of Methods and Applications](https://arxiv.org/pdf/1812.08434.pdf), Zhou et al. +# * [Graph Neural Networks: A Review of Methods and Applications](https://arxiv.org/abs/1812.08434), Zhou et al. # 2019 # * [Link Prediction Based on Graph Neural Networks](https://papers.nips.cc/paper/2018/file/53f0d7c537d99b3824f0f99d62ea2428-Paper.pdf), Zhang and Chen, 2018. diff --git a/course_UvA-DL/09-normalizing-flows/NF_image_modeling.py b/course_UvA-DL/09-normalizing-flows/NF_image_modeling.py index 9b0c02e67..fe4177f4c 100644 --- a/course_UvA-DL/09-normalizing-flows/NF_image_modeling.py +++ b/course_UvA-DL/09-normalizing-flows/NF_image_modeling.py @@ -1396,7 +1396,7 @@ def visualize_dequant_distribution(model: ImageFlow, imgs: Tensor, title: str = # and we have the guarantee that every possible input $x$ has a corresponding latent vector $z$. # However, even beyond continuous inputs and images, flows can be applied and allow us to exploit # the data structure in latent space, as e.g. on graphs for the task of molecule generation [6]. -# Recent advances in [Neural ODEs](https://arxiv.org/pdf/1806.07366.pdf) allow a flow with infinite number of layers, +# Recent advances in [Neural ODEs](https://arxiv.org/abs/1806.07366) allow a flow with infinite number of layers, # called Continuous Normalizing Flows, whose potential is yet to fully explore. # Overall, normalizing flows are an exciting research area which will continue over the next couple of years. diff --git a/course_UvA-DL/10-autoregressive-image-modeling/Autoregressive_Image_Modeling.py b/course_UvA-DL/10-autoregressive-image-modeling/Autoregressive_Image_Modeling.py index 3367a1496..7c329b6f4 100644 --- a/course_UvA-DL/10-autoregressive-image-modeling/Autoregressive_Image_Modeling.py +++ b/course_UvA-DL/10-autoregressive-image-modeling/Autoregressive_Image_Modeling.py @@ -18,10 +18,10 @@ # For instance, in autoregressive models, we cannot interpolate between two images because of the lack of a latent representation. # We will explore and discuss these benefits and drawbacks alongside with our implementation. # -# Our implementation will focus on the [PixelCNN](https://arxiv.org/pdf/1606.05328.pdf) [2] model which has been discussed in detail in the lecture. +# Our implementation will focus on the [PixelCNN](https://arxiv.org/abs/1606.05328) [2] model which has been discussed in detail in the lecture. # Most current SOTA models use PixelCNN as their fundamental architecture, # and various additions have been proposed to improve the performance -# (e.g. [PixelCNN++](https://arxiv.org/pdf/1701.05517.pdf) and [PixelSNAIL](http://proceedings.mlr.press/v80/chen18h/chen18h.pdf)). +# (e.g. [PixelCNN++](https://arxiv.org/abs/1701.05517) and [PixelSNAIL](http://proceedings.mlr.press/v80/chen18h/chen18h.pdf)). # Hence, implementing PixelCNN is a good starting point for our short tutorial. # # First of all, we need to import our standard libraries. Similarly as in @@ -173,7 +173,7 @@ def show_imgs(imgs): # If we now want to apply this to our convolutions, we need to ensure that the prediction of pixel 1 # is not influenced by its own "true" input, and all pixels on its right and in any lower row. # In convolutions, this means that we want to set those entries of the weight matrix to zero that take pixels on the right and below into account. -# As an example for a 5x5 kernel, see a mask below (figure credit - [Aaron van den Oord](https://arxiv.org/pdf/1606.05328.pdf)): +# As an example for a 5x5 kernel, see a mask below (figure credit - [Aaron van den Oord](https://arxiv.org/abs/1606.05328)): # #
# @@ -217,10 +217,10 @@ def forward(self, x): # # To build our own autoregressive image model, we could simply stack a few masked convolutions on top of each other. # This was actually the case for the original PixelCNN model, discussed in the paper -# [Pixel Recurrent Neural Networks](https://arxiv.org/pdf/1601.06759.pdf), but this leads to a considerable issue. +# [Pixel Recurrent Neural Networks](https://arxiv.org/abs/1601.06759), but this leads to a considerable issue. # When sequentially applying a couple of masked convolutions, the receptive field of a pixel # show to have a "blind spot" on the right upper side, as shown in the figure below -# (figure credit - [Aaron van den Oord et al. ](https://arxiv.org/pdf/1606.05328.pdf)): +# (figure credit - [Aaron van den Oord et al. ](https://arxiv.org/abs/1606.05328)): # #
# @@ -447,7 +447,7 @@ def show_center_recep_field(img, out): # For visualizing the receptive field, we assumed a very simplified stack of vertical and horizontal convolutions. # Obviously, there are more sophisticated ways of doing it, and PixelCNN uses gated convolutions for this. # Specifically, the Gated Convolution block in PixelCNN looks as follows -# (figure credit - [Aaron van den Oord et al. ](https://arxiv.org/pdf/1606.05328.pdf)): +# (figure credit - [Aaron van den Oord et al. ](https://arxiv.org/abs/1606.05328)): # #
# @@ -508,7 +508,7 @@ def forward(self, v_stack, h_stack): # The architecture consists of multiple stacked GatedMaskedConv blocks, where we add an additional dilation factor to a few convolutions. # This is used to increase the receptive field of the model and allows to take a larger context into account during generation. # As a reminder, dilation on a convolution works looks as follows -# (figure credit - [Vincent Dumoulin and Francesco Visin](https://arxiv.org/pdf/1603.07285.pdf)): +# (figure credit - [Vincent Dumoulin and Francesco Visin](https://arxiv.org/abs/1603.07285)): # #
# @@ -659,7 +659,7 @@ def test_step(self, batch, batch_idx): # %% [markdown] # The visualization shows that for predicting any pixel, we can take almost half of the image into account. # However, keep in mind that this is the "theoretical" receptive field and not necessarily -# the [effective receptive field](https://arxiv.org/pdf/1701.04128.pdf), which is usually much smaller. +# the [effective receptive field](https://arxiv.org/abs/1701.04128), which is usually much smaller. # For a stronger model, we should therefore try to increase the receptive # field even further. Especially, for the pixel on the bottom right, the # very last pixel, we would be allowed to take into account the whole @@ -873,7 +873,7 @@ def autocomplete_image(img): # Interestingly, the pixel values 64, 128 and 191 also stand out which is likely due to the quantization used during the creation of the dataset. # For RGB images, we would also see two peaks around 0 and 255, # but the values in between would be much more frequent than in MNIST -# (see Figure 1 in the [PixelCNN++](https://arxiv.org/pdf/1701.05517.pdf) for a visualization on CIFAR10). +# (see Figure 1 in the [PixelCNN++](https://arxiv.org/abs/1701.05517) for a visualization on CIFAR10). # # Next, we can visualize the distribution our model predicts (in average): diff --git a/course_UvA-DL/11-vision-transformer/Vision_Transformer.py b/course_UvA-DL/11-vision-transformer/Vision_Transformer.py index fd7a6d45a..6606b0ef6 100644 --- a/course_UvA-DL/11-vision-transformer/Vision_Transformer.py +++ b/course_UvA-DL/11-vision-transformer/Vision_Transformer.py @@ -515,7 +515,7 @@ def train_model(**kwargs): # Dosovitskiy, Alexey, et al. # "An image is worth 16x16 words: Transformers for image recognition at scale." # International Conference on Representation Learning (2021). -# [link](https://arxiv.org/pdf/2010.11929.pdf) +# [link](https://arxiv.org/abs/2010.11929) # # Chen, Xiangning, et al. # "When Vision Transformers Outperform ResNets without Pretraining or Strong Data Augmentations." diff --git a/course_UvA-DL/12-meta-learning/Meta_Learning.py b/course_UvA-DL/12-meta-learning/Meta_Learning.py index 6cb110823..f4a266f84 100644 --- a/course_UvA-DL/12-meta-learning/Meta_Learning.py +++ b/course_UvA-DL/12-meta-learning/Meta_Learning.py @@ -1,6 +1,6 @@ # %% [markdown] #
-# Meta-Learning offers solutions to these situations, and we will discuss three popular algorithms: __Prototypical Networks__ ([Snell et al., 2017](https://arxiv.org/pdf/1703.05175.pdf)), __Model-Agnostic Meta-Learning / MAML__ ([Finn et al., 2017](http://proceedings.mlr.press/v70/finn17a.html)), and __Proto-MAML__ ([Triantafillou et al., 2020](https://openreview.net/pdf?id=rkgAGAVKPr)). +# Meta-Learning offers solutions to these situations, and we will discuss three popular algorithms: __Prototypical Networks__ ([Snell et al., 2017](https://arxiv.org/abs/1703.05175)), __Model-Agnostic Meta-Learning / MAML__ ([Finn et al., 2017](http://proceedings.mlr.press/v70/finn17a.html)), and __Proto-MAML__ ([Triantafillou et al., 2020](https://openreview.net/pdf?id=rkgAGAVKPr)). # We will focus on the task of few-shot classification where the training and test set have distinct sets of classes. # For instance, we would train the model on the binary classifications of cats-birds and flowers-bikes, but during test time, the model would need to learn from 4 examples each the difference between dogs and otters, two classes we have not seen during training (Figure credit - [Lilian Weng](https://lilianweng.github.io/lil-log/2018/11/30/meta-learning.html)). # @@ -418,7 +418,7 @@ def split_batch(imgs, targets): # $$\mathbf{v}_c=\frac{1}{|S_c|}\sum_{(\mathbf{x}_i,y_i)\in S_c}f_{\theta}(\mathbf{x}_i)$$ # # where $S_c$ is the part of the support set $S$ for which $y_i=c$, and $\mathbf{v}_c$ represents the _prototype_ of class $c$. -# The prototype calculation is visualized below for a 2-dimensional feature space and 3 classes (Figure credit - [Snell et al.](https://arxiv.org/pdf/1703.05175.pdf)). +# The prototype calculation is visualized below for a 2-dimensional feature space and 3 classes (Figure credit - [Snell et al.](https://arxiv.org/abs/1703.05175)). # The colored dots represent encoded support elements with color-corresponding class label, and the black dots next to the class label are the averaged prototypes. # #
@@ -1329,7 +1329,7 @@ def test_protomaml(model, dataset, k_shot=4): # [1] Snell, Jake, Kevin Swersky, and Richard S. Zemel. # "Prototypical networks for few-shot learning." # NeurIPS 2017. -# ([link](https://arxiv.org/pdf/1703.05175.pdf)) +# ([link](https://arxiv.org/abs/1703.05175)) # # [2] Chelsea Finn, Pieter Abbeel, Sergey Levine. # "Model-Agnostic Meta-Learning for Fast Adaptation of Deep Networks." diff --git a/lightning_examples/finetuning-scheduler/finetuning-scheduler.py b/lightning_examples/finetuning-scheduler/finetuning-scheduler.py index 9b69ddaae..22128320d 100644 --- a/lightning_examples/finetuning-scheduler/finetuning-scheduler.py +++ b/lightning_examples/finetuning-scheduler/finetuning-scheduler.py @@ -611,18 +611,18 @@ def train() -> None: # %% [markdown] # ## Footnotes # -# - [Howard, J., & Ruder, S. (2018)](https://arxiv.org/pdf/1801.06146.pdf). Fine-tuned Language +# - [Howard, J., & Ruder, S. (2018)](https://arxiv.org/abs/1801.06146). Fine-tuned Language # Models for Text Classification. ArXiv, abs/1801.06146. [↩](#Scheduled-Fine-Tuning-with-the-Fine-Tuning-Scheduler-Extension) -# - [Chronopoulou, A., Baziotis, C., & Potamianos, A. (2019)](https://arxiv.org/pdf/1902.10547.pdf). +# - [Chronopoulou, A., Baziotis, C., & Potamianos, A. (2019)](https://arxiv.org/abs/1902.10547). # An embarrassingly simple approach for transfer learning from pretrained language models. arXiv # preprint arXiv:1902.10547. [↩](#Scheduled-Fine-Tuning-with-the-Fine-Tuning-Scheduler-Extension) -# - [Peters, M. E., Ruder, S., & Smith, N. A. (2019)](https://arxiv.org/pdf/1903.05987.pdf). To tune or not to +# - [Peters, M. E., Ruder, S., & Smith, N. A. (2019)](https://arxiv.org/abs/1903.05987). To tune or not to # tune? adapting pretrained representations to diverse tasks. arXiv preprint arXiv:1903.05987. [↩](#Scheduled-Fine-Tuning-with-the-Fine-Tuning-Scheduler-Extension) -# - [Sivaprasad, P. T., Mai, F., Vogels, T., Jaggi, M., & Fleuret, F. (2020)](https://arxiv.org/pdf/1910.11758.pdf). +# - [Sivaprasad, P. T., Mai, F., Vogels, T., Jaggi, M., & Fleuret, F. (2020)](https://arxiv.org/abs/1910.11758). # Optimizer benchmarking needs to account for hyperparameter tuning. In International Conference on Machine Learning # (pp. 9036-9045). PMLR. [↩](#Optimizer-Configuration) -# - [Mosbach, M., Andriushchenko, M., & Klakow, D. (2020)](https://arxiv.org/pdf/2006.04884.pdf). On the stability of +# - [Mosbach, M., Andriushchenko, M., & Klakow, D. (2020)](https://arxiv.org/abs/2006.04884). On the stability of # fine-tuning bert: Misconceptions, explanations, and strong baselines. arXiv preprint arXiv:2006.04884. [↩](#Optimizer-Configuration) -# - [Loshchilov, I., & Hutter, F. (2016)](https://arxiv.org/pdf/1608.03983.pdf). Sgdr: Stochastic gradient descent with +# - [Loshchilov, I., & Hutter, F. (2016)](https://arxiv.org/abs/1608.03983). Sgdr: Stochastic gradient descent with # warm restarts. arXiv preprint arXiv:1608.03983. [↩](#LR-Scheduler-Configuration) #