From 92b69bc504c84208cea5626a1e24fb7ed0eb9830 Mon Sep 17 00:00:00 2001 From: Junghwan Park Date: Sat, 6 May 2023 11:05:46 +0900 Subject: [PATCH] =?UTF-8?q?PyTorch=20v2.0=20=EB=B0=98=EC=98=81,=20pytorch/?= =?UTF-8?q?tutorials@9efe789b=20(#626)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .build/get_files_to_run.py | 107 +++ .build/get_sphinx_filenames.py | 13 + .build/validate_tutorials_built.py | 89 ++- .github/ISSUE_TEMPLATE/1_TRANSLATE_REQUEST.md | 2 +- LICENSE | 2 +- README.md | 8 +- _static/img/invpendulum.gif | Bin 0 -> 30332 bytes .../img/reinforcement_learning_diagram.drawio | 1 + .../img/reinforcement_learning_diagram.jpg | Bin 23737 -> 34793 bytes _templates/layout.html | 4 +- advanced_source/cpp_frontend.rst | 38 +- advanced_source/ddp_pipeline.py | 49 +- .../dynamic_quantization_tutorial.py | 6 +- advanced_source/generic_join.rst | 10 +- advanced_source/neural_style_tutorial.py | 27 +- advanced_source/numpy_extensions_tutorial.py | 9 +- .../static_quantization_tutorial.rst | 14 +- .../super_resolution_with_onnxruntime.py | 8 +- .../Intro_to_TorchScript_tutorial.py | 6 +- .../audio_data_augmentation_tutorial.py | 434 ----------- .../audio_data_augmentation_tutorial.rst | 10 + beginner_source/audio_datasets_tutorial.py | 87 --- beginner_source/audio_datasets_tutorial.rst | 10 + .../audio_feature_augmentation_tutorial.py | 168 ----- .../audio_feature_augmentation_tutorial.rst | 10 + .../audio_feature_extractions_tutorial.py | 457 ----------- .../audio_feature_extractions_tutorial.rst | 10 + beginner_source/audio_io_tutorial.py | 385 ---------- beginner_source/audio_io_tutorial.rst | 10 + beginner_source/audio_resampling_tutorial.py | 476 ------------ beginner_source/audio_resampling_tutorial.rst | 9 + beginner_source/basics/autogradqs_tutorial.py | 4 +- beginner_source/basics/buildmodel_tutorial.py | 18 +- beginner_source/basics/intro.py | 2 +- .../basics/optimization_tutorial.py | 4 +- beginner_source/basics/quickstart_tutorial.py | 19 +- .../basics/saveloadrun_tutorial.py | 4 +- beginner_source/basics/tensorqs_tutorial.py | 1 + .../bettertransformer_tutorial.rst | 4 +- beginner_source/blitz/autograd_tutorial.py | 4 - .../blitz/neural_networks_tutorial.py | 36 +- beginner_source/chatbot_tutorial.py | 139 ++-- beginner_source/colab.rst | 26 +- beginner_source/data_loading_tutorial.py | 41 +- beginner_source/dcgan_faces_tutorial.py | 115 +-- .../ddp_series_fault_tolerance.rst | 8 +- beginner_source/ddp_series_multigpu.rst | 35 +- beginner_source/ddp_series_theory.rst | 8 +- ...deploy_seq2seq_hybrid_frontend_tutorial.py | 50 +- beginner_source/dist_overview.rst | 4 +- beginner_source/fgsm_tutorial.py | 20 +- beginner_source/flava_finetuning_tutorial.py | 28 +- .../former_torchies/parallelism_tutorial.py | 2 +- .../hyperparameter_tuning_tutorial.py | 84 ++- .../introyt/autogradyt_tutorial.py | 8 +- beginner_source/introyt/captumyt.py | 10 +- beginner_source/introyt/introyt1_tutorial.py | 8 +- beginner_source/introyt/modelsyt_tutorial.py | 92 +-- .../introyt/tensorboardyt_tutorial.py | 64 +- .../introyt/tensors_deeper_tutorial.py | 4 +- beginner_source/introyt/trainingyt.py | 112 +-- beginner_source/nn_tutorial.py | 110 ++- beginner_source/saving_loading_models.py | 6 +- beginner_source/t5_tutorial.py | 456 +++++++++++ beginner_source/template_tutorial.py | 92 +++ .../text_sentiment_ngrams_tutorial.py | 10 +- beginner_source/transfer_learning_tutorial.py | 13 +- beginner_source/transformer_tutorial.py | 85 ++- beginner_source/translation_transformer.py | 22 +- beginner_source/vt_tutorial.py | 24 +- conf.py | 17 +- distributed/home.rst | 22 +- .../maskedtensor_sparsity.ipynb | 6 +- .../maskedtensor_adagrad.py | 6 +- .../tuning_guide.ipynb | 4 +- .../maskedtensor_advanced_semantics.ipynb | 4 +- .../maskedtensor_overview.py | 2 +- .../maskedtensor_overview.ipynb | 2 +- .../modelsyt_tutorial.py | 92 +-- .../fx_profiling_tutorial.py | 2 +- .../tuning_guide.py | 6 +- .../maskedtensor_sparsity.py | 6 +- .../fx_profiling_tutorial.ipynb | 2 +- .../maskedtensor_adagrad.ipynb | 4 +- .../tensorboardyt_tutorial.py | 54 +- .../maskedtensor_advanced_semantics.py | 10 +- .../tensorboardyt_tutorial.ipynb | 2 +- .../modelsyt_tutorial.ipynb | 2 +- docs/advanced/generic_join.html | 158 ++-- docs/beginner/ddp_series_multigpu.html | 146 ++-- docs/beginner/ddp_series_theory.html | 144 ++-- docs/beginner/introyt/modelsyt_tutorial.html | 156 ++-- .../introyt/tensorboardyt_tutorial.html | 154 ++-- docs/distributed/home.html | 172 ++--- docs/intermediate/FSDP_adavnced_tutorial.html | 156 ++-- .../dist_pipeline_parallel_tutorial.html | 156 ++-- docs/intermediate/fx_profiling_tutorial.html | 154 ++-- .../process_group_cpp_extension_tutorial.html | 154 ++-- docs/intermediate/rpc_async_execution.html | 156 ++-- .../rpc_param_server_tutorial.html | 154 ++-- docs/intermediate/torchrec_tutorial.html | 154 ++-- docs/intermediate/torchserve_with_ipex.html | 156 ++-- docs/intermediate/torchserve_with_ipex_2.html | 156 ++-- docs/prototype/fx_graph_mode_ptq_static.html | 150 ++-- docs/prototype/maskedtensor_adagrad.html | 148 ++-- .../maskedtensor_advanced_semantics.html | 148 ++-- docs/prototype/maskedtensor_overview.html | 144 ++-- docs/prototype/maskedtensor_sparsity.html | 150 ++-- docs/recipes/intel_extension_for_pytorch.html | 144 ++-- docs/recipes/quantization.html | 144 ++-- docs/recipes/recipes/tuning_guide.html | 160 ++-- index.rst | 89 ++- .../FSDP_adavnced_tutorial.rst | 48 +- intermediate_source/FSDP_tutorial.rst | 4 +- .../autograd_saved_tensors_hooks_tutorial.py | 31 +- .../ax_multiobjective_nas_tutorial.py | 48 +- .../char_rnn_classification_tutorial.py | 9 +- .../char_rnn_generation_tutorial.py | 9 +- .../custom_function_conv_bn_tutorial.py | 58 +- intermediate_source/ddp_series_minGPT.rst | 8 +- intermediate_source/ddp_series_multinode.rst | 4 +- intermediate_source/ddp_tutorial.rst | 3 +- .../dist_pipeline_parallel_tutorial.rst | 6 +- intermediate_source/dist_tuto.rst | 4 +- intermediate_source/ensembling.py | 175 +++++ .../flask_rest_api_tutorial.py | 12 +- ...rced_alignment_with_torchaudio_tutorial.py | 528 ------------- ...ced_alignment_with_torchaudio_tutorial.rst | 11 + intermediate_source/forward_ad_usage.py | 37 +- intermediate_source/fx_conv_bn_fuser.py | 24 +- intermediate_source/fx_profiling_tutorial.py | 12 +- intermediate_source/jacobians_hessians.py | 349 +++++++++ intermediate_source/mario_rl_tutorial.py | 36 +- intermediate_source/memory_format_tutorial.py | 9 +- intermediate_source/mnist_train_nas.py | 8 +- .../model_parallel_tutorial.py | 7 +- intermediate_source/neural_tangent_kernels.py | 248 ++++++ intermediate_source/nvfuser_intro_tutorial.py | 687 ----------------- .../nvfuser_intro_tutorial.rst | 8 + intermediate_source/parametrizations.py | 10 +- intermediate_source/per_sample_grads.py | 226 ++++++ intermediate_source/pipeline_tutorial.py | 35 +- .../process_group_cpp_extension_tutorial.rst | 6 +- intermediate_source/pruning_tutorial.py | 13 +- intermediate_source/reinforcement_ppo.py | 702 +++++++++++++++++ .../reinforcement_q_learning.py | 262 +++---- intermediate_source/rpc_async_execution.rst | 6 +- .../rpc_param_server_tutorial.rst | 4 +- intermediate_source/rpc_tutorial.rst | 2 +- .../scaled_dot_product_attention_tutorial.py | 343 +++++++++ .../seq2seq_translation_tutorial.py | 8 +- ...classification_with_torchaudio_tutorial.py | 545 -------------- .../speech_recognition_pipeline_tutorial.py | 288 ------- .../speech_recognition_pipeline_tutorial.rst | 10 + .../tensorboard_profiler_tutorial.py | 18 +- .../text_to_speech_with_torchaudio.py | 306 -------- .../text_to_speech_with_torchaudio.rst | 10 + intermediate_source/torch_compile_tutorial.py | 509 +++++++++++++ intermediate_source/torchrec_tutorial.rst | 20 +- intermediate_source/torchserve_with_ipex.rst | 176 ++--- .../torchserve_with_ipex_2.rst | 156 ++-- prototype_source/README.md | 6 +- prototype_source/README.txt | 8 +- prototype_source/backend_config_tutorial.rst | 326 ++++++++ prototype_source/fx_graph_mode_ptq_dynamic.py | 68 +- prototype_source/fx_graph_mode_ptq_static.rst | 655 ++++++++-------- .../fx_graph_mode_quant_guide.rst | 59 +- prototype_source/fx_numeric_suite_tutorial.py | 231 ------ prototype_source/maskedtensor_adagrad.py | 6 +- .../maskedtensor_advanced_semantics.py | 10 +- prototype_source/maskedtensor_overview.py | 2 +- prototype_source/maskedtensor_sparsity.py | 6 +- prototype_source/nestedtensor.py | 383 +++++----- prototype_source/prototype_index.rst | 4 +- prototype_source/skip_param_init.rst | 18 +- prototype_source/vulkan_workflow.rst | 12 +- .../android_native_app_with_custom_op.rst | 18 +- recipes_source/bundled_inputs.rst | 8 +- recipes_source/deployment_with_flask.rst | 4 +- .../intel_extension_for_pytorch.rst | 708 ++++++++++++++---- recipes_source/mobile_perf.rst | 86 ++- recipes_source/quantization.rst | 26 +- recipes_source/recipes/amp_recipe.py | 10 +- .../recipes/changing_default_device.py | 50 ++ .../recipes/reasoning_about_shapes.py | 88 +++ ...saving_and_loading_a_general_checkpoint.py | 2 +- recipes_source/recipes/tuning_guide.py | 28 +- recipes_source/recipes_index.rst | 11 +- recipes_source/script_optimized.rst | 17 +- requirements.txt | 8 +- 190 files changed, 8384 insertions(+), 8589 deletions(-) create mode 100644 .build/get_files_to_run.py create mode 100644 .build/get_sphinx_filenames.py create mode 100644 _static/img/invpendulum.gif create mode 100644 _static/img/reinforcement_learning_diagram.drawio delete mode 100644 beginner_source/audio_data_augmentation_tutorial.py create mode 100644 beginner_source/audio_data_augmentation_tutorial.rst delete mode 100644 beginner_source/audio_datasets_tutorial.py create mode 100644 beginner_source/audio_datasets_tutorial.rst delete mode 100644 beginner_source/audio_feature_augmentation_tutorial.py create mode 100644 beginner_source/audio_feature_augmentation_tutorial.rst delete mode 100644 beginner_source/audio_feature_extractions_tutorial.py create mode 100644 beginner_source/audio_feature_extractions_tutorial.rst delete mode 100644 beginner_source/audio_io_tutorial.py create mode 100644 beginner_source/audio_io_tutorial.rst delete mode 100644 beginner_source/audio_resampling_tutorial.py create mode 100644 beginner_source/audio_resampling_tutorial.rst create mode 100644 beginner_source/t5_tutorial.py create mode 100644 beginner_source/template_tutorial.py create mode 100644 intermediate_source/ensembling.py delete mode 100644 intermediate_source/forced_alignment_with_torchaudio_tutorial.py create mode 100644 intermediate_source/forced_alignment_with_torchaudio_tutorial.rst create mode 100644 intermediate_source/jacobians_hessians.py create mode 100644 intermediate_source/neural_tangent_kernels.py delete mode 100644 intermediate_source/nvfuser_intro_tutorial.py create mode 100644 intermediate_source/nvfuser_intro_tutorial.rst create mode 100644 intermediate_source/per_sample_grads.py create mode 100644 intermediate_source/reinforcement_ppo.py create mode 100644 intermediate_source/scaled_dot_product_attention_tutorial.py delete mode 100644 intermediate_source/speech_command_classification_with_torchaudio_tutorial.py delete mode 100644 intermediate_source/speech_recognition_pipeline_tutorial.py create mode 100644 intermediate_source/speech_recognition_pipeline_tutorial.rst delete mode 100644 intermediate_source/text_to_speech_with_torchaudio.py create mode 100644 intermediate_source/text_to_speech_with_torchaudio.rst create mode 100644 intermediate_source/torch_compile_tutorial.py create mode 100644 prototype_source/backend_config_tutorial.rst delete mode 100644 prototype_source/fx_numeric_suite_tutorial.py create mode 100644 recipes_source/recipes/changing_default_device.py create mode 100644 recipes_source/recipes/reasoning_about_shapes.py diff --git a/.build/get_files_to_run.py b/.build/get_files_to_run.py new file mode 100644 index 000000000..80f958f50 --- /dev/null +++ b/.build/get_files_to_run.py @@ -0,0 +1,107 @@ +from typing import Any, Dict, List, Optional, Tuple +import json +import os +from pathlib import Path +# from remove_runnable_code import remove_runnable_code + + +# Calculate repo base dir +REPO_BASE_DIR = Path(__file__).absolute().parent.parent + + +def get_all_files() -> List[str]: + sources = [x.relative_to(REPO_BASE_DIR) for x in REPO_BASE_DIR.glob("*_source/**/*.py") if 'data' not in x.parts] + return [str(x) for x in sources] + + +def read_metadata() -> Dict[str, Any]: + with (REPO_BASE_DIR / ".jenkins" / "metadata.json").open() as fp: + return json.load(fp) + + +def calculate_shards(all_files: List[str], num_shards: int = 20) -> List[List[str]]: + sharded_files: List[Tuple[float, List[str]]] = [(0.0, []) for _ in range(num_shards)] + metadata = read_metadata() + + def get_duration(file: str) -> int: + # tutorials not listed in the metadata.json file usually take + # <3min to run, so we'll default to 1min if it's not listed + return metadata.get(file, {}).get("duration", 60) + + def get_needs_machine(file: str) -> Optional[str]: + return metadata.get(file, {}).get("needs", None) + + def add_to_shard(i, filename): + shard_time, shard_jobs = sharded_files[i] + shard_jobs.append(filename) + sharded_files[i] = ( + shard_time + get_duration(filename), + shard_jobs, + ) + + all_other_files = all_files.copy() + needs_gpu_nvidia_small_multi = list( + filter(lambda x: get_needs_machine(x) == "gpu.nvidia.small.multi", all_files,) + ) + needs_gpu_nvidia_medium = list( + filter(lambda x: get_needs_machine(x) == "gpu.nvidia.large", all_files,) + ) + for filename in needs_gpu_nvidia_small_multi: + # currently, the only job that uses gpu.nvidia.small.multi is the 0th worker, + # so we'll add all the jobs that need this machine to the 0th worker + add_to_shard(0, filename) + all_other_files.remove(filename) + for filename in needs_gpu_nvidia_medium: + # currently, the only job that uses gpu.nvidia.large is the 1st worker, + # so we'll add all the jobs that need this machine to the 1st worker + add_to_shard(1, filename) + all_other_files.remove(filename) + + sorted_files = sorted(all_other_files, key=get_duration, reverse=True,) + + for filename in sorted_files: + min_shard_index = sorted(range(num_shards), key=lambda i: sharded_files[i][0])[ + 0 + ] + add_to_shard(min_shard_index, filename) + return [x[1] for x in sharded_files] + + +def compute_files_to_keep(files_to_run: List[str]) -> List[str]: + metadata = read_metadata() + files_to_keep = list(files_to_run) + for file in files_to_run: + extra_files = metadata.get(file, {}).get("extra_files", []) + files_to_keep.extend(extra_files) + return files_to_keep + + +def remove_other_files(all_files, files_to_keep) -> None: + + for file in all_files: + if file not in files_to_keep: + remove_runnable_code(file, file) + + +def parse_args() -> Any: + from argparse import ArgumentParser + parser = ArgumentParser("Select files to run") + parser.add_argument("--dry-run", action="store_true") + parser.add_argument("--num-shards", type=int, default=int(os.environ.get("NUM_WORKERS", 20))) + parser.add_argument("--shard-num", type=int, default=int(os.environ.get("WORKER_ID", 0))) + return parser.parse_args() + + +def main() -> None: + args = parse_args() + + all_files = get_all_files() + files_to_run = calculate_shards(all_files, num_shards=args.num_shards)[args.shard_num] + if not args.dry_run: + remove_other_files(all_files, compute_files_to_keep(files_to_run)) + stripped_file_names = [Path(x).stem for x in files_to_run] + print(" ".join(stripped_file_names)) + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/.build/get_sphinx_filenames.py b/.build/get_sphinx_filenames.py new file mode 100644 index 000000000..b84267b48 --- /dev/null +++ b/.build/get_sphinx_filenames.py @@ -0,0 +1,13 @@ +from pathlib import Path +from typing import List + +from get_files_to_run import get_all_files +from validate_tutorials_built import NOT_RUN + + +def get_files_for_sphinx() -> List[str]: + all_py_files = get_all_files() + return [x for x in all_py_files if all(y not in x for y in NOT_RUN)] + + +SPHINX_SHOULD_RUN = "|".join(get_files_for_sphinx()) diff --git a/.build/validate_tutorials_built.py b/.build/validate_tutorials_built.py index 992469ebf..46452cc18 100644 --- a/.build/validate_tutorials_built.py +++ b/.build/validate_tutorials_built.py @@ -9,51 +9,49 @@ # the file name to explain why, like intro.html), or fix the tutorial and remove it from this list). NOT_RUN = [ - "basics/intro", # no code - "translation_transformer", - "profiler", - "saving_loading_models", - "introyt/captumyt", - "introyt/trainingyt", - "examples_nn/polynomial_module", - "examples_nn/dynamic_net", - "examples_nn/polynomial_optim", - "former_torchies/autograd_tutorial_old", - "former_torchies/tensor_tutorial_old", - "examples_autograd/polynomial_autograd", - "examples_autograd/polynomial_custom_function", - "parametrizations", - "mnist_train_nas", # used by ax_multiobjective_nas_tutorial.py - "fx_conv_bn_fuser", - "super_resolution_with_onnxruntime", - "ddp_pipeline", # requires 4 gpus - "fx_graph_mode_ptq_dynamic", - "vmap_recipe", - "torchscript_freezing", - "nestedtensor", - "recipes/saving_and_loading_models_for_inference", - "recipes/saving_multiple_models_in_one_file", - "recipes/loading_data_recipe", - "recipes/tensorboard_with_pytorch", - "recipes/what_is_state_dict", - "recipes/profiler_recipe", - "recipes/save_load_across_devices", - "recipes/warmstarting_model_using_parameters_from_a_different_model", - "recipes/dynamic_quantization", - "recipes/saving_and_loading_a_general_checkpoint", - "recipes/benchmark", - "recipes/tuning_guide", - "recipes/zeroing_out_gradients", - "recipes/defining_a_neural_network", - "recipes/timer_quick_start", - "recipes/amp_recipe", - "recipes/Captum_Recipe", - "hyperparameter_tuning_tutorial", - "flask_rest_api_tutorial", - "text_to_speech_with_torchaudio", + "beginner_source/basics/intro", # no code + "beginner_source/translation_transformer", + "beginner_source/profiler", + "beginner_source/saving_loading_models", + "beginner_source/introyt/captumyt", + "beginner_source/examples_nn/polynomial_module", + "beginner_source/examples_nn/dynamic_net", + "beginner_source/examples_nn/polynomial_optim", + "beginner_source/former_torchies/autograd_tutorial_old", + "beginner_source/former_torchies/tensor_tutorial_old", + "beginner_source/examples_autograd/polynomial_autograd", + "beginner_source/examples_autograd/polynomial_custom_function", + "intermediate_source/parametrizations", + "intermediate_source/mnist_train_nas", # used by ax_multiobjective_nas_tutorial.py + "intermediate_source/fx_conv_bn_fuser", + "advanced_source/super_resolution_with_onnxruntime", + "advanced_source/ddp_pipeline", # requires 4 gpus + "prototype_source/fx_graph_mode_ptq_dynamic", + "prototype_source/vmap_recipe", + "prototype_source/torchscript_freezing", + "prototype_source/nestedtensor", + "recipes_source/recipes/saving_and_loading_models_for_inference", + "recipes_source/recipes/saving_multiple_models_in_one_file", + "recipes_source/recipes/loading_data_recipe", + "recipes_source/recipes/tensorboard_with_pytorch", + "recipes_source/recipes/what_is_state_dict", + "recipes_source/recipes/profiler_recipe", + "recipes_source/recipes/save_load_across_devices", + "recipes_source/recipes/warmstarting_model_using_parameters_from_a_different_model", + "recipes_source/recipes/dynamic_quantization", + "recipes_source/recipes/saving_and_loading_a_general_checkpoint", + "recipes_source/recipes/benchmark", + "recipes_source/recipes/tuning_guide", + "recipes_source/recipes/zeroing_out_gradients", + "recipes_source/recipes/defining_a_neural_network", + "recipes_source/recipes/timer_quick_start", + "recipes_source/recipes/amp_recipe", + "recipes_source/recipes/Captum_Recipe", + "intermediate_source/flask_rest_api_tutorial", + "intermediate_source/text_to_speech_with_torchaudio", + "intermediate_source/tensorboard_profiler_tutorial" # reenable after 2.0 release. ] - def tutorial_source_dirs() -> List[Path]: return [ p.relative_to(REPO_ROOT).with_name(p.stem[:-7]) @@ -68,6 +66,7 @@ def main() -> None: glob_path = f"{tutorial_source_dir}/**/*.html" html_file_paths += docs_dir.glob(glob_path) + should_not_run = [f'{x.replace("_source", "")}.html' for x in NOT_RUN] did_not_run = [] for html_file_path in html_file_paths: with open(html_file_path, "r", encoding="utf-8") as html_file: @@ -78,9 +77,7 @@ def main() -> None: if ( "Total running time of the script: ( 0 minutes 0.000 seconds)" in elem.text - and not any( - html_file_path.match(file) for file in NOT_RUN - ) + and not any(html_file_path.match(file) for file in should_not_run) ): did_not_run.append(html_file_path.as_posix()) diff --git a/.github/ISSUE_TEMPLATE/1_TRANSLATE_REQUEST.md b/.github/ISSUE_TEMPLATE/1_TRANSLATE_REQUEST.md index f17e1c17c..bac5e213c 100644 --- a/.github/ISSUE_TEMPLATE/1_TRANSLATE_REQUEST.md +++ b/.github/ISSUE_TEMPLATE/1_TRANSLATE_REQUEST.md @@ -15,4 +15,4 @@ _(반드시 지키셔야 하는 일정이 아닙니다 - 일정이 너무 늦어 ## 관련 이슈 _현재 번역 요청 / 진행 내역을 보기 위해 각 버전의 메인 이슈를 참조합니다._
_(특별한 일이 없다면 변경하지 않으셔도 됩니다.)_ -* 관련 이슈: #615 (v1.13) +* 관련 이슈: #660 (v2.0) diff --git a/LICENSE b/LICENSE index 1ba2484d4..f0d2e189a 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ BSD 3-Clause License -Copyright (c) 2017, Pytorch contributors +Copyright (c) 2017, PyTorch contributors All rights reserved. Redistribution and use in source and binary forms, with or without diff --git a/README.md b/README.md index dc71ab7e2..482b29eb2 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,10 @@ -# PyTorch 한국어 튜토리얼 +# 파이토치 한국어 튜토리얼 (PyTorch tutorials in Korean) ## 소개 PyTorch에서 제공하는 튜토리얼의 한국어 번역을 위한 저장소입니다.\ 번역의 결과물은 [https://tutorials.pytorch.kr](https://tutorials.pytorch.kr)에서 확인하실 수 있습니다. (번역을 진행하며 **비정기적으로** 반영합니다.)\ -현재 버전의 번역 / 변경 관련 이슈는 [#615 이슈](https://github.com/PyTorchKorea/tutorials-kr/issues/615)를 참고해주세요. +현재 버전의 번역 / 변경 관련 이슈는 [#660 이슈](https://github.com/PyTorchKorea/tutorials-kr/issues/660)를 참고해주세요. ## 기여하기 @@ -22,7 +22,7 @@ PyTorch에서 제공하는 튜토리얼의 한국어 번역을 위한 저장소 ## 원문 -현재 PyTorch v1.13 튜토리얼([pytorch/tutorials@db34a77](https://github.com/pytorch/tutorials/commit/db34a779242f1a71346db4a9e5d6ac962a8d9b77) 기준) 번역이 진행 중입니다. +현재 PyTorch v2.0 튜토리얼([pytorch/tutorials@9efe789b](https://github.com/pytorch/tutorials/commit/9efe789bfc3763ec359b60f12b5e6dda4e6d5db0) 기준) 번역이 진행 중입니다. 최신 버전의 튜토리얼(공식, 영어)은 [PyTorch tutorials 사이트](https://pytorch.org/tutorials) 및 [PyTorch tutorials 저장소](https://github.com/pytorch/tutorials)를 참고해주세요. @@ -46,5 +46,5 @@ v1.0 이후 번역은 별도 저장소로 관리하지 않습니다. [이 저장 빌드 방법은 [기여하기 문서의 `2-5. (내 컴퓨터에서) 결과 확인하기`](https://github.com/PyTorchKorea/tutorials-kr/blob/master/CONTRIBUTING.md#2-5-내-컴퓨터에서-결과-확인하기) 부분을 참고해주세요. --- -This is a project to translate [pytorch/tutorials@db34a77](https://github.com/pytorch/tutorials/commit/db34a779242f1a71346db4a9e5d6ac962a8d9b77) into Korean. +This is a project to translate [pytorch/tutorials@9efe789b](https://github.com/pytorch/tutorials/commit/9efe789bfc3763ec359b60f12b5e6dda4e6d5db0) into Korean. For the latest version, please visit to the [official PyTorch tutorials repo](https://github.com/pytorch/tutorials). diff --git a/_static/img/invpendulum.gif b/_static/img/invpendulum.gif new file mode 100644 index 0000000000000000000000000000000000000000..3102c5b55cfbdd9ed11bf4da7422980287334753 GIT binary patch literal 30332 zcma&NcQo7a|38|TNo*lj5PPe=f)YUxYSpfaO;MZbB7#uVu3EKe)!tN#+N);m7DesS z($eLZe!lnqzTb0x_uPB)M{-WyIVU-<*DLSGeCiwMsi@+#fLg#GfPW7$DGHc`@l4DTm+Tvv;BeT-63#HBeIYLaO;HDSB$E z2Z~)W0-$t&`bHEMmI~JlIQ#>Rtg%KmMtT+oX0}+Yr5V=FT+jZxnXQ?Q2}ak(L0eNS zDuVkVUKAfKPRYZ`E(#(S;06nGLd7{blsI5A^uiKk;!ndDT!!UPoQS zKur~`q+~A0qr<}~IOK+-Na+yGnKQW@C73@l(yHzAHr zU~4BPXBS#`PX;e9XmBw6-W{&+F!*hM4m3tWTUQ=ssE9UFH^8bGS*TsRt{QfS@9sT; zJK=o5tJ)NrdMsD9Rh3N@`lYWgM zSWl016e#WNX^jj8bx{(Ax{}5QLI%1DMrd^-w7iLds-d2QnUSIyR^0-tW`R++G}ExP zP_iXKrR~iPo_;bh1UGobV4tqarvT;>8|Bi9|<>6Qae0AL9RW>i@4N zzDpVtT}ulcZBt!2X&G`L>21CX3JM76Rr23Q{{2k=pi96o7>=nc?1=_J#T|y~ir&UC z3K-^L>WljyvCDg}4AqwmK1S#yzzrHohtowZUN{Ull#M=-aT&=oXe=MkRSDW#8E&k2 z_e?7q!fn`8IaPv57r!ym^n9kmw8Svqu=&MYl})4f>PU0de67>l1a702my3;_GcRt8 zwp1^-`hOb9H)^d}?F|36w>sKd`|(XAISmiCt#17-5h~$0)>i*%Fh#(q0NdWMHJUAd z>%&-kjV)BmFS)3snirttR<# zm`^x|H`^!B=6>z9W_kMhfq{Sl*l=2-WTUYn8x^9xE@>f>j;VV-E;l7U!}@ z%SpEzs_Ij-?dsj3;krTbRr-~Iu`+en9$tJ9_Vep67WD)NvnxgRzw2vz^qZlddT;tg z4ZdY$o12h_JmegN-EaGZ8!$Y>)F;_T_9TZ13e1@(+Odr8af!Wo8#t=hL*6(>XaG_v zscm3clsqG_rKu>cIXKd-aMZiVP9-{ETHScwgx3VBRyI3l&dpL1>H5bFZezLIJn)b( zH8A6vd;h!XAw@&~3p5YAQ;#;iFAnA-q584P7?)Tz< z)AM|2?L^B0DWuy?0w3hvs+=|Hx?>zH3K!yEqe6C=OfTXh-p$rbbi{3`;3;dqH%P?q zG_(qG54IVN`1N#snuRMu^p_5gU$K3;o8A-lYk=c%@Xl{z`-q;PR^A^|zt6;^ZZzdo zK6DuPu~spgT-X-$Zs1~fIk>sv+OL(6-*V%(71&ibc+=+|X50VqXHmF(-shc)u=g+WrR*#-GDdP{kIzFdtz*x;AyEz@r>N=+Xl1~n9Eh#Gr4ZKt>j z^EC_?8|*pSFRY4c^ftX*CCf}>+!E)Ldt83s+o?WdOOngn$htU?xno60Ct`QlzA4J7 zhou~~bV6Ow^))*RQ!dBUi{&d|&Y8|D*M4D(_3{43e;7_6({>w+E&7s6?G-0Da53^& zl{TN%?6Zo*c3o}{lz%O|BB^)nvg@uD`q}he#Z?iqNqL7v_y!|UgE}Y0S7W^JvumXm z*+r8K+_D%{|2bIP`i-0oLkUIdL{Ry+dZ1%8!WzY(vkcT$Z7wO*uvjcx;}eN-Pk+2C zLoFF99-p7E4vGE6z+QL)v1Vswu0v3(Q8bylnN%?862+Oy4Jo9~D?Ky-WY?c_#)x(E zb2EhqsM|47@=Hd6v7UIl+_f0q-$VB`K)rVJBLpGAD2O5JZji7oeZXeVOJjC|5S5XQ z{O>gw3dPM8krLO#jAtbH_A(O@l(sv~&nMfTYb!`Fg=;BRNB2_*O9z0b@0UrgPxO#XsT(MR?*j;&-@8BY{5l`hqHjov z!@BZ)v4NO-;)T%mIE$CPu$ID@46h={^@X=Q*9#glO2#19B8#DuzGP_+U&hMiUW|ET zqI2fQwv`&&FUNF~gC2)Te%7c2GR!bygMw3O9FLvZI*%I*siiz`p5rHp(oIDS)t*P zzTG>usWzS_>ENQzfyayQBI#S|qVxBixqdA)(o@OE`8rPFH=vbu{TjyskG(%e&%SR7 z)v&=qOg&Le1e@mS8}|i?|v9v1t3fCG%Kytc!Nr2p!fhrBv4d8C&D{r9SXJ@ObFY#?9m8mphlu z($i--$$FC7!z|^svlS*crtEa*S?p-$7yh;#PXxHr5oDg=(enZ1>yzoSmu_cj zqk#(iV2}K&ZRK|t!<8x|Ns$rNKu+k*x6&dx9j6zQ4>c+Npnqj`%$~0HyXtU)?jJlV z{e_@=({es|?eEvz%ilZK-u#?@_xF3}<)7onZ+?HicKLJp^6&Y(H-CS;yZrm>@)E!v zNv1;j%1Em4NKkepq%D$mA(H+)l94@%StW|aCJGiF#hx9-*%k#~h~hbqLa^icRq%o~ zc;RroXf|HF4KKNXmp;eKut&?OL@U@tD}_g^WJjyDMXN7FYo14Iu@g`#1hfr7FPwnM zCK$F6unPo}bAl;*jJZmTrA>@gc#KVUj9pvIb#}&oRF3k0Q#l}j#O^>803Zd$e_x0N zXaQ~m!ble`0YDHOlLA-8Q!?=XH^&2YJGDQe8ZOe&|1XYbMG#@+Q4eI(C2_oIzV_h5 zA;qB5oyE4$ExWyy=UXhkgve*f~paF6_fVu9!yO6a82bxma^zR!)>y%3F1?{sL-VG`rbMFp6pS!;hSuZS0M35m0==X-xwZCMXcoYJ>&5R3T zh5Mftv&fCamLTEFWn^T5pKb=3d1A9o^WeXE77JYp0f|dJ#N1w zTgCf>skn@l?tJiHPW#0a7g#9qQOceekkPq{pQq;@ZRQy0I#(NjsRqrm^A`f%$JuuC z+z2+~)|xQ-XiWPs!%Qt1J(icSCh2g3X1gkol5%RWaWa_8fp*J&L-~R~S`?Y(p1rl& zfOsiubeg8%s1+l`e>)F2G^{G0Y4QyE7k>IkH~L=FvlB(J9oT)wRRtX?KJK&3fV`8< zIYvIi-64EK6gBb?ufbSc(9g%ne_0P4Kzdc(NVUG-tz^3Blz3^;x@7#PNLf4Zrc0}l zKBYMa$|pQvye{rzlBBEkz@e_xr3%BO(j)q)% z$?olWkIFJuHKeQkMEy?2+EtD1yuNF_S-JD;p|xn*?(vOG3u>P_sLRsVu{Wo$Lf-TD z_QRW^>8{BuG>%Ymk6l4;rM1#1OMm;^`6m>PTX57wIdxp#go~qWH4W`XGW=3x*T-z=M?p>tri~FvZ*BYIEyg7A>4E-3+!@Xj;VQvg;cK+< zO!ycXAmEXgmapsLdat03FGT6rC5NCX5YDgYAvzaJfDPD_0?9_?I*D0C(o2uk9>SVoXW5BO4F)&U5PVZ=bGmIy)!9HdqyUQ)3+ z>yZKqV&vJ2062k-Oq&?709@g3TOx{aqlC4Yf-gIgf**^5(??h2Ig-)v3l@P1m7>Bj z0r5tzc*bHlo$P>2EY{VzT&s(Kp%IU<-t@{blOw8?15BTi0ie0WJuC+kp^*I(Oxp=9 zQn(af=|@&cu1Lnp@?u&5N5NAqio$wy#G3?@0enMli9%9|N|Me@F>3_-A0yT|cKFgY z%wSDI5U&n9t~%SlN(95lcuggXrHJSws=2-}BKI9y=r18GIYl9e<>&5?pXJd*E-5hf zfC+7AXXJXk0tVfwKFSZz{yxHLIMm>n*q5-C_;ezBU>ll?-8> zrryrE{G~>W)M?S6cLfit2LS|Dgt6Lju+0XLx+~Bu-i}3(&zDU3s{Q*+_fmMBKVAcDm?{i#Dx?y^ z^EjL3C9RF((y~!3ZHr8P!npjT@OUj4rT$)88=m@0u?{-Ot ztTz>@(?!PGRrA47kT@p|M%#U_`*h>;s8coxyN}g29AcPfpA246@S8*ef&Y!o4@uY@ z`aiIltF>JTeXc)zc1Fy;YRd?y#n@PUn* zliJD?cgrv}9TycaRNt>qb$@~D4hN*KAYL@9qS!+;<{xr{Sc>AK!c|IK!1VzF{6C(u z$3MRMH)aYxsU5xui$px#wC9k$v*f?N&bO6e??u5NZ@hP2nk5%#;D(%Vh$_jp_49JK zxT+Z4pE=_@6^H!Ob-|{3j47eaZ}A1{S%G(lGoQ?P@;T%?zG_S<@;iCDQ_M8b6U#QQ z4%sc*cOs{<)8Px`=+ z;H&m-Y*&^`q+_>H=U%x1VgkUZ5kIA`Jp1{XFgfG5R6c!QQ?=>vfTdw8->ByI+^IoH zo&lS*-CXv+r{ot|n?LRPI5!-IzW1j=nt1Ca~bEFy^n@(L~v__2Kb=l9Pb#YG6y90%{k zT9JZ%j0gRTQTyp_SKhrz?Z}@l2+Ek|7#i`E(0gB@g+wN*l=~^6TiLl?y>&4^`mQ|? z&r62)9v46%NBm#3a%t^{o}E{}u+m+^t6oN^XdP8nXra?7n9e-Vpw(WCX(tZ)WhPeY zbSU-_&5iBK1*QJoTJ9(nX96*_m2E&!BanUM&5LXmH?c=yvbZOl&t~#Z#<(!XWm}sDs@L z%VSS!^(+I7Sa}J&jwmcC2&B>dE#}*CN``P;Pp}4bv(|yA)k^0uwr+ylJDzj>k;T4zQ?gu(Af&Kf2%_s%ulN?rOR-)^roF>V z)FN2tMGGXJW|>dRg8zfUbp(ulP8`qVw$;H!6o;fXF)MkN+1Lifa_TQ0X}&MgkkZRs z;1qAL>FI4A@JRBq029m z(O3p05&Oi<_3#P<#*@tIY%l&=I0{j4j26>kN-&KqW+|yW5CJ&Zj_$ESWfW1YdS5Z> z<=zZ+*WYp<)e%sazcU&t>tdup#U#6;r1iz5lvHfzvSl_IOTLXbU!^I9Vs|Kmx9KF0 zIvK6gP&5N(sL%H*4q~@M4)~}@)EqE!{93cY@@%Ki>WX%Z=sAl0HL>>I_*)uBh0@ow z&ugX6>nIMLi=ivV{TdKCNBnFF&kxM8#%U_VG91-a>05hG@^ zS)8jhGP*zawpL9GF4KXb*LIVC>@jumw19OzV{Vc}y|_&gI18|+b!^8}XjjZ=G!(6- zAXFPT+xg~N|H(xDKZG=a1l9rn6RZ)Dfm=gj4xaxz zERp3$#q3N!bC$DOrDVCd%INls;(Ej2^LJA$%^A*57 zBdhOBOL@1x?V%U8eH;8fbO;FyY%|z(5$&Bb1uAS7Tn9G@yFtpgHb(hJhr!S@_DGk$ zC7+(?(0sSCN^X92Oq@}keV;Rw=SI83-ZGW-ld2%0p`@jzKtsnqEWwRPd`4=|Gbye$|WAs^nhK3;583xZ z%+wYcvsdHziqX#TM6pd14yKV%$4Bs<>}=Gx45rwq&$y|?M_2YLVUO?s7RAAB&ew}m zpIn>{!hA62u_W2rXp(B`w`Cz{`N5h67PPtG<|t!qDx6a|EhXjeXr05r;Lm4PhW7N|}8_M&6Bd_`x*anJG0cRi6Wd# zED!mmm&VL|%fX_SfTMj<`9<?-p$tvEuCxtS*$8`50t#hL=;<#{;i8xc z`Fwc4klop-;LgCYLZ|b`o_Ng;t<6#aoU>r?Z5MS82#6rAUk9CG!Ur59!K%<%Tvw*-+Haw33ba>;v_G^|8e2%l4%+V=exW z50dCdnH&ikvSDGJ%EMA_4k>eitv-63KaXF>zW)(`btx0tu|$ z6)IAx#ZQvhlOYt6cBF9P6&Y+DQ!b>M$rAWq@>(rki@<+ch=00-Vkqr4iDK^oTX+&w zq!lhF#Qn9beJ141t(ulXfnw}s&IXACWK!d?i8$JN zj)cWLL_#4%u{_49m&xuVe?MR!o`mRyN(O+jJ{wgx&%_1rYq>=fIHtU{K4GroUW zEM5~cP%@kd9dm|Y5opu-rFauiX>HL^AAfQO5k)Mnz5hqT`HD7Cgb~Lq;O!pXG@BsG zSX_1QtWQL}$96oen577$E2gOpnSi)9<{nB2vv*QxRO%q?rkU%$n^KHkhBnt0XvIkR zXw%pMv{<{Fsr|#2?Ewc_t>wpPkq=7*O?W0-Z}zsiAjzXyO4Soge9H#moa3i2uM^ zwC>K?$%ku0^UT<_Xz9qJ8^5zW*;vJTS;%M;u9fz}$WE$t5^k;h$RT4H?I&n|LB#Bq z^U=*oO@;i>`ozgb@)=}M(C@k5)Mr{P!Lbsa|kH$_X#-)>U&K0z4INo+;E_#>)@OE94FqfdYe5LKht3p;1#a%x zAqi5X76drbLCxET2o}G@B?y?u`lWPI_tu@$q`B7p#;qjNjbmTgIL!8X>kZTu-RXj0U_w-y-Xh2O6Tc%-6}W6` zUr#^mQ-*aR;i7Dl9$`PkG$_>vNLhzj2ZPq{BO1!MutM~$&X0JO*j&616D>ZOa`>rC zm?7oDW*?)f+bI2PSfBE!Vw_dD;#yUoFQIS*CTjkM`udpbG`6I>P;k?rTUuZ+6DHHS zTq*9U)=0gspifIX2e)47}_Q|cBE&%m!Gb98?>j<}z399N9 z=d(-}q%eANz?y)?p6J7V4e6CPIO5_4dBa!&*##;uk+yt#$n^IC-xXJf%TXt0(71q z`FaO9f+xn>bTr%FH>_sf)MPSTIOe>Vx@C8CM|(1j>DFvjfuoA*RLzkaZ)wLw7&6>b z)t9|`Ir$FqVN>hfFR!^yrv)gkjZ)c_vt61W*F7~-VqaAFa@S%Q8a>qR=w#t|Lf0G} z5WM`3>fpy3n`QE2;gR?wbRXQuX`JeQq;g546On z9$R)lP$Z86_n9nk5-IhyWFkRkeIH$8V$`rwk(x}FBC2r4$+I*~2^aV}R3t?Up`Kp7pseR(iQr8IWsK?Rh*me^KWg=Q40mndBJ4(ocd z)`Ps{i+xhAt-CDcdi59o1gvwXrO=R+e=$2|wGs2_S#6Y)vl-x~^Y zS^W8D@mkTHD{O;34U#-bq5SQJ2#&$4It@h(M|Ia!S>n-6)sG(ooOMd?kq>R;3mVZn zle7C|D#z^5`O+|vPnh>pA?gJ5G0!=}+XRGA;Nrn!wsy36YN0CmY^dH>2aB!}$i zJvslm@f#p>^(J#F&4SGEZ%klHL7towD2iIesmIt)oJ`>c8H(3&9TNC@gLNZiQ7vUx z@>yXCQ-lnPY1@E;pOg!VkX_??0JtPX{7eVw74{0w2J~bkp zY#fuzCrK00H3W;_PNFIltEsFl4L=?eqW^3&fklS)S>-IE@t`tr3D?PPY&KXQww6#%Ng< zLGGGbvrb82#Ia0^Rd#p?8sQfJtfR(PYSWF0UD1yO(%v}+g0pOnM6lX4_Z85*=i{|n zUvJS^h8IIEDZgAH^f)>JBD9(Mk;t7eioUeA!i6z$d0bWG=;F|DHu=g_545{tby(U{VfG)r8CBeVNtMb zZK8XI(r_w~9QK(&A&B4y8_-g~*0iYA-+R?-ON^@@Mh@196z?Ymz|Re-T>H#+7!YMEO5r?%@9e#Cz~t%vn%O@=DMDU1ijb-{Mqq-8nA) zS7pSHVdx7C!UYu*NFcsNQs*@qLFjLG$RzF?q?o&rTT!hK&n)oiU|X*MlOEw3CPm-i zYML(OOg`>uQfAW#bDjVEzQJ`QLkJmKMT^X{n{gI;a%9KhJeU#iN%%c`wzt^a-qvsn zCwp69*L@?sL>kA;yU2b_}S5TQrXUH9(ljHbZr?+9XF4Xz=`%O9t#IPURs~uyo^*+q)rxx^zgIf#s zKki5?M8Kt;U!-52-e>LHi#_7ecq8-+`6NILDX#pi9f$O^->l>P`)8zWbbhLpGa_dC zJ=JyCLz|GTJBR8)lwS55>swFGnrFtcHT`hgM9(A2nH=Bq@PXXt{m)_8X-^|*iT z`t^Rr_M*!i7L$XNp1lM|r=aqe&%H@*f``MN_{+4fd*(X19W(lM3Iiup43ni6FY6`w zI+Gh(ADrvRPX`)GVd#oUu&oY!8&=#kMgJ?1a3Y%E)j zx_H_phF>4zdJ^5flli3HpXnRr0qi(y50--^e9ZLdq2#h&)?)!L`uTBVu?g@;+5yps zI7@mifoH$gS%Xt_0!Gw^>-!%h4gkNYSxt|nrTUIcnY`1Cwx{QIfy7|o&OSXs{km7t z6I0e}ubSAc)sq5eSu#sOuz-WG5aOqRKTP)TB9r80m8jdB7n$nkJQh|L;lsxM&sMY8 z>gq3Bn^w`EHOy|_)1g+G#qVdCD8|rG0v?446sWi)jSjf2#T#vMR!xCBZfys=gRW%V zzenFO>kz{S2VC7?+Fh=YpPk^<_x@dCdpS^JkKC(H-bM26OLxBhO4YCSe4$TVU0a6f zi%R+MXP&!Ax_uAz@}=z8ue1*R7a9(IJk656FMrGmsM?Ttqw!tXe#v-4I?pWqtldvt&dX$Y|FFK(aQdJS% zrM$;8H;xIt>}UkGMWC$3_Bi&H{kwL=5A>vBZD^|#6XFF(ZI3C@%(o5B(C*I5k|`Fy zzwgMNKpi=;zAibO%Co}EA1A|J8{`IXghz|0{Ef$59h8WEF0C{4r#YJ5A>1Z7c1{X4 zBI`{mIPp__EwNT?NZGO=2*!P*l5+TzmfTdV_o8H21wr=UBG>L7p@Q+3Xo}wlSsOiA z1N}-UEyGk#Qur#=2{wXk160!ba9elDs&lHZn$K zh)>$#g9LZOWHi%&7X?2%5Ts>V6!zdanm;2QbgV#fyb&}64Rb3woj9Jg_Dc%k^K`Ir zHj3c`DO^FI?Cp*{ps7L?z3eR%RiIpGnGW1Srk6R#DLy%3mQjQKsrHla(XymK zo9ECIn!RQ!Ugw5xhL=Q-1SH?r1eQJNCYZ-XRrPp1sV;#|i0PuG~5=*zKpa zV+0xbE3y~RB2AD;lk-gGI1HK})&nx}@2!1scuOQSVOG?#N3c23$JI!|Hhxx)yUqp= zl}xqhi`Zx^q@)x@F(r_Yho$h3q?qZxH8yw_Y$6)(>OnJaa(Rej2x}u}D(JR-6S2Y4 zQb#G}$;Wt|eV%84u0|`iD&cxzFSNS34@yk2SG1Y$xLa4_?Kir5MApr=Mj0E<%k1bh zsl$e@#G4>bz~*%*-k$)id(>n%&zVbK|J3Ri9eOb%&bZEY%2<1)kB0X61=TB%i3}jT zXN~GxF@jF-_)2gT#Tm(KFF;U=9|tf*FSH_lw8d%qoH+b-cJAmnr;LXBtc@MuMcQ>y ze4zQp8`OmZ*E6%eZfc zA~gc0&55W5#bz5)xDhQX4mcf|PdGSb_EJPiBuD4nE#8GE)dlK@L74MHgFBKuz3;iI z%>QgrCJu1xv>lNZvre(d&X)5_RdK8$HMkZtIN{c!}V^r3F^QIpp_yD4D zl3%rA!g|yBw=+Jg-;82*<;uN*ubGg0dXZp_K{1njc{Bn@<4>FG4DI^fnlhfc*G&%S zS#d+Z2+Ff-$knFl{f_1c98Fk@_(<7P`kK5?Kl)&M?SkrQx2?dl%JQ-I`4G^5=jZ?C z7ychhZz2Cju&y0NVgmw$DgT2F5WzqNmW9dn{(plFP1z}YuhNfU?(l!X`lHPy;fZ~VIpmFDA!%_8xg78O)7n92F6)LI)zdml2>n-;?V9BO62|* ztT(s>jMTXdH6OlzHrEYDzE9#(b{kAgeV&zEz5GUf-CHaa`}D>u-3wvLv2a0$E6!y( zBuP)x^!fqUt-t5&dn>PqOXr?cw@D~er^=T3bd8bxZ*Ms+)bViY2g|=~ zyhU?mD21D26)pBCltTTR|9byu-XdM$4+j1E{HwL&$2T}uTkQrf+s&!P-oIS-k`K0a zH_AJHHMnfMAp6HZ@aVYh8jr{CJVJJn$49SoU_(FG@BaG!ho@f3Ywm|yjl5>9qN1_Z zXBtGiD376f`xtv4*$Gs(R~rtQ>cdBUFQdOFhDZnTgaijOh)n~fnb({bvwQ;E=7r2|s_hnL z+Yb-8!aeBwT@jy$hlDM*nsnlx4$E>FzT5vm@&VPnIboTTp)VR|>#u`f>LX#ej z(B&<0=sz7}^*zvAgk_2+?AjkHMC+FJS%&LX;*$3)ROdV|qa1D(lI`u8JNBzd@DoLe zaY(%ndltx!qxbt9ZVTc*PKQo!`(FYqhho`2QFQtk(Ylcj)ci5NYaBT*zP0-DdB=8t zYjvyy^Mj94cck0)j{11Q`%d?AU|^+7Z((BO0BCRckbxasQHVsIcWUX4@dc{f5_*Em%VdsESGy80TCG>K_Gq>8?MYc;OePBTbCD+N{RG(I^~e z6d`8&0J~_&X`Wnm$ID5sWNm8$rmX9;erv`Mq0-C`;#6K?1H6YO}N zKBhurss#A4sXou^b_!vI^#WrM8RG~4OB%=ib-Jq{GR<5-|{9U7X2h_Q7Q&9F`2K|eszi>UN4E3cwO)PRvMk62j}L1cO2J)o|mv%0%o zFYJ<>h_6CQW1^Vt)f}TJgl*>P3J@_OnkV3E;l(V4AYD-{Gst*gR^!t%w6lcwylui^ zb6wXAk9%hv@^Xb#`D@#C)QE!EFW_NKj6`uu+~f5cvbJu@$lDeHkhzUnX2bky&hjd% zr~@Ft3(y@>LBqZj@uUTCOeqt2!g!O&#(p|U$pq1!HM!Tz(li-&?@T-8f>dj7(drjb zGqc)ZUT1s5UVDd<)QZa+V0(qB!?TS&BIC+Nfc0dq1Q$ILs$+v5De8(J^{nHEtPybf z6OGe)h-GEV8$(Ke0aW1c(Pzrh9xxm9FjPwydRzlizz@<(Co_7+A^VAQ_7Wg`BK zS2G1GF(p<3+d91B`!%wrY9#uV5_cT}1r1&Tn`Uo}-lU2YqhNNIeQb-Ukm(oIs<%Tm zWv#;NL!MVXaJEimzFrGZIne#y5pTp;3`R4?7;btiVKYm)tFw0yM|pF4x0?AAg!-6^ z5i~|%8Hc^ex6Iz&b5(dkCF^}M=7pijTm8PnKVEt2UM+qv1TTUl230;*lcUjK#DU5C zJ#@4Er|sXO(i=3*^+xy}sgpGS=ilz=F)xCsX%SOHq`nE~UR_%mInPF9zl0+Yj3Kds zRi&X;N;0rbBzb{}IVST*GxN_)Fb`m;%!h|RH>DjJ5v$PTuxP$ldG^btG2;%>h3ef$h7=MpM1^oYqyPx9J9!-kv1+D+v-QRdia`)~2%{b~m z-dv0KnLjEVcubmc%)r#ZkzQljs@_&LtHYTpK@SAy{qp#g`V#p%=R|UiW7N{c^DAX? zpQ#L{8E6a$PqdZ%bN7dWmh&n{G7NMFsO*rzG_5b@4AMAqD_38I7_~^6@P>M()~x2H ziEyj>Zwbr0KUplWpe)XTkGAIh?1Wy-K~u&Idi1CK&giW)kQ~0B3y;;}j5@(Mc_%RL`VG@Exmk zeN{WkN5voVow)g}?5e_G$7{1Yl=97PSFdodd{dU@?nt*uZ0z1Hsg&@s?W-$Kr4AIV1srS6*UF^QY~grmEx zeV@Kyb_MUp<@|XFD0%`nQ&<4q{7sR@NP;KDYqKl&$}#<+DUZ!IfzkIeJezVvooSdN zqG7*v2IStaLk?%~@-GK;0XwmZvb}hLQkz;Ozegel@{_51$CVMg&*L~;Y=LjTH6WQ9 zQub9!_tEc}tGo0O$>OB|F@5^qfJZbX;-%fc+p1_@585%`V~wz4uS%>dVUQu~mLe!7 zQ5u`OHAV>45!|v8UzP~&zI8Lg@Tc7=^fPP<)>lQ7F{JsJJCYlbJOXWF8|+)7s0g~6 zuZVc-wvF6bTCg2YsF5)#lwY6JVM**Z;l~tFTRV-7_W8%~V4^_hf@H{w!b@t~!na|? zl0yGAU$|aqOq%CQ@sHWPe*3mYHVXAGw%EYac2Z~WupXh z!bamWM2!1Y5-T9Qhu*(lC6p~QgXwtR?7tN4^8-OCb}7@toPE|O=F?+uH;F>LFpbf` zS3y2PN?R+n4xMvNmo+Na+b6HT4gQW`x@j}d)(y}@98D4mLVKa5m>A6g8o+&!>#H5z zFL%z&82(lB9{p#^kJiayp3QwDMk~DM=3(G`=b9U(<`JnCUuun@aIBRwJ^5dao%LIk zf4}XA?qR5*VTNvLX<_JY7*blgK}1o898y3UL>i==p-ZK^J4As21px&WL`B)R-)HaZ z*=L_~u514R_s{qGthHY2on*w5(-qnIB24D|%|>`$!#ho)fcw8TsWuU?SLdPu=4Wq> zJQ0mkIWZy6gdxzcUhJnC@$|XTo4w&N^flt~f}m0y>RbM(B5^ZdbvRYa{%uzH@l5Nc zha5PQF|G)?{IaDP5^5vU2^4YDF8~JuZ~fT5BdhNUnMR~M&+B;yc*I>6b=WKGX|!|% z{2zU!Sq5{FSza2;7g(f+)~Zu$I1RwG^bXP*a>zbqYbwOp2nFvALv)b?MuHo0$vjKt zTArFZ3wU$N0c7y^bA$971+z)pPoA325%4UiWnJeV!G?H9Wd6zO5wMKtA@yJ?ioiHR zebFQlK~4Hq+)^d`G*6Oh_DKfN@L20(tLjZQ)U=G;SZmm$ds$1>ZtC&NgkP<)!;t&a zBjpgcK_~#d{QJlLe?8@4Ku3Nu({608^xqo=C`u{W-ha}0#_`73)h0K^IF@nLUt@nL z_@r6c%%aAqB}?FD_nUD#p04oK-607NN}FPpb=r@|sp@$1JB#oUIE60*s99-$L$ z!fL^%CF5mv1~!&%zSV8&iT?Gg9mqI;h(nbt}Y*n_I9bp4+qgLPWR0AzUzBRp)HrF&vv-W41Y`=}GH<;-V0_}xF9*-o&7HSZJTlO&GQTAsQWa|Y zr}a&qx|tMTnoV zd#P#@!JXoim{udjjfk5SkA>R#ywEwHzL{71`6kyC?HhcwsV#Sz(0s03cl~}?hF?%a zu%0n$1DFKBIDdq;=y@MCcAD`VR;@+v_%!r6F79LMp}$P?K-o{ynqSWdW!0_gZqC;o zXJZ>en-zX}xxP&pnN!^LxIbM~TFttEIKdnVXHN?W1!z*X4T70*3v0d)hCcn8LwpAH zdS$iHvNiTY;cxsO=gUm_bwJEpA&P5eY+8e&9t{%mJH*G_tT1Y!Fa5X9gGL4f7FoVb z7`ORyW^*n^N=!rBzK%|E)TD~dHvcYz6>vUQ7oS(gVKw{mQD@-1H( zIU^Vfx1u!@%;=vndx5Qy!`)_d1aJH4l(6hG&Klm8-&Rt_?ZT{2BGeX@8s!fqiyn4w zlqTYZO$FaCk&WC-9gzWg3g*9#X7^`^bCJ>n)So{qK{y zU-A<;9Q~#r+L;u-+Oq~brs?0b)7k#)*nv$lB%$dwoy0$E&$x$#(4aF0pRNJpUm0f&aI4s7LWLjYS0D1oK1i$n>3J5Cu*q z0=7>mkU(l(sA`%liTBLZQ&VAXP|uTr0-lBjJbqPl7Ld*Z{kcUkG+Nxg3=$6;(!HZM z&jYmIJ|5I+=M;dE5IC73W{5W-IjKCk9AXVoe>?;MsJ8N26J;;DlIS?DRzwY6j=4N2 z5UdO#Mmu)#4jYI3F!rg<`KE(7qYC)7+aiN|5ha&Fs_D%s*$=x4OiJrxjk zQlv>nh`3OuAT{6!iM5e-nWkKE0g;8E)o;oN=0~*%TGF7T@^F0mzpr{N)(zvh-PST# zyr=S6(n@nUCddz;7C$pR>BVHu9h`? z+wLcR26}nhTcV(W`lp{1?)4Hm>&Q#9t{4Im3i#LOc1-Py`7nc?k)H9A&J%JS;IVfO z`&H!%QgCN1Ej6a6g=jb_zE2t0WfN4>!FDId@prB^x>JRg^3pEz9h*m6T$#xaWyZZ4 zIG?BQCj^N(C#i|R$(v0`%UhDGgDWc-%$N=hv<*KdnS?%lf?2K}FR%`-C4d@<59kQ% z9VKh=^jt2*ihw4(g`N{C@8)YLeAPXEph8%oH8*H^voOo`yn9jHi$tyL$3MNiwS+iT zi~}Svp(Uh@eos4u)-Nl+l0AV=NB1MtPuI&`cym|!8K!$#<34Ta2R&=|uCb&0c2-9I zx_nxHUQRxm{H6cjVYvUVnePp};86f4_g@72-<6{M2{In-Pybc`nsFc|Z?}kl5$v=) zB-C%a!v+2o?P~{rYLck0K(EV$vjWlnxVcl$9bYsvceoF@EnEmN^Sv|SieCRvav}gF zcQVK8KGAkG7P>wGQLJw8ZFUFDeEpiRzY0KCp(x}SdDDL@Ta;Ui*seQ`YKQ!-ZZmUU z_f~I^dH6B3z}q(jTr$-xo`)TuqD`r&Vo~bLR!?kYyW_j{_bK1Cl3Q`GYREirKY@1! zPyjdUU%iL6e@0WT_Pd7fhx=;;wMX2mSvddvfjE}>L-3@W#NI?ua47iYeXaGW*0pPu z61LQfgf}m5t*(hZC|>9^50Ln*@G|nHXiv6yK|QVp##gW}ZHEg_s^Bj|^stwvgs8|Eoy11bstt~SOnOWD}^y(6t zN$B|QA9HoJ&5@QzC7spP!g?6u^?EaLhZZ8t8FnX@U2r&3#93Hq)3ZfoX@2IOq0y#U z9FSRPSIv0a8I`~N0=#PJIhaRTkpc@dT?^Gmw2E`+UxLu|Rw;1zg_Rcg)lZ{C0;&9T zmQzpR_liyZq+h#FJB~WPjxk>yUZ_iv8DELYfBqKa*J2xJ-+<+arFZ6J>X{s&M(P$z z4xV6&gS>5Z=tL)^?rPLei`{Jx=}@-T5QZ^55tgpjOp!KbK5^`{9W}gPX`&dB+FKC3 zYU0o~E907awBi^z`E`}P&0LGgFZHQC^Vi}BvKzEm^^uaRwz6%dz20k~n_G-IA!&@Z z9oYEPsMiWNw@JDRsF?lS?0I&RZp?J5?JG64u_a2?%FX974nbs{JeB1$9wl;^B384- z?tLg1YKT;edDqCzY7NT59Jk!CNSEdMxK>dp`Q1nnv~NK$lA;lhB{~DK-pD|V`zl zpT$fz%qjZMj{u&%d~MG11lBi)O#!7o?~*lgB?fc)*)P7nO9o*@?)s^TGG{v}4)9ia z;a%Z3ss_CA5hU++aHaFC0}ZMy29 zFZTw$nNFh#Rb#3nW2=Q zY}aUFiwFybP*v17$DCt%8j;VWWv2yy4o;^JVTe?hlb(2{n5=B$sQ5^5g5M`q23Y-@_P=g*^Qiv^Tf{~1mLseQpwC%>a%nWnXq4!`hbcL#<2;}bb0@x7K2uM$( z6;6pLjpHr;xl|*IBGbq~sBc*q-Z#C$ZYh9oE{aNOZ$GCk5RZl0R}-rDp40lBd($On z5C)~1KU*}@QE)1=Kx_kXfU2*{O|>M!wZ9%fEVEoc*)ZQqWA49te>t?43CY2W^53H4 z5ZhsNPY;Iptco^F0$pf4fN}s>Eotd`W2HTB`k1y_JY{X)#Ed6NlRjUSbf|iTuH0jI*5_VvB*mKPg?9F>rGzT$Se5>+BmUuO#2&OGQ} z4`8=G`Fry!v|cryUv-}BSXPbP%p^8?5w?v_!2}`+&$(dukv%hsG6i($I4Mw6`~bZFG=WZ|G%K1&t`A;3+uu_g<6TqqiP<OMK-I_)hWfw<^0+soV93eph+*g_7}vbMO3*ui{rWck7Ug9 zW$SH5@1^bAUj5j2I3JB_IAk1Uux^!RiC%G7CSeUY_1I>x2erXsN3h?V*nY?}l0ZWQ zbQlvx%ZNspX=y_yAF3?TZ20ncAdO_~TY>8S(A7mT^-Qn8?bIq>LTVYi<3!xUS+Ah! zc-5>C+Dt(%=7g*z8D=;6S$do8hyAF`G=p2VUh9@q^hp)TeVX6RbzC-xmwZfB?rL)K zHqv<%-OU7KfG;MK6Z9)Si0Hxaju(?6?wd_%qMeh(Gyw38%*Z7B-`i%usam>Mx?>5Z zp}N?SEtE?uVV_$uODy}As_wRWbl{gyuN>)F>5Sfe$NdNpt!shj#KlNtvQ*qt<%yy7Ezt`le-|Q2kWq;Q8 z(p#qjsh1I2(1A4I2&;W&RL>def=-z4<*%U3mbYu2|H!Jcwi9e2@rZJX$JRfoh5IaR zHAEZwo~HGo+)Z_4nKols4yTafsFs^CD|?vfvG3Z)<&dMukbb%T^<{T zYwYb2jr0wTp){Fy)6UzMW&*Dz0@=b@-(0+9Y5Qit^r)u;yB24?L@R}6?A+~1Ioi6^ z5EbOY#Ql0%bhw?TzeLN9|AEnSQqzY$QeAW$uALp=X{yzp`{IvD^P%SY2z6-tN&5+b zgNz|lX9?LGtjY?wSB(t&m9)v!Q91wfq~emyH=<;Hv6DLMTXO~{5l^`T?XuPES6*As z(sC@eCl0?-R6mJ2-NOvNiu=SWQuO8PquWN8C%=q7Gp)Xkkv4vyuz`P7Ypsf6gINFg zLFH@4f9tc%XKKhBvmXL$FF(dtvNH`m3!xwDH@rtJaDvu23;b3jC6e)oNW+<@{ghY( zb1JidawafEbI2AS-gE{L1YJk0^u?40vfVj_`)uHN@u8WplYFhQ{Cui8WmDLG>3VqX zW$}spSu$MLII+94@`ehQ4L8Q#)>}JBoTUzSzAS`dP_Mq7-;i<5c4U9y$sjKxs#_|7 zV>ag=Rc*(jf+6Z07m&KP)OPVcQi%Tn;>QZFNKE(fWcvea1k{t zed%%%Kg>?%Lr{W#YFoqKp}`E3I4ujaN(KrtC1CZ1H+mTW0~?&E)ChC7%1KiDhIpZ@ z$YwqUGK~l#E9T1^@s$@#(=3tj%fu?KDg81YeqE-iXL zDGvrCuSIsq@<@;Q6y}p=KIL{~lOniiP06$XsIAvfEPz#bm1`*G_6O;ulIr_lWsx#S z_kJUP`=o<$Ha^C`kkFXu1pV{Vjjxi zB^#FxKreM9R6h0ZLMbYTzt1J6nmdylha#}Km>8P4GYaYuVD0dNC?i5`kBiO*rp4DR zcA0V8(6gM5uRT7yCe?-Eb?5MTZK5dBIoU`q;-$i$;Ttgq9&l}M#vT}&E?qYI#AF0G%XKiG~AaK4NA#antxJE zJ1q}Wi+eF}{w5c#WB&|8D7d6h&u^|H?;=6Ob7@(3Idhkey!kVq2wt-;MBB<;Q8=#) zL2W9Iuw-;xjvPE+!Sk*QryR1V`T3#oMYHzLOuv#BJWlzN_+MkFuZ*-A$>#fi`*JC4qWpyw zO7QXj9Zfg^f?B0#5b zsPoi!EW2_s_+*7fIox1<*)^EH4YzTQTMb{`fC)`D7Fm`{Y(_R*W;<}ohVd|8Kj^25?iJzb=2@A{|RA#C7Q|Nm24T~ zg4=kN+Oa{)&b=4oC{0f1#t@H3pD#^0ih|InVHxM?(D#V-CAd>SP#}Cn;fCXwyZq*q zL~XLX7P>3}Fc?$bM9K(>Bos5k2VN*W;4jv`_dr7H4?S&i;5s-DKs)8>iDw-iOA4K4>| zenz!<%7;dK-Db7k4%)6uNe{7zLRXINN*|TP!7Va<)wB~eyvpJ+d~HImoX(^=ORO#D zt!DM6N$|XIs-Gakmlu4VUi}cM4@8ROB$TfDp-u(|iBYxP3oV0bxF$9W{|-$v3a@)7 zXvqtOgJiqjL8DxX!F)qzzc*JS(AjVIB&Sa-xKF07z6gfQUZH>y=ANf_DMH|19)!j? zWtxAQcD9!|8-K_*HZwsyiXDc6o=^$h6Yx$vWn0NgYxY>p-Vv4vfLwbzP1(*;0vg>{ z5f;BHp1r@wYSRl|?qw$88a7E>rDc7_{6H%u`#$Au#m@~%q-8!GejQD`jyHMBROav= zZzU&;E^xl$YA$13?W6PGKgGt9um;tm&*gf)ayYm~vQCXKxQB|#JxH6T2~C{(z4HKP zf5&z7+oa!_j`1+-V_W%+jQ*FWH(GFl$9L~C@Gz-87Q14|ik^M=>lTezLyb%=h@2ux)@<6zVUmq>iN zr%ULn%~hbIC-y08Eu@l90wHEVMM?>+bJJ8k93{lp1UqA>S^G}GeNpov2KUHW2j(PY zEzp@Al-qt5>glEohfr5DJ@MGJoEM$mh7-<4UwwbjgSt<+P295dNARZK`j)wg#w9YZ z)@WpBs~7J^$1AwwQRo^VD8b3^p{IdV)@yX2ZBICm7+<%e-H6!_GvYbQo#J9xT1j2S zV}t!^+BrZdQV1_F@KXwO4_u#3e&aWoD+4S4%`TAmbXpUH{4h1jkO;#_7KM1e&d^Go zXix5?eq(Fszvcyi?4rbWw1NMClRq0AvUf2QYS^?Pdf+aL<@7-s$IIFNfh9FMhOq;qOA1 zN;(i*O;4!hi%<86LC5;WnH7%?9NO_MiBl@poL>(0qUb<_|IbhB>4b7f3ezAL)D&&v zdC~e5>0|vaGES&h%zi~$r#*$m+TQlrYy$~G(2KRUv>HO1# z!j!4w3e~<5lORq8XhA1e5+~NVE<&-z>!DrQX(v^6x9WNvwQkCkpmRIxG&*D+$rc9`M6U7i9FsIDNlz!36_qc#w%8*}*mk|9Nb zg#48R`7)s2;u;yRN zLA#E=ljiTq1;#p_bxGjl0^gmFq3D(x<5We~Tw&|4HS-mcv9cNyt%s^rdkg(V8|n7) zHOQz>BCEhz1@DDY2UZ2?`t>}uE|m(p8qud-Cle)4?IZ8YoB!h^Zx(GjSUy}4!LMp`Jdl<$3ahyTeO1WUi0MmqZXUG`~4Ql-nU=^WadkBE-c{@{7IAWolGtR$PA&gn*9d&kb# zX`V!p6JIZ1OQl`2q)@h!eOfHX8L)ah$zG zG8r=oBNDAOBd26)cVet1**YT~uq}xz3%Huomd7nAW^{eH$o2>Fi8JDWmXUCm)@$8_ zfL9V!?tAXDZtKSx?_5yA)^1gG!UH#{mS^PN4cDz@B+WSU*f4OWN*GvTaW^FQq72?Z zZ>4Gm$MQY1&$2fYD}RGi9uRprf+2yn<}mrkG6#3yGkU^YcM)UfJ*TN+0$HrY*y%xi8V*(hoG zdR2ng1hWc}cc;JW^GM+W8}_!$!aF~SG_>!g^tkbj5=5qkm4`Zc|9D&QL1RYoAxO|i z0*wiZXRJx)@HiAl8Kz4+wf4HRSznfxTbeJ-Ts}HV{Yo71=7INS#KdxW69)wc10Q!C`54&iZn16*WHQLSlVR0^Iks957QsZiR$Nk zeYFoslzX4a!fZMb=8jI)`g14ix-yokk*&KKjUoxu&AVdcjkRiWhn&#J!VCNznRkkHG56LG8K*Qx_FSnVcL@MFJEfTV! z=~h!uw0K74bd$t+;@Js+#Y=H8Vgd7{B+h{Ip(V$zegAzaHFx*U7!oTNNz+GtFgkbn z%&?_qsz!tCIi7L-IB7TbQu3;_c1*li!(jHj@O$$CLTc|TZxvdRO@@y|hF*mUg)7-G zzIz>m$VhYeM+H|F>|uV-$zHK!&@!jQ5}yZsO8)bh8dY)kYh9U*vCLU?iqSVONLpG2 z)fZ=eR?-Wj7+N=hJ3rHJO%fBAOl za*mj~Mu8TdG)33v1P2%Lh-%ADM4OjUrk02|PeY*7` zjXS|3^Tjf4c((qnM^hRTlkHk)_EpSYA#K?6e%t+9qRQ^j7x)ED$adjR`NXWz>v|&F z&K&z06DA#cTETtlY#5zJ5i<%3-Mt;9eXlL(A8{({>q;xOgXjsd^FpxA0i0Zmv?(yn zB#y(^nX-|d0GJ{r;)1M_Mh3?FCq=U6rR%bP#3U=QpnyGN6xlXa0A2N5EKb)X_|_@v z??S_c7>ZN_jS#^|ZCx`okZXC~fRj3t{juGBoqEt8J;a>IdDZv&I_$=S+iJIKuU-gm z{uX;AflAUhxg1D<;a~80rrKS>NEw>~V~T*$r!kk-o4_x^&KY1W1_t2Or+l*gNd}mH z=Z?;JT8()!!?;V@e~3=;c;cS6&}?39hGHi&=DWM&QYB|k6xEXSc(`bCoZ_-;-Xu|w z6p(616d4)l_QcD$kr`u7vz7bb%XTnd|3hx*nUdeU4~#KHNb>#pR5cPFuXoo8iO+}6(zm_FY4 z@umIM%a`63_0H5@q&NAh4U!F5ueJ#aq=mG~K3prPEmc~nvzYmLC%<3;Rn4rHQ7BJa zN$il(Kl}n%DdcuVyL$rH#$6No;N%2~0@5Z`lF~()ufuIJ?B%8Sa7l?%Bj1=wRqCHl zh$Qz7foO?`FGeAlqO2cbIDD8t35zX6^#_xAed@6XZHSE?~ngxn_mY?P68Za=DNS`^`c zAV=KiH~ukU&H*B??vbl5vL2~XXG7&9=(qKME)>1`)}MyY@a>fRduQ#}>mu?uX1|iH zRd+8sXJ7L0z#%$)_-xf%m1am6hmGfn!g+=dtx2;=zNLyBXEFCoKP}u#pyf=LMk$}5 zyCP0c!b^jAw#Wr51b@5azE1jIl|Xbrx*$i8-~Z4}LiT9joo$@pU`{Um!?bELL&We^v5iEvL7^PuMsGpOo&KjvQRGCnX0@gu4LgqW z0{T6(V#8ax!XHi#$7)69%j&%u_^c8w*uA$lmXECKP8O=GY@3cpYRzx74-%g+w`i|! zw1wHV2wH|7zHx9JLA2{Z9Dq2-PIG z@W5r{F~3xlh*5q}UD@?@Im0ZC`p&0fwA?M)=FlF&YRPVS|6bM7bgq^ zoX)n=QaMr3b!i66Bez9LXOLC|lI_+dBq@ZH&vPMW5135!MPD(NMWcPOL>$`~OHLnq zyd2vwKORJ?1PzBdC6`}Z{~+6b-;0X`m|CN% zT*c4gA#Q0mA70nRGiFA2q^p`{=^E8NPyd<926rKWg$FdFh?ug>&Rt`AjaI?Z=rC_w z)>{Trj;9Hq3Wbn+0+J)5(t4eH&!980joD^7Pgdk2T2VS_qax$(9w#F0+s(J+D_NAl zTq8Ac_FuL~ZJ+PnjGSHqkr_P%J0%IBPgtadc z!~Q!&($2KLr`o&OdHk#+WJQ(Hgg>!H`Ee|1pEP&f9A>0KG0*pFtex=2<}_#>&Zf<) zq-ME&+OJdDU3}vnq%Cli+;3O^SWB?mGZ@2847E>{5~w1+t<)Nr1ltYEg59PeQ#>V% z54iVP+Lfu7&Oe9Em43)Xv%HGZp*9x#! zzJ&Kq&IgFX4; zaF(jvpJ;1++AqC?kf*Y(WM34RBH`Zy?xNJh@%27dMBrMav8pBaIj2j zEZcP@-}CZQafOqJ1Zec78r*+}S|P5C?Cv0?Sg)6U&>B9}mcN^+S`*wC%;EyMOc!t* ziZcx+Rcyjm3v-Mhr^Rv1GE7xi_X_YQy;;$X($~Fln#p|i+)C}JDwa=Y0hET!oByJ&TN`W;;XZ{dp|>31dWc zYnb?kDw(8~1^SAGAU7CO;bq9i#S!s!RLXV%*0ZN?xdBg;6whSpm|J8h?X1r?z!pZG z4v;I36NjC|B_1=4LiaG!a+lG}zgs*q(hJI-W4LLBu3*j);++9aQuoaZv^m#cfNq}H zAs+-T1ikOn`Bhub(o9y))Fhs^2X8D56LMvEEM3-#x}!EWSS3KU4~OT{QG_uS9!Yu| zzgu%-$3doLXG@8E$1(+W3(~p9&%mlfc=jXPd(a8UOf}(nn+4Dybae39{@&xs8eqo$os#<7Tb;0K|Fcapo9Yo? zX%Z)(0fa(2tTsJaa(w7eCFz~R!$xX3(wr-|GeAm+bzt{oe*?i zh~oMA44lbhfh1^5oL@-E`G*UlO$&JckZA7Gx@Pd**9l7g9GJMN(h2lTG=)42a4atpxjn$?y{X&sMu8GE>nf_{)_rU3=*>%_}&o5%(H2Z=#`zM})h6Rvn zDV7Z}(0G;BJjN||i&XqR&h{bOM`sj@c7-z$;kqmfr3uP9Wy5)$!Tc)y$rhOMB?Y7N zbmJW|9XhTtZIF7_GUkvw7mei2CwbX8wq_)PK?MHR$anYkfzkIzM7ypqDZIABHS)Jvca`^F z?(=|kwHa!ilAJ!Ty!s$}H;TmZwt>^#=~&O0d+tF_uzfe{M-B)$Tk%%1-2+%R5ojPs zKh9c?v6@hVO^+iu-5TcH@~p_PYr=XaSRVxI($7F}C)+GgB6#1mI>b2n^=y>sxX!7W%ZH0Ke?iklgqn9;f4$VcOeXeyDQNzM8JTmN) z-^Y$QaU0Lc!M^5JFIghTR&JNSXzg>YjaV(Y(Ak)r7}?s<-eDN~;2ARDqjPjzOr}4A Z%O1X7LXV_AuqN+j$(&W@2ZW5`e*h;#*6RQO literal 0 HcmV?d00001 diff --git a/_static/img/reinforcement_learning_diagram.drawio b/_static/img/reinforcement_learning_diagram.drawio new file mode 100644 index 000000000..2ff4e6f02 --- /dev/null +++ b/_static/img/reinforcement_learning_diagram.drawio @@ -0,0 +1 @@ +5Vpbc+MmFP41nmkfmpGEpMiPjTftzrTZZtbbbbYvHSxhiRQJFeHb/vqChG4gx95ElqfTeCaGwwEO37lwDskMLNL9zwzmyQONEJk5VrSfgXczx7GtuSW+JOVQUXzXrggxw5FiaglL/BXVMxV1gyNU9Bg5pYTjvE8MaZahkPdokDG667OtKenvmsMYGYRlCIlJ/QNHPKmogWe19PcIx0m9s22pkRTWzIpQJDCiuw4J3M/AglHKq1a6XyAiwatxqeb9dGS0EYyhjJ8z4flQfCient3PwfKXVfTn40P6/vMPapUtJBt14I8oJ/AgaA8opeygZOeHGhBGN1mE5JrWDNztEszRMoehHN0JExC0hKdE9GzRXGNCFpRQVs4Fa09+BD1mMMJC7s7YqvyIsYIz+jfqjPjljxhRsiLG0f4oCHYDrbBJRFPE5SEsNcGt1aPMMVDdXatbu1ZY0tGrr2hQmVPcrNwiLhoK9G9QgGMo4Lec41T6gWN9535v4C/WFLaPTmM/AlrebR8t2z0TruBScAEDLgaziKaCtoI8TAy4wg3bltYqIUFZ9KOMCaIbElgUOOxD1rftai0UGQFCg0/sRzcsRKfdjEMWI37KGkx1dOD2BtCuaQwRyPG2L+6QCtQOjxSLgxz1DRdoWqyOqWZ1I42+kNdfCOjmUOFgLCR0U0aemi2XDMVxgQ3ztK0X5fJtjR/0+EWjkqA1z0YHr7dYz7DYR0pwKM/5AfFRw2sEUbAOh8PrLYI+sgbDaxig1foy4dWxrh1fAzOACodeqi5lPKExzSC5b6laGGh5fqU0V8g/I84PKluBG06HQu8okcN/W+Q4OyS8CWTfsPFPlbz/Cxu/eg5hm0nEmPcg2mP+JNs3nup96Yy823c7h/HvTu8/cXfqV9H8lXen5xxJuUa+O91A2yd4+e709LvTP8Hvvsh/mbvWNtPDUR0hE+I9NVNF50vrFrLb+kLZq52hcaCO+9hTuY9zpvt413Qf3Vqc29e6z1xbyJkm9TSKb51fS4mdKVJP2zXc4fc8grwsLb3rlpaODog3cItaU96ijUdNmirWscHuXq03jjdVeLDnZ8aHI+qcJsGspewFdp8Iee8ivJU7Ehxn5YD/z0a+qN0RtOZtT7Ri9Q1Tac3ZqsjLvvWJQZzhLBbNUmtqXSFnuXQzb5zd7Bvxa5FQWkgvbB4vJDxCgXVlCEOOaXZhURwpylJQiRQFZdsL7wfkfh9RSFkkGql6XrQ2KiRddG9X7t2+rF10L6/ElUpu5VZ/ZWUt1D/piuk76/K8pWyq5S+lHiVi23oGaA9E7PlAxG4Yxw/ZZr4X1q5Vu9AE6V8wP5UAyt4jYlgcG7HrlUhVGL1WkgeO5EDf/r5oDdcuo9dIeqUPXk7ygK/xn3iPNACxJkgKHTMpHNVJBmod6+Z2snzmqmWMrlCgVx/nWjjQLc+7jIUDvYw5ZeFA43emsFjzCf0iYd2ava6q7z2LTVbX18XdyaDvX9UjNIMBevl2tkdo71VATyrG8ghd4LcV6qLb/oW/Ym//TwLc/ws= \ No newline at end of file diff --git a/_static/img/reinforcement_learning_diagram.jpg b/_static/img/reinforcement_learning_diagram.jpg index bdcbc3225026bb186debc90fdf241dd6439720c2..7e04efc253461493d0475101eeef876e00d5b5dd 100644 GIT binary patch literal 34793 zcmeFZcU)87wlBIuDAJpBAxIHLiXcTgh%^yFPj2A#+Bv3Gj5g=g% zff+&Py#R#RPcqQI9KgRFAQCVs89BuTN-ApN1@#O73Gvw^q+l{KQc~j80mR<{QbsbS z%Mxnj%nzPW@V&Sw8JwJVf&X@O7mMKpM&Rbtmm!o?tZeKYoPt+`u3i(Cl9rLZB`2?b zM?+IfTj#FP!$-y@re@~0&+P0S9G#p!JiWYqV7`8#Vc`*xQPD9esjptArN4Qb@jkzx z@Iz5?$;X=7y84F3rskIJp5DIxfzN|OlT*`QXJ)_6AyMenwe^kfn_Ju1gP(^-$0xX7 zzyF{M1c3jQEaLmW67~;t5eET8LP{J`ia+QAk@yh*z>K71mnF!V)E-bgdBMyl8GPa5 z?c}`bE=vBJh8UKoFDIy21*A}d*gr`77s~$c5f<|Qh_Zht>_6y20yJO{aq_^7fC_MC zluHss1zZLG&&U6JLm;;dfxX}FB5*-n^e0i%t{KPO-PFDn@oiASuM5W8=Bf#g9*(f1 z*jH>Xf^X3~7&r&QD$&bl7nB*!f$o|g=fLY}0MTuNbt8m9 zE+9~5UgyBjHlTvKj%z#zZqtXUV9Hv~fpsqcf_aU%I0y15x*<4YE5t!d5IycLLG&CL z4@J@AuZwm5vQnkM3nEz0flW0`B!MN}249g2CNOozA`W?R>gNE}a3#US0QBG9D;Bqo z%lJDJoc-UeFhUWl)h*LIQd`gJup&QoA;^mNjV7kZJ}}ftox$oQXd_5T_`h?}q5^Mb zbx;PkD{7!tiE|Mv4^$uLooxfk(<<$%6v)VvV&K$(a0Eb{AxpX7UG>vxAfOoG7errA zzi$Bgzqk}M0>AxEIkB+950}}~`P-*1m)`8ZlNoQR#22{*Gqy{g1N1rgQltSHr0#Sa zbe!V{C{OJZ$qU#7PW9UXh|>v?HswPgplB(G{-0h#hv@&gKXv$e%!V1}e-7NAc?H^i z2pE7=$$*Rh-+zFT(PL~llTUtRV=-+|w333_L>W@Bd0Lw5RbxZYAYXsx+DjIYkH#Rc zT{!U4hSPPoQkhc&Vdufp*YxOy@b^7o>Ro z!CAhd?}yFb-{UuMc6oy+Zt7kiKhjza^xOL-HyWooM~1mG%`Uuso9-UF%fTO{gAP(+ zLc|eA7K%4sqRQ;0Y00ppmQk^7k{R}V`ctF2Wp0gf1aT`8@kWipO$sz<75G(!W*^R2 z4(sePkNdrrVIQgz1r>FuZJnAmtdF7{a}&XYzXnTQSA5ca6ZphL26&pM4|GQO=|@$- zMFY^(g&lQiy1p4o!Uwu`P1>|Q7O}6Qx*ZOJ5dv~a2{HN?LGAf?bxdWHG*`fN%^B9c zjeVU^t7zN!D|l%fdeG)*k;aHm5WT|EONrQ%*_Axm?ONiBmls-$|D;Csrk%8!v~bbkZrc+`<-| zY<*C*%uCrovp1+;@Ur_^$e1U*fN=+vlo2L~vc{91VG(BW5G##zmN6;$aIbpvWa^H% zHA{xV1IgRAPD?U|$=BK7L3xQ6L88mM)R>B4jj8%yidtTR6(1YjT+@E*?VJdw)%g18 zi?+Ss$TfZTOTdZo4z!2Ckwi~_^`v|8%JD|`_r5DyH8r7zEh#ho8yj5yI-hQg2~=xN z6u8n4tq0vwBzEVk(~|gv7u42x-1Eo2{ORo!_WBFGuB7xLPahigLhU7L8ZM@Q#a<*Q z(W=x#_6^8|V!n&J<%|8ny63=e@A_PN_`mWlNk!pDrwpaa`T79Ong02a7>N9aU9{pL zw{5(v1lU92WAV0G(4}Nm)?y%+s@gf@+3)z#MDlzW!LxS{^j|oqUdv!#7vuL)^%1U4 zP%Vh#j*nXzC!HE2D`T><(z|q{J3>bwIppJ24iTLdK<*20xw(OL`ME43wR^T(X4~_* zxV^l=$Pe+O}w+#-9{CpNzeLI;<|YW4C^PkBM8aj6y^>38a;u4HfcE41^lvt$q^*B_BATQV^|OfeCP zGa&2486g-En>&HR=fFy1cPIV>QOjIh?zQ=?0Ey|uiDD|xfop##x^P)68zEr;T}hyZ zUniKr)8AgZzd)87B4@lz(-<`E~%;BrvnthoJ0ST{VM z^^Hi~sd{chTN^lOn=Igtou}(#L9*Q$Q=67yfmYr$5P`%svE~%J3DMtoGN-}`I)*01# z=&iznz*wMc2u6RHB~)%`@iXqkJ{vqM&g2|;^@m}UiHki4v~01&?_?b;L_7P!xS`Q4nw&}RykLuaXxGT^| z;`_a=wlQ80DaF6jEtTlP$Du2?c(6;Y%Iy;scoDDLfs3SwJ5sXIX`4DsWckoZdeXty zc_;BcW1aonb5i>mws`LO(c!vG_deM)18PJ~M384CtxJahXPD118}eO-R-~8#Ls3r) zJVmYOW9muz+eK6N;j2XT8Dx~W0*RUJoso~4k$Q7DywA9u5gD~&@~VA3c!ulYRK9)^ z$(ER*##1)C-cPJ640P^8VfUr^tgdxm1@||D5;+)g-Qx1Yp$k_rqrP)E)I{6YjtE$^&t8rFoUF@0r}X@ zKoKtXK2Ga{%aYWqFK8Q<>e+gy44L?Bo!7<;L(7t{8xj^TD8|2AEd=};|5z9vOY$N6 zrjwp$VgnX!6MJH9YI%LEdC!vS`%1YDzftlt4+vTJ?#|c?#YB7GX_=j=G{jX(l=kFOYHH1(lIkkA z3McThlLh9F9*yP>gJ>CzHMZn{p(xP*vs=BmrPNPvh3)T zN2bB?cPc1>wGIv#0__%yH-fI51I*iqi#zyhP&D*DY(rL_@Wy@$qE+MK=`#E_@SA~6 zKvm`28%dJ8!cpDAz(&y&;RY_f`}ZXk#*#f86Q= z=SFJx^nD?u>g(Ixw&07P((*p+5|rLD_I4(^%=$*ECRQHzL6b0k%5>iWG8bR4PreLe zKj`SC-)sD}tb7&RI)SHNU8cnAw6srpjYXUIxxU4+4&0ksnf7YGOPHv;R(DmP#fP~y zdiC`ARN(L11dDz~4wBcoH2Mh{Ch=%o&*~SWZes}3Wh~i#nA8P0vrnFSpw!#?;);5v z!2kj9&MxP#h7FBYWrO%-h(s=(|G>;Sz^bUTI>xkBwRsLOP4-wISR8#CH0n#rc^2lZ z#IpG0qjoMQ=v)rZm)RVB|48Ti(ntcR6{Xml(U~&~(-j!`X!%irjN+EDQaoUck?qrufG|%U=tp!i zDT<y(}@;Vse^tF)hZ&$Zgx-Wr`shzOFc*R!}gzBB(|rd~l$dBXZuhJQ)g z)t=AW$tCu@c3H1g_eOxV7$9SNd!~QJ<;RaV<-5D9PWBll7n0w{)!yU*04ALkDnM6Z zSWk13R$ad(Vq<1QI)f7BjveNrDJ`Yr(B=)f#XC+q4bt<9zkKAYsGL!r>6|n#xR%D} zB}?hL^Eu_s&nA9J`G|f>O<;`^X#SsgA{4LTT<1VaEOwqSM3zH%-$D!cmH+9t_?kqW z18Z9Z9>kmqHl6SeE~m;<`e$Q2UReAt&Q8J4MV^Cb#grEUzD3A^ascAOQwIc;`4Jte z>OYjhf9VuR{ZSMBi`)-H^HS{`m@2FX0#yEdAkm!ysas9CNe74psOZn`ADXRO3LeF% z{_GBTsDCLrm2hw)mtK)Dh5L%xxPoAC!oqbY`0XIiCw5n>XL%DTJcgY|Mfb3AX~B7L z_p}391G<5(90o-Qmb~BP9Juh~g>|eMschOlSC^07HI9P{XyK!bB})c)^8-tgTe)-* z{}cb4UKk6(XQ}jzoeB{oiTzHDyb7MIcl$-QgqUs1`AwNipQOxY<&40D5yn0*@nNkx zlJntYM9d?_N?ffi z0}JX9!1XJPQFrznPL@BN6*C+aubbcc@bgO}{kkP(?+AyT39x5}+!;}oYLi&?FRe^a zp@X^+uEH`2%up?~c#PCO3(WL~EcxiNUZnnvbr;Wo&uf~ew?x*$yDM`>3O<{>1pI`e zJB093IEPa+=UEj-!eyL8cd`v$VbX9$&&{~8+9JYA*Fc2-rhFtXW75sI7ywnSeA&v` zhWT>ql=77yY=J5*xLEG2#)u~~cK(!Yd~48VT{2s+1sC&io*zT1?BK>ZFf0%Jq-TWR z$J!NjyY1%Cz%8(G1jQzdOlbgCcu2_@jn=1gE;)YLVO9OK#j|>$uBwN|g*{F5OM_nP zCyGf(?M!=t4PG6amWWwJvsSO#0A^eAikNNrgnH!)(WAFKJ67`pJ_-t{YlWGEbZ9=G zPn2!Nz{${Gql*WXp@@SAfeMH@)!K7l+C(jv{?VVf$6u|Wua5r)(1E|41CB)fqwVIywNa`@E_|gJTO<_O#Iza7_YPdnNx`vydd8@Gz znXX&5H+fhUHal#eT-&@JIPfHb2`W1|xM3o;qI@Zr z?zFE%9Mc(uSMWzKbNrYwmx(XpP1!y^cyQ?fNtcPhiO5hco$d{QNi~-oC*GH#p9ts9 zgMC-M$$hie+^(YhnY~^LyBMFD?LvE|OI$*KkX#NI{BaE?Cz`p-Pbj2KZ&jS(EX+Aw z-Zhw|I%V`rR`Qh$-IA4XaNA2#z4hhcROhTBfI|jRx7_v1ZlbfkO^-fuxBknudR>=x zZ_~QXM`^1&`;G+vQqiFyj(h%o`5t|MccTqM8`usVt3qlGTkXNX5+B5l)lLddqb%m% zRNcp|h;GQE^m!S(4s+>LdXVCu)UfIX0pMQqN8jTPBbz&#C$*vrS^|W1<9_|D$eyb6 zP851GU!a|oBN413_nAWL1Kbk#pxlu&C+5&CdWpGnLdry9+x&GyJyf(jm9A9k<2pY- ze*}Ly@O&%q8fLT+g~OY@$0h1^j<1JT@K5i$@uy9oE%%b{734h#O?^iLxeK5aeX#V$ zaI9OnAf_`;`8rO{Gly~Lw_ya9yH3UV?OoOgp`YYipl3rB+6%Wfg5)!EqL*o(w>^M_ zh$~-)+0!%Xsm$3<3Vto0Ioddwv=LFY84+I}X5Q`h{v4T4qAKY|xCOsiiyckIpr`u$ zT#JZ?{=-!5v>yGA#_Rx@XVuMZqCV`Y@T<19$!fy)H`L;oK9L+H_7PR7r_MIyCr8+i> zZ!lMXt!`8>ak){xp=$d#-y~UNf*+tzr1v_B{0uTrqGR4;w_g~2j;Uj zKzpjGy=EW8--sK-{F2IYu))$tNu2|@TE!Wq4Qf_Vv3%)V5V}x#Zgj1_ETWr6M`cg+~r^ z7Ey?%K|GsV*ml=Y&UrPP-BPo% z%?Qn?5y)frm=O~>s)ECdpz%$f%j1eB`SY;IZsbNznUl#!YvieZvu>>KG?Qe?W#Jy) zxX@`%0Aq6jexJBH#L=?@+~Ne5{dli^*#TA)h|`2rPQB_|Oh-g3$4T6+`4owXS0}l2 zvjn#Nvj;XrWJd2CI36NmFoEa5SC&5zn6Tw@;5iYz`7d5DFz!|EhKB5tp%U5V@d{Yi z@O&JitAl^ab+7)55|pLFP2AIoH$kGUHYH7#H0g#+T+W+k;Q2qLANY%kV^1SW>8GD_ z#xuGsoRsqJ3fqu+p&Ii4}Yk_u~x6#7!V?QEK zyZvl}>H@4Wjxvotv{3I9;R1buO8(5xNlgc4bKOBrf;GM1!}tH7Oa49hc2LQ94h&A= z70-bh#H#q&E!@i>CGm$57Fm*eODu~5RoYQE}h(qMwZCMiXgH;KlE!KsrWkO%&Fwu2vMgj(NE zkji1(Y=MS<{g8=pfch6Lerq2c=vy1_#b~En?gng5FfpFsb1(J4MRED)MQ?pJye?)m z-lyCeFEev^O?gm6&kgF~?IK>LHGZI2_!V+@i)z|WSyX^$W**=pk%B~FbM(1H>gDX4O zEkvwozQT<@TH!)gA2ue;_a-~bw8Tlwmqcf5%Vh26rJ)L=Z%cO$*p-4kUc;7dtv=L@%-dfVyx3M?4 zID-^82(+*<<9h5;yLhnjmoiv7nj)zF^iflDolCIohYw+{)be*fmxC>w`9aZ7%WGkt zBA_l<;;|pD3++Y&vp_w{=_!`-&!JU`D=)(KILm~K(Ulf#r(vvmOD;E$)Gn(2jEpyX zhBO&5uu_#rFyr}g;aFk+cx(Mo37qC26wdZuH%~v{iTHfp6kgrU*Gp(m(6QqSS-fJhL zrn_B(z%jKR^TQV0+hSP0Vf~!#lY*YYEgCBw`!sdY)ga2t7Q?mrnG`o9`Y#B8H!_zg zah?4giny;UW3=#_x;eXuXaY~Wve>Cz*^8~G!+H}!_Q~9RT`AIjjf+PJdiT6J3}bqaG!0mp|nXE=%sKTn0$0)C|>*n%)eI!EZV03q5`Jh^_#mlBcbRiw9f3X7+*5; zotcA&#oaYerrX#dIaEX|14y*NkzpNrXE+-pr@BiWOQ6`>$ZPxJyB)*Dw6?uz@`Z5@ z!^vx=6{0gR%NmjhIdw4_dsW)u{jd%NTqxFX0Bu}^Da1<`GEr{G-Lf(l4v}>Sd}A(oaiaP$9oecS+K)7P+SrEH?H!31Oh?2W)o6wj83(VOT3dJ z^$OJS9bV<@EqXMWrjmPFWcNALI4&{JI|CaC)LlH$F@LYSGKObBo&(qxqK5ka<*AHe z6kB(^%<=MrsSbA7-I=|ahuM@X6VMdx0z;~n8}~{{K%_4(BDj(GR$xaD z;5~bduvcRyrXkQAVOC0OS{=HN@1y}iFKtIPH@+{7lgREgV?ej#6;KK31sAy<)On}t z*>MO(G~zNKsd;gi2y;G1iDpJC`{rmy8{td7(-x zkJ!D8lA?c7iIYXM)G%iT+(%lG<0>zmeW0&X<3H*?%N7&@YC@Ah(&+eLvwF%T#G~@#MO+13S)NQ^_NnuJ>R$cU*eLl;7ctf=|97MQS6*xxE*`z;L1gas z*$6nd7aD3|(AhXuw(~TphattdD-G?QxTYGzk*yb}%1$++%1Z>3Cfo)WAD0avd9QEq@|wS3i!$+2tahnd4f1 zn2dXg8r`p+-V+h4vuGBW$nt0|AG2S6SlCo zG$3CKebL2=&7j$|ZXm^!Fe?{B&WgR>!}ob|hmnCaQ&vn>6oRDaLI#`zvAkHew@z*- zBG_tE%>+kqx}kS+6lIdYJFz zk-L3kIkPlsN!PJ2^OG3;yXwwYcVJ2J#b!|#K?n`pIe`=CaqSHO;1>tvqsUd_lLke(kSn}2hA@*3r^-)+4Y5UKjb|=A2%Lj7d}ni!AitW zOQs}%uXS-40|{Z^e+dbb1o6i#T`)I^U|)608F}?F$f}CX!|$DNT=pkF1JYN30lB9# z7iPDi-2HLVOKGWSRcy{C#zUN-%z9n{FNa2| zEV`SWo(U(+KXfl1asN01+tLCSe&b;Hw&8_lLidU5Y$65b<4CpHkf+kO2b*v~)>p#(&Ijo8OxapVj z-um0ky>C_z-tx$PJG~O0eN9GY09qbI_cuoUUzzbzf%2O+EN!|gUK7fUHd`}(Szpgu zwwbK9e>%A$-37)ju^G%SC^9usg{UzKm#yDzG6op&S25>6d>ePJBY`~=H4p+9Ei@nW z7AyC(VYJ#-xEKyAG`mt0{2@)p-1b^3?K|aP9N(@|C}1;D4)KaU<@X?62s&k6oc+e= z)!oBHcGtQ9R;=!ROG9(SBx>y^SnHLt1^;a-ZP_QPik9$%bfHV zw>o67v>!t4iY;)lS{+TX1g3I_UZ(c=FgBDH@`$lHbF|YzJMGq_MBBdi^Kr1E-Ki+7 zwYPi0{6~g#RR=b(35j!fbynN$S3YTJZ6?Ni=6{wk=N_o`6F(3a6d*~==nZ21lQ;jC z+vtUHvgd#x+iK(4D{7T94Np$s$o#KC)W77-pG5fx_r^J(+KpFxqRYoBe$13H1#tB= z+}u9FNQ{z>BlGKk75wCmHJvsvPW~8|h4nB_=ixlds(&Lo{KQpJtJ3MWU(?tUrR#}d z@XlsA;d3s<0NMs=6{aWDH5PfH{4$d`UPNJX^WEd%v-k)V|G3~!-?IUJWo6v2XLqF94oZg(k2iCzOs9*WkLD2Dl1 zRDBiI`IJ4hE)>g0c0xXYh{w;;V)CWo41 zefTtVV?M2)B?_?Ngbiq6gd#VvboJi{ITgK+k7ouqEUhhv(6^wdd+`T@{K8j1bGT{) zNubD;9MWyw5=3_oE<#icZ|4L46br6P%Mi?La;ut411G*#{1N~0M3S$TG@dsEphN%X zi3O)%XM<*YcU{o5RA-(xY@37P$1*j1lD-PqjwSSs;p0pls-jHLldR&+3B5}$ICU`{8D{L_A-mY}_a z3JF);98ME*VE_T^2>YB%u*88Jy1jbHMdw-b_@$gmXFp4iZB|O#;`%l=nyRywuPm%f z--nTABtm3dPhF9K;>oIbZw`Z#4NoPWOQladdWmhkOp)!wj~f;^{w|3hUqq)HTf$zp z_aN++>U3+C7;@hbBaEpjfZQVg@z)kiWzRkpghM6l2*G3tQ-!rYOgrgqXpv~G_WhnP z_Z&UXXK8b{;efT#I6=xw_?cxfF=8;O;>c~=CqME#GPL88&PN=VUT6K3my+l9Q?4d~ zj7hGvw;~zhOBTK9jFgu~D(KFto9-piS^qq4n5k1C8(!^X#$S<<_byp!JqIqqtZ#XB zDgKVn`l!xtu_&JOeiD`69Nt}$CDF#V(8PN&@Frmb!hQ&eL@?Q5(t8xRL>)d6*wfAd zlTJO~EAn~^7+t}t#u!~A?$thq2;a~*6Y8(kmF0k=oLChmxZaADdKZ!+w2jRPt!LZs z*NfDvGs$S4D&HOPmX(Ey)RZyHEH;!iJrokTL4OxGSX4RwoITuKvYHqLp3EFgb^o9)f10I~_JyawuI+IFHZtsR+Kmm* z!J5%u4KjB0DlG23Men zPcv1pCRpt4iOW;F(h=FzZ`^jnduNkJW;XkHwWal6^;4e(9b`Lv27UX{$=F#P$lJA| zzvHGbis{cstJDYNyDASW)mh*FQnVjXq;T@0mh1jYK+#|GtOEX^bjv(FZlLS)Qw$>V z@wltkqAr2yhU?>4!>O`;j@-u68_Z`}ZYsVl+BH!evNzeMNuXNCVYt{bT+SzxB_7PJ zYK&HtqDzV6gKeD>b?K!d+N`t!vuE`wf)`KgZfzN~89?g)4zYYXeljv&ioK>|c zeZ=1j{dCh5=f#)VJB5=AZ;au~F^)9~9npCfAUkBnIyM%KG8^6(a(VNb%N_X5_orO> zUoqz7=sDqcClv*KSc4s&_@n8W7rTyQMZvm{{F~n|F^|6NvuzBM=*qCyK9C794d=Zy z1((ED_8J{m#$}77EbAqabTDE17G@gmN(?P7RSajYz6+t4Td`BHrszk5Hm!LuvyqB| zg?)KABh@KSs65j>Z@+*tZ_H&S_b8IYp&+T^f3+xG-fPESiYkkvs;Ts;bZBkf&++z6X)UiohxG<C{ z%fENaLYbxv>uX$K`_jI<$={n_n*N&Xa&d1cz434`d)Jh0VH$Lp^=I_|?~|8(9r{Q* z>G5_%S0pZN6l<<4T@((#7o%)e=2~&b+lPLPd}bRSu~}_p#XK;Q%Tx4c)c&usFD$Us zpEwb{N_?~TTV|x8Jo|bbu|`w%O2W?)wuiQF3<>B<@W>1)Mj7AnwDTs@1MG0jD8a-Y zEaha|6SP9JU`P8`X@$9*7ifPdEg}|{T_Vm6uIls4_vV`ndj06E`8!l^|+k8&DnQa!zk2qoNbqe^WY3!Ay72 z{NP)Y3se6ED~zp4X;;04DuY6&TY2m#&_s-ra-A*lg}nd$CUF`7$`&W^xlmQpJye#^HLn>xJ-pUqy}@u!+%lX$16x^=7Jm-k58=Pmxuf zo>E&jTr!-ri-Sp%>0uZ0N2)516G~-{Y3%-2ync~0K@uwryM3^)7O|`u@*7NEY5T#+ zn011ByYvwfQS`1Q0xtoVR@A`&ml7M`XXU3YlHOkMKJ zb6f3lCg~J5!$%ESr=MTxh9g0fn-CJuF*@9PrNH5VB}u5V(%n@Hc~E8so2>t3eNg&g z=@RA^NV&#oyp>X8j*}DbtrThj0nx?J>$*Ql#cmK>^5>K2GP?pnnAxCRvCHG-92G~D ztO5Fl%?Ozj!*^G0=XDwIeeh3d2R$=I17~R*5 zeORNisb-> zI;hj;C0enKhhN>qMDVQx5fEPX!iMDyEh_T(&vPx3rX)F0osN|;<>()7R0AmP>$lq` zp3OdQGjidnKhCPL*fLnQ&~vzHhJGkvskzdk=@X%KPfVM%=(rfW9v}0i2+yzQuRjIp zZh`LQiASwAwluZRpzAH%#K`IozKNW?5Gkm8@+kCuqeL+|Ar(6?U@eG^c#VqHr=0r+ z3!zlhDY9{*uOOeg(pJT)$IR7PEwv$|n>+M_lVzc6^(FZ@oj-w#&9Hj>aDJZ&rrzZz zI1qI-e0HnF?2s)R1-hRf9>EY35Dc4dBSekE6}Dv;JEz#lG9sD zgmH73WPxgn&6CCnr+zPlJq*M|Ao~g@t&Z`N9VJ zM-OM4J#wYL*siKISd90x0lBS12H^KP*eu%KR+ak;aejy44>4_lvN${RG6k{hAl~gx z$5d}JlR$i7@VGo<2!{@_Qihb|Du8>0ZdUZgbaoMLy$KbHs-smgeh!ekicT7R+B(?; zm;5Vh{iS;{f!oC*G;!i9FVNij4<~lSrFFm6L9ggNJrWhsIP8j%?j3z(8_pu|di15C z3>C*v39xYvJQI%+LTi0mlr?L?Mcb`H?@vAWDs`r1+|U$ZBk8zrKi78WZvQ=6@JDa0 zxWrc>s=Qx4e=NsBxD_9**f79)dNY*6SI*dUT56}HMD15~(hmG*TEu;%jw@2-#ph-O z;%+6r95S=HUI!k-vtV^s`@SbeeyXH^<@d-3Pr9}lLGS`gSF19P{WGm(prVg&r%inl zcx_?i;v^uzJ=6DD^tC&Uu6{hx7so#-w}cH@nIp&UOiLw(dx(XCTJ@_Jw|TBN*R(qB zJ#EsL&<=S#Fib6laL}?!{#+a66$rr%pK9SsdpTJmGR1o`6~x(Gc&<;@9p`9wq*+Nz z#SZ2rI5jL5H~&myq7qN7YH}Y1B?MjFw5el@lisIwfiUyW)Va|}U{pfk8UnHzUVHV- z`o*F+7X^zu?TzG4S4{ci^5`!MS6tfp1}rlKPAo&(uBf0mXW)Vb3$UJ^4#qmHOuKC% zOxJ9>{D``>#I>zlX34_jwFp&Vo^f-N?}IGDBV?H>#q{rh<@p`a%8oG+h*P-WL7`P8 z&pL$*U9SVF-#A&!KS3?Py^7o`%QSHHnNG(sFm6mM^3nLw80Y+)&D@uynqI=iaWxJ@ z4t@6_^=m-L1@a&TCH&Qwk(bhp}XM-F8otJ{H2h- zP~i$KD=%zoYf8AltC?nI=U&2jb=n#oVxY;A3+EXBY|8*xBJIR~Grq*3wpc8bnd0A5!;Yvt&%F-&&-8PMPcaUJv-i z6S;Z-e;a3)4_D`WwBBFxaWi8y*!{#@v%cVdvbE{wO+(efiB(TeL36nWtIw}pe{BdT zjtxL+FMpM~P=1)hh6^>s@)V-8=uKzf`o&AV8JQN}^fxs5ByLw}@(tIy7hLE0{_FL zO6r>jV%se87hTFr>t@aE7TY|CkztlH6TT}jF1JAkcEgvNzvKie#wlly{WT`*ETnOb ztENZvG}TrV#*Lv`4Na4~8I07(zKb%H&oPpuL%P%}?^%Q&X#<<}m0>M1wHSXxoUBY` zsjhAD<}9b~wrn~P=x%xV*wtKXeVuOp=cfg7VtqJh7}j`fi_^je((?BrSOZyLx82x8 ze!J$E+H@6|>x+KvtGW$c^38J+Y*0|FDG#v)1zLaNjtqu#`R2hw`>jR4BKeDPl_qRl zd{2kTvf8deq3i-QDajm}2|Pa5w?kAFgn_Hz99-ldsdF%J?~>HC&;itwmgnSQw*H`!vFWm7?t8(+Yvn&$+z(;`?i_it+_=4p0Bm5k? zzBs5)91)X+G6*?kj(&Rm`Nz~*%)Lz3rf(|>C9NnVMkYpB*XhPGj5_^=wkKOjKvSYJOoyRYl=?hEG#$aR7H8>!+%AEF zn1IFhSf}hun8e7VfGflLlj3`aTQe3vV&%;3n(Nc#d@Y>9f~tQi9++mo1ifFTML4|0 ztJB4j`blwq2zSYI4ArTfS!+C%GL}_An%`qG*ATAQTo2!f^>e-U9k}PlS4E-o0j`8| zLG9Y8*H6`*`a^Yf(@nm57IXeK63!gN5C3}2G3X_^5I;v|Fcj>)t^s(Y=hS@fx;{^P z{f~?*Tf@3o_1Jrn`Y!a0&fYKQE3&wIy-m}o!(Q6zwgyo?)qz(FJ`AEZAKO_a#*Pfk z=5e7KUymf5oM-3-zeh{U5eqma>pJZ_lJ}^WRe7}dxKf3$J|ziyp~@4DKoo6p2J{z| zELqH%&fWgGV5!eHaFLNO&B6OQ*&_oTffY~&P)&3-O**DUq=r3wO04IRs@(jNW5l}G ztPsBSC=Po&1BDb|?O!A_$%mp+CqL1m`J=46Gu`uFV(5t_iZLpZW~{eTJs4ya6Qc4< z>o*!9JR+G6B_({Dl! zM@)~+WCiD#QLW+22V%@LwxS{99h*5|zZ>Wvxo=0yu{pGBx(Ifsm&D4%9@KHXhen<0 zk;}uKS#Rot1<;c2Jqabzput@Fd$lbwzbC1`8QUeGrC+l;T3;in`b?7m=Jyz?BIO}! zlq>&Cj$hyA439{C;_-32vEkU;*uA~)%a`8RO9s6$W{ZnaRJkNNLI8;#;M?Btx&-c+ zp=8^aVKgQ;ORfn*}s$62g?+l zcwQA&yza{NIV)mnFNcm$tfI2TnfYDTt4G&0_mTx7rzk^TSf{(t*+ zupCD$r=(2CN9~6+Fm-046^f6P90w*v=|*7%%_edymMX7Z63S{vOwKG={-%|`3RPV` z`;a?cgyBgeT)}=l39TEpVcL?db@85ITqsXfa~F(Wmn_wwd1|^kGFhq_Ussc?dhr}^ z*pFetMQS%q!u3~Ol)Uz&&+ZR*`}GXawsi(Gn z0`rGLZFN(IKJ)T`8du)quiZnY1 zO)`C|vjN46)4SX9TA>}1F#mzsmh`CdN|6p$9L8y@z}Zmih0?TM;oW2li&idQSH;Tu zyNs*uful9@RgJfUc!=>!9$chhW^W+*I6-ttH2k;{v!}}y4;diV*Cjt~pRuc-&R8~V zzT-m|OQl^6rg?sThh(8k@30$W^TaOH4kkQZLNY+S#1~ZwU#)&-|mYU=t~cj8Hd7cK5nzQ zJ5xHFv#(X1inwo)%MGMd3FL|KIbAI84mkwQ|vVdQuM8mCuPR^IWENwicA> zQ2cr%S9T;(Zq6YlCX})?=h?`RSu2{+#Q5t)Q-X?!G!Wo`%^C1>>ltIjU&9h>4;b@c z8ui1i0TvIMr)gx!uS|;5b|OF6e=``W-j%$>A@g|ex++9sbNM1(0S(n0FvR6xSNoSZ z#w+2XXj&tk6bPgB){++*7i*C44WIDAA}sh3Up=&2HS?Y;KS=eu3y?Omsvip{t-`5EQ|31Kvu0|gq1B2=_%g)hDwcFp!t>KqMC5lElPK%mqSCE}u%r2f zU8_(>b68fJ(|cz>tJ#D~F}I!pH_zj=pH9^S+zlzHaxBT6C3rPWwZ{f05Zva!$|W^% zSV;-js+;QTj8qisr2jIOAT2GsFW~ITqi{jAg3Vm?#fF-&4nMo0^rtZR8u*asKVX8t ztfe4^5o@v3{VkB#Lva!q&uRB>y`{puAIg_=9%2QU_3V?jb{#(e)w+zv94EtzT1%RmL(EY`;qXG5&zbj%jT=trX3^%61003=-J+9RGL}Q(DhQMd*AEKp{d$iQmul+;JZhQ%d)tl)z#0A`rPU23d)pWjDV&Z$sO0}lg&8RkRWRAA1-Y=QeB!r2k&wUu0< zh+?Xf+y;6767tLAxGnh!qBBkoiC5H835TnQ)w&c7_}^%%Z%CjNx{)t+!q`oH%S0HI zDn#XHqRAT4`{?0(1Hu0azLkCU<7U^F>$|wtOPzQjf;@4<7;#nNPBRg11ocYwQ<-!; z9)Tm)-LECsR6HbToSXxn=yCdZFXCP^*1^P*eVS!_q}DN1{MR1|Ff7ERWr|s01m7t! zE1BXSo6G;3)0R!JtN@nFCN=5v4>F`k}_K&cZ$-IpiMGIa@oDtI!)f!fhf&gYJE z00$y!v~wV#b5nO*1(TO~4kT?7z7(DVDi8um=P%yh>-uXp2>c&g<(vZpHiv?n5S#(A zGU&4>!hZ4`$Vog0KsNZKfR{xF;++IykvL+t@;B2VJ^m^YI9`)isdYk(4V(i(l>}|& zW}{zh#4LAU&1%=aBEBHqi557+{260=JL zqo*)|$=vl{&k9r9IM-@EXT!HHJdStHt9nMk6$viCW>@#?*{ZB(CeEMO;Ts~)_EKZM z7Iyl$4&<98`ThIO6PLz%=}FaBvv{Z->p7&_Ep%J|M|1WA$v1q78KIg8}jjf## zDzYF$N7hx)#K~-cqK<}hok=nIv$t}0NMQ<$Kq`IOyi~%8aeQT+Wk84QCacrm(l}uG z1qAad9Q<*iwkF7D^U{%3@v59os)~2+G>xsfqlc%Fh!N|jm$~XpnnK~xZCfdT6ps={ zpKT|agDGl_r?S9Qv=q=J=J(5M5|z3J?Fcy~#9p6h;C^l|kTI>9Ay6QA9<-7DYq|DQ z(CZJP4hlKMlO+6V^drpt@|Bu+1Ff{^ zO5$}nu#-b%bF6f;Z>(7gYToC*AUA<2G4kio>?CD*csfLZR*B$U3dLK7IFc9|5o4*- zq*b0AWNs}nx>d50pd(-+$q3_ln-zWsJ>JN-Ut4|4UYPHt=Sc#Oy^-hchlATmrFbR( zL0kPQ+5K4=j~6_TB=F*Ek8`n~zKDW{&I_8`2ewc-jKgGHTh9}oqO~zT-9UXo_OqW5 zGTjH*-5d|dp&c_osgU$m`d!px6DxQCJR3Vm3i5HcFxuE7Wstb{h-ZV>$K;p^NQtBGQMC#oMOF zT0w5fbP(FP^+RE-sVj=B=_`mzWKP8iv0vk3kW&!JuvzwHo?rgH;#!iOiTy2x@-u#v z;MEyCI2SvWS6DM3W-6Ob&2Jlj7&_uhabAj%fHHAZKj`|1iw1%C)!J~DGXYA>N6Afy zz{HU^#8LS$DOABpV8KA0|H!ghc^pF_Nn^sbXSp3p1TKsRDdcxi6+vv8pAi)HPCUK4 zX6ak^7I@7&VlK{C-i}H%)UY;s@I|1-kBa6dfxh3xzOXE3e~UUP3l6~fLT$Qcr|%$k zRrG}C^~;%Aq|sqo{IP}4jo>#u?2c*AWCJ$c-N0hLg^^01Da!^rSQ_`MWeHwFN7XqV zfX7!BDQ+HHusr^&fBi602x-jVE38Wc+Gv`lgPAWO=HhWNjq+X4yf5nICf#haKg|1V z(G@tVbuD2D)1fj86>YRro(_AEh`TwW)L}<^4%WfpN3b7~eDCh{bYd1x&2zf^HsSbD zLG|M6)2yX~r8DEC&Ej4X`Zb}_j%T`x-rl&<3^lmQ+WPt=TtfCz`#p)tSpdxHIOk`> z5zxU^2NM3nSI+jZ2y5}w*9-y1iVPsnoDM! zo>-45+lroNrafg_xO(;^5l2KA%?P7S6pGQ75Q?UD(@O+>ZV!j@Vx^3u=I@3A1E^)- z=|!(s&BoNs*`pV~g0u(eVXyfa<344OsfY&=0kn)*pGs^D8_{S$}8jgMyQ+9Ipgi|pBttcY|p z#$N+401)bt)U^x14fyQA9`By}C28SP)6m@lv3!GF??d?ucFVgpD96yJJGE7+ zi)uAW8t1^DsV?PE-7>%b`7Q^m(QrZ8X6cAYIr@hAq}qTMO*SjH8iS1ugi4tsV8jp7 zj-zo;qIPvf>9&nd6JM6KplXn!q)dXZyo-J@!FNm-GeMCKpeQ5|Z*)o1<1}=8V=0=v z_I8ANqglQ|9OF{@`#LdAdPEGr2nE>SHHgT0+?B5N%qG4y(PBEJct`z$ zMLUacbr455l?F#Y*gOrUi?N^QM;eyW`kx18GkUOc*T{PpzExK?7 z!!ug!3c*=ZOeiqjZjL^zKZhO-E-fXHFY>s^!EjV_8nnoBk_iVZ^pu;?M>%12Jx;m% zAV|k29b%KrY(K5%Q%a_U6R$~UFFwx3^j!zBD_Ec42enT%-8vJlZ^DxN{ba2$ceX#>#zkb! znc&V;1C<;j(Cf%3zkeNYpt8pp0j1piH(x<*c0i#SboC@oab+A%iAtjikisP0fHvbvYQcdn#Pq`VLIVI}4<7Dd2bPn8kIZqe zC}#z47PRknzq=bPy#W{WVmB}c4`P+#2@Hla`!^w#HoWFzF{a$% zP;)>A1-v# z$8@_Co5?vHYGvg26x9$G_TZa{3*<`0@>ZCi9KneMoCcS*{k!)AQFqMF55ulWjvE$a zuX{Q=ITN)W7u3JX*0JT8OuBrzm2xZg(_1HAgo{xxBWOAuQ*;=KA#c zn&&SsvY<)>s2sn7%8mxQfa(#sk2I52c^O(LvnI#61q{rJM;1gjmeOg8sO}vbgNHfn zKm|Gl(^^O-!bmS@mZEK+?!IALu&fW${+L=<8_6iTwRkg!>~ss`jW~7zfA794n6y=$ zE})&GFe7KRTIxl0TyeJf^-m2o8I@HDw<5*YdRjoxQoLn1m@g``0+MLelt&7Ta*v)$ zMxRYd@s(qwa^oAi5q2FlNlPtR=h@bBMu`f5HgeJIEtf=a@(%lt0s~;(vzt7NOq@|$ z7Bzv-4YphISO{-R&-pok_DDGVM@TlEtRoq$m04WdQjKWHh(InNmRo+@zcL2vN-Rlb zsQ5XPkg0SuWwYXGhscH?$0@pTH)RG{k>G4zKSi;6ajfsjYZ&FR?+TJ~6Ap%5U?+vh zZO$QS6|tOVu0cANj9W(Yr%mubsIb^_#GaYL!_>I2F*jYrRB`t2Wg6K`M>iM6EK!CNoc;W@rj_?@9rloT`8JqNN&5$ z$qIM2;3_{?rGMAj)U5%@Rg-5<_hj{r>TTlYvJ%X*#T7)|YD61d=c zwE#bnB!WtQ{pcD%G|*Fsa5eh)8YQ*+@@jfNtG`k@C>SQS5n}kV5?{z`e6=NR{0kT4 zB6sl$l-Rh?HFJsqLhd!&A!bAc9|8(=b5=_nbTe*9l`>VyyyKTCUq5~o-!`eea#t@! zFuMa9r(YEAzlTj|e<^Rh1Z2VHll-I6F}GxSmt#{P?}@NG0>yA(^QP@cq+Zn`g1jEj zOR$Lz9IQ12IX4#q(G?CHj@_@yTpbV|JUniGS1F#K9t)#>G|Iz5!#`N^^z#<7XC)`} z#j^WU4TFpo36$F_s9n7RO*<3hl!2;`a89-Fy$#u@$QIo_4lU#*r`oyND<}4>=p})? z4ccGA>q?2{m2p8)1p{QNcYg*^5AW@QO#Nu_hpNfZrYN>Mr5=Q`BDmpGh_gU1(yBwHz5jl@H)!pB6u6T)~7& zZpGtX>>~WJ>iQYk=?92-tUQ*YrZt(2zAZ@*&6b*fqn$0G!0W-)j|GeVYUVXs-R60D zPpYnpnFFc4yT{?QWae}_keC0h7IV@>m z(qck4-kO=WxPFuwZC|}@R8_Azxbf0n{m20PxPj1*PC2|npker$&Zj!(JT2oHT?^#d z)Oq`g%&@DW8!_vLNyVryjZE==Se2X*;@9ImA7V31$6wk++k|d%h`kUOvwoMY&)j== zjYQFFZ=kta-!~sgxn9+$pZnUx7jYH|L_lGr0dp|m^~aL>Ydom(y+L?qn3~V@+6W!B zy<)o2K-EdW+@Tarkax}o64BKi-NE&={VcOy4vrD7JrZ`Zp)+aGaRXx4k)1b$ub$i5-sPLI&K?`31yppHE1NCB&cT*AP+YUkuTQk{vzYTtHd1 zemuOjy&sFoxEO97BzY0`L9=tGbVvc+c9Ec72P6saOH}yCu(u7&8EN*6Zjll!Xsxp> zC~R2bgB6I0K&FF;yEBg;d2BpNN0LiMnCm?WVBN1Q%Ck7fI^x(Cr{vpw*H3}-d|HrX z5W~pD1-DKI#=Qm}>tE z0RPs#6d|-1a(2X#kwL}vaW{mTD=G9oVEmknCYV?EhjPMrMx0@+^CY6<&3c39>dWqU zi3;(^^q-*}E=|2#{+#8|?X9$I>mzTO;|(a{Nb6z=`++?SDd{Z(c-Q$N`M*FK0Vg~4 z^8j4b{!4ks2Gn$7fW_VSdf=ZpL?@Z&2l#{>*%sCTF#qZuslM=tcf*R!qMfg-r`A=5 z;kF}XhpT(3e*z`HV=LfTuPEHr6_Zi`$>SezV?Onvs& zz{JzTjf?v!lz{+9pF%J{vnh@?nahku1!$q$gKA~xG}ukUS`KjJXr8Cad5}31eEG9- zn$#0*s~pOu9U9gne$clk$v6oN^ITNZiD=^t16}s!^S&z>I?GwmjPR z%@tSQHB<>vAxXeDozI@_xyo~-6rJhT<;U}N(o-#!~B|&3vPx)3YeD!e$$oeGNT>&NpP}nx}NH%Tt zZH}Qb-m*dS*R!kpaGjFu706KZJK4M3#h^_HA20)k;+4{|5uu=u=O%0zs8;3fNr!-s ziv^y3pYIZKhzT@Md2s+Jsdf4jU==#5?@tLSz>u9)Q{Is;RoXSJi!N~okYc) z$h+z1tII&w;>{ux?yn$3))#C-W5QQZb>Grq+LBf2@#xBARtJ!Cd{>qLj9Rzu+r-lT z;QOo+y8A8vWKBoveFl~#_@0H59|3a`VD*CUY3i>}k%I6*fs}!{w~+!h}33>N&oT6o-!eIE?)^-26;HqhS0yFaOh%e!6qH)^R~WvC-@v{n?3L z-)sjH{AP_oVf22))!nwOax}iuhmJ+hyXUwfWx3t5!y5<)nia{mL_G4G*q7v5t0nK z^2X~nVDSwsUV>F+w`@i9b^2~&o7YMKR=GoQ&KL`rjlT7mi^9^F0~RqfzQ0|BGPfUK zu>pD{&>Yo`3mpwoL|tzTb4V`z+PtfnM( zY=kM?Vu<$S%)qGlJH6jHmfu4dZ^@GDE9iL^#v5n?0Ki$RBUjYY4^F**O3!~a{^KnH zmV%Aw?TA_x^j*whoTrJ2B^|2kluA2oM(MQ0X4m77np^Yke0)H+O>{)>GfKS_G6X%Q zqInu0_02_x&Vk)}pw!${)VUCr72g?1Y+A0Czv(A;o^RZbT&s1O?IT1z%}Yyf2vT6) za?0(<1NmZexu7~v`yr7)jnNRc#TIhcPx?H8pe9-^u+X7;Iy7U(`=Fk+I=q9Mzu@P{iNK91+h7@a)Jm&)TbxilrhZ`SIQ z*(!=&eSw#u#6lScmBmVP)Yi{vY{)WP8Z{X~ zQz^uqEN*QBSDvoS<_VGofwQ?PQz#`cY9BEwH_+^jjtQolwc`7}>dn2}jgu=6@9J`% zoh0>6sjAR(gS{;1tD!Z|AR%{mY6z@Jo0PY1F5Qxt4JDzT{?OdV2HuGvZY- zwBFt4JIHKo`I4$qD0NdPWr5XSxC$``v!X4O#2Cwd3rD8*VCT(y!rf24EZdpi@p*-( zB~8!mC(%cPH=A5)65D#&ygVTI7gEUT`L@>zq+>Vjn*P zJCMI~tA9J3@c7V%kRU#}f!(3Z9!G~|N5jKr4L{aG>J7_`? z^V(N$$GEx`C%jO--yM^E1u(_Wz=~lW^EIRPtte`hZ5B@9SnR5W4h}3CD`lL>BWunN z3ymx3mDDHEQlE1Pav$UL)MEhoo;&8p03w>^{Uq_Kpe|pT=A)oIf>kpIjNC%gz8Fn; z#NC*|t5r#+a}i_Y6C)fSxg)bxwr^4@47!sOFEk*4Rgu{35NEUERfk|EPB~UyB0zS9 z4jRYcloiLh1`&f-x;c9~(6|jmw|2f?KiC@eo;{>*!73oFz0VWJ_a5U52yB`(4v5IQ zSt`tIKN+ugAHw^m4Ok*&R3B7?_k#!A0b=u|!&9!4OswR{Tw^7Qw^9U6H4dv&(K?1M zuSb8LoO6x5exjrR{g0tq3~N6xixr$Z^mt_fmH!mjT2ShuDI3U`RT)WT+Ek@SmT=MF zW3^Z*9cOOlRK=H1GD~A^btY;Q_EyfVV?YY|qmbv(ACx?9C;t@<(*M(%p#KA!t^Zq} z7jZ>P5$(9H%}Lhm38%bc-mw;+msF=sH01CjRU^}|%FPZ6849qOez*NjXWRW%8PvWB z?&kH*DL&dfE%??{lv)wT?oWG+e&#Cn57{_TieMlHjfu!mIN4Obi*%STv@A7y3KyMb zrOVx6@{q96Wt48o@J$X4nd|35C53M`QW*ty<5nT1KHHS*aa1$*)o)76AI6o4tStMh z@^jyl&l99lPfa0pZHrbbzMOguWOS_`uV0TK`y22g@M-mt=NqY|)`%Bkn%TPhd6^MP z5;{xiJ`=;`YGKK$G$7TC)&4aTrWRwpg1t#`^F9|E?-QQ)<^B5w^BM1rOXL+CJT1iluoR967>B6YF9}Fl6^D z@Iw%esrJP1Obi-*q{JCIts?;!EN{!0ub_{noe?Lrhxg=vSrzkB0ZsU( z!$J>}{-gDAMcixQ71vTe8~{xSovls#rd6t4s()=*e{Zm?!A4?yWqR3iWzR-Bx zky0HcdpK`Dcg%wN(3QB$4|U!vyZA-Pja)0$q*J@n&Pfw@K3$0fb_UJV*D7Pe#x&x- z>P|_w8XYRdO0jw%m+2xRyx2f*oDW}I5l@b8am$b909*eFhha0%_6kBoh(4H8B$&2n zp52}{TTkLEjZKvaAhPV%PH-`e5)@vvy{F>tQkDE*0`$l@Icv;bnr_@O-p1M1#`d)N zgt>WJaVwcd>a{a8pj@DzN(=$}Ka#%^}RrqTg0RP1KobWKCPcHcBj*Y>_kpVUXb%bxy;LwUX~mJ8}P_C`yiPDK4P z)19t4@r2Aj|iU_&?YG{q$S^=}rpgwi!y8AR2#UM3ic*s(LU( zhUtEqqoDcp>R@z~^tAiA`CAv|uQF;Hlcnc?76V7y0ENXX>za3A^YGx)GDj9Qjait- zpiaD5PI-G0P8bRD1Rq*PJ?xvN!koL&(KKEAqm z9t+UvM_Av1TCET$6~@XOjg84;*e)G|_N7Hu3TO9~gKtD$Cbk6yyz{>Rl4%xYfTmo3GJ$r1aB*a|yO&=uUr|vJaVv=7 z8K>G)vE1B?K;#f)fV6&(PnQ{{$eG{@oz-l6+pi!2y1kVp#ckJJK(_V)f;g@@gxl@( zeWx@>hjxS69B2qNDp9#<;7J`z`$uCFBDq|)NX8DK*bU0PC;*S066 zOL&zDV+D*1bJfW0j`k<8O;iuUVpcSgSs=1@N8^|v*^R3S^NbKhd-8mVnaMS3o>CK{ zG=GwDgTUxeBja%{p{sI=PPFj*kyG0lts4_%H8);0BbaI>=4Y7MNxWbdmY0rb%WH_U zSWDYR>@s66k8$(AJF|3fZ}I(tn;4-$+k?d@$4Q=3YLLFV$n|e%Wf_t``EOr;i;gSu z+5r7S3m-uF_Hji}0A0oisDZ4W3%sh+I4bI8*BEVOzw%?|{VlKm>oCS|0|bp!FTo|W z^ed5wc^$fD<9ErSPR<1}UREcIE`~TxKgbHMCmAekC>prSs@piHc_*sLY!4TB`gzph zZ5zpJXBuWx*@i$1C>7I+3y3@Les-GDz2|RF>YKjUa-i;Sd{!v#1Y8mqur{co{ony9 zW(CJ4WGP=1%;4?BWmVjho2bpW4Eg)qn$B+S!+{^flC+cb6n*+`IC0Qo&!u7|&|7aI z!iCs-m;%0#1js5@wg8i~t~5+*axN;`cHh?P2)83VAl69MrOd$=oeJ+uCW0vBv=@GY zNezg2geaX^tA4>gPQM<0o8E+1vxahEO+tw+@b{0H23U{s+(`;H^=KUS3^7gQL(i<@SlazXX2Yn3Nj*blO%;s5x zcNLB&I!s41HoGMZSj=3zxc_m+2Q10HWPRK<+#zK$kVu4@Po}Lzd)WZ{{E~Gm>n_;d zFN|-cZ$6-uprF@JY3+ozW_`r~VDUjdybjAhA{D>W@DkF`83uH@mcO=NT<9v{8W;6 z1`G<+P%V}2ec-ht+kJ)jKp3g(T#e5sp=~F!ic8x%ag&{OoLnx09UkT0Mq*cbCPrD% z%|iutQg_ns=jB%1pAubj5_p&8&F53I3s??9k*+f4?sVTLJEcv~$vo72teGX8 zKRV;Qb0H9rDmc62){{)Ku=toTYiHuD9IwbetC7-eDhGNOkM+@mjed@pud zGN>3q$(#I0c=82D%@e-5v1?n$w$UGwLQs-cQn+lM3GI>9uk0j%xtaU@U%*RDWPye*!Z8P?7(Z z&i|O>t-X*9^`w}`09$zrv(S`yweA*!#ybb;P`S2+v`m!e#WhBkh8+?~ycYJy)j|G` z;Y_ew=)S=DOQu5iMQ@L6MCraLeEVEGM7&8wQpufxMUNn{d;iTxCfyQD@LbkY&gCxi z2$5se7i1{5%ZZ!U7b+vvk2(FAU>3)kW-0W^gfLq7+1GCr+e_r4+ep5Yjz#K1Joh9) z$*3eOoR9Pvy1Fn`YYSpeC-!5hKYx<3HQb79ECPK(??`P&-46wG<+f&2-1`cmcW(1L zyOCVRmYI6B4zOK&^1QMzHrtV#Qu$Tb)kizNl__}jv{=It{3sguKs{HWYbXb5IwEtm zShI6&Q00MK)B%q{yBUam-xi~p^9CpRanG4q93wp zzd(%UTBZOhO>y_Vz+!vbm>F^o5nWGPVF~0bXh%_R-}~R#v!4=S|I(dbipI${F zUz|A+d|Fw;Fab8fitfmXxz%%idms}g8ZGiSModEV-@MxY1gy)z904wVQ Vz>?zc%Txc*@cKXb(%`S3{ufzeZlwSK literal 23737 zcmeFZbyQrmR_w!NJDC$0H!X$HT`bc=Y%Y!9${l`1piCLZZh{o&cW^ zJR&9`enRr+`pG|yVEj`P6ASN8#U~H(AO3mzKMD6=06;vz5ym|x1_jxbFeb z{qw4r{{;d66>za}@USo*VB-HN)_4N=)A54`IM{f99%5r*0v=#sVgaymaDjNlB$N#L z>64@&K9L4|GIAL`pNwe=Mt;pOB$I%!xtn)tn&1mr3uY~C_aqjfSD!uvhyH1h`cHQc z{`B`3U}8PM#`))6c!7V0>;Wb=E++PW`@x@EEFc-Nrq-u%Y*Ge30k>Kb!CxR5bN3|K z`vt(GKMg$qVgdnDfIUen5vw)iJ)qk;)y$<;(!Q#1%dK-?Yg;uV>Wl9sZxcqc+bk_h zYV1Iok`m+(=#~d)H|ifw7Ml{Hd**Yli`Vb!g+Q0p#7bRj zmr9jeX`GK8DwY3F(f=8Z|MPA9`+o-yy&;M2nGbIkDSyywhuH;w+_)$+6TEuSHyhg( z#UAhfSz1!T$Vt`r{eYhJ0kMv0ui;_EmRZ`0m0|-H9}- zIS{{5$h~A{m;T?rNT%((&1WmI@`qfHsvn~)GmJH!sgOpVUw1BUw2it%2%6yL#L=cq zXM+l=&e?b!3Z{iV(Q@4bMjJl<#7NkxWH$LuTFmx8BH2&}hg4%;$vpu3cMr#p|B0#7 z;nelKz?gzF)i3v!%}Gk78R>VP;6q=vp{a7ZU1D$E%8f(U7+%Q;kTR*A^J`Q&zWI^c z)$g6l@eA5Am|rnbZ1NYU{{Kf&)7^N{l|k{1_L^CtgkUcpZ_epGfctbM^YeJVpPqtS zf3sx9Cybq~-NzZQfTm#=hgs>@?8<-eH$+BGO9!msP=!0nxkg4NEGH*-HPLQIHdG^p zggV~?M8Dh_ARtRGV$@P}=*E4H%f>yVgO^4tRg;8_(`-9@RNNuj_y{6a;A1FZv5 zuOpwLg-}5`t3h1zY(gAbHBIrB32#JBb4uCMPaU1K?NNl8)kJlvPluKae;rZSb#`24)Lg0MJnm~Hu-5f7^!8UM#?xWqS-1D@UzC( z?E3x3DqQc9=|QKn_Buu20m%>QsIQ{EcPPI6qnr5g;4gv83NJR$+e|JR4^UZxZx2w;{1G+IPXD zi4whhHS#6mS;hV<^hgT|NGKmGSv(BDkV3*ufqlyJye-q9JzM-W{3Hxp~%_*XSUSK+5d=|`W-j2^e4>m!;J z2RLB~#o|3c1wNOf=jN!^b@C!Mc$~#`*jXIwd;LN!dzd{$MpMamvO&UhY^CI{+UPU! zhiVFwPSqN^TPJ>2=LX0imG6FL3a8zJvQN#SCy~A}x44q6i*G$I8!(y~x{$+Y*QtnU_C%`@s&3G*aG1J5tRmE^W&X zECMyLJkd*d@IZ$#vM0*5eS z;-UotFo`MixStpY8x$o8m-5Pfleh0%U{pv`s?|o?X4_Zz&`Dnf6fb;C*x*&Ft`R1{k`a#)+;` zwAxEFWX~yj(>#TRzIBqO(2#_;4i3iZyDSts<_4#$F58NYjR~o$CV44L7`#_LEg)Yf z-8{`pC}1TCOeof@QsG_mfBkG1dzRGyg(VMaO<(S~_k#h00Ws->yIkAAXQn3BXI-!I zyG|cSKYDN)Y7^b!9#*O5h3G98u8bF=Y}bf0$$u5+iD!U?5xw8y8763Bq{tHZiNuBj zCx6dMTx5=;r_)!A^0|yCLTy)&2TK7<2J3$ z#(=c_Tck3!@2XEtbC^xECFW-+e#y(GC&TRjf;NkbY zsYUiG1BBm91SsXNw4F5Hnh6SISJPQIVC73`?oNRwut61 z>!>9Ofx*&KIQM`adaI7#L}ccZd%gw7!-GZd_T0|BgJeQiYozoXMwoBcq{n07zuH)Q zstuN=@yQ72iU(#y3Y(!nxP-#Bf^Lh*mtatc@@wV(aBfO3-)i!QZMP6IGWSR6J3cF( z^FF6dM0&&TIDU`=SGjMh5iklL4InpdvB~;Z&Oo)wTjx6thX?`K=t=+EQ=y zU>c+t=HJnUTPCgdA(Y&&JqKLk6M0pg8B=o2rVB=s`3 z1>pLECEORH&kb<%pNz(9G4i__#V*k7Cnpn>i79t^Iui{Qe(I^|L=cuCjF`7|F{Jq3 zPv{?ra8HcWts?8$~g(H7%k=t(wrL(E-`ho+&O_Zm3#@Yn`wR z{C*75!qzGpLDZJ3ZzAsb2M~W?4wLUjybVITv26WlngDsvja6VZTJBz9G4DThd>6p3c zlv%mAnFBLc3dgsxwy2d@kPr9J zrbyNbN34Wh#923Ze56XIi4A?%G`0TOM>}KTnQ>X_OP36UP-}^(Mzezt$vG6fOeMZ`G#4vFo|vpSs1!R#;b1Y1Htl! zuUBtGXUS@WGS*}nh;eLD2iJ--QWa(q8cnyNyd`u}rZsc`o8BZicoIsuz7c)?lK_0|lFcn<1>%jQ@J z)Dyf)Fzj0pZo8{hr0{)RPWb7mrS8=WXs=mUf7RuyZdQI6{DXa1R(ChE$sx-hCm6Vg zVHh>Yy46o%^vuaPbx7e=GW-1D-MSy_hGOGM zWkqFLpiYpRgxcN1qevYxQ)F?RnqV8CH-hlvzb_*3&l0ynvN$GhjbQrA&d{vSpgh+-ToM7W%-| zInn)LThScy<;Hk#V;!A!TaGI@PDh4JN!kZu5=Pu`Q>yRY$9pMJ2<)W!XxA&6Z}L;! zI1FXJNQ5`+lFWd+H&4EtF!59wEBeQ?5Eya%8%n2BX%0h(XTPI*-k?|SMgr;{CLg=K)-T}LmEY{G9fu0Bx>OLz4+JH#SwJ!-?nbVDZ)F)upHOz8)C4>=)&54byB1d#IsEl|o zh^kr@3FEIn$3}s&++&=P3oM%)^EEF084M4;)5jv9has;`wPfP29f}qN_XfwRCRz5d z4-WYIJ>FSrFnw*cLKJq!S!IcsSMoN3ncy$;72_fph?DU-f}KIiWe>Mvy0+tF8k+3G z>^h#m98c#;KdvkrL56yI>N}98rcBD(;k;Undk&3Y9K+>h=g>WK>8y2xqz@-_Hk`V; zUC?+hFSM_vI0F(tB_*+osEjEWIVdYD%QeGKfg$VdcAJ+x;uIPn<95dc5AO-&VptQA znm5+>UwIa?g-Z0C;rofNNdqQi7gIpUS=R8oQ99DH>li7p;3+jjCo@oP#E8mbl%DCD;oitDL%- zYM4WuPfL`mN(C%JIhr2%nATq>=VBtk#3niII})xaQ}4RCE* zgm8~cA$>eA9tUnxL_ldL(9I}+3JHeg!K$89X&+~33+0)p_tw?4Xp0@6hOIz1#|Vve zx+!AqJe)Q_CP@FFr?Y?2TJJ7iahI|!w>>%L%zHHdh!-}RV%MvoU@14kpLxFf`?l#O ziJYX?T0Vn8*|;WFj;=2pEA%;zsT{~QG44@X3+-p231m+6lB3<5wiM38(WE^Q1%^KC zywt=D)Ek^$5>N0GQne%wx42)qMQ`sIX(-t)v7)$M00Q4$X?oM7umVe|ZO5%Y7bgKf zQEcWTh6@%RMMOYjQG|v&y(%*iMOGO`v}Ngv`_8 ziG$#K9qfnV*u#;JzCMg&@{&^29Tnt8i6(}l1Dx0BH(10q7cFRrIMt|IuOwFsNGxmn zyhw6mg7z?)rnok$%4#pEnL?PEWprk( zevOAe$}uEE%^>Cw&xcvD*LE{v(z>?0*`|agB#lE{v5RE(ncQ@YoUn~Zb)KE>l{Aiy zw@5U5P`0%n@ax!n+>*s|!i`07kIvlSrkr&NR14ls@8E`Q|t%H4d67u);5{FcYVemT?q1 zi#zm?_)~CKt17Cp)+(PV@++5~(Qc+kfn#o|==-$VMLmZvQr+Y+!`_Z5M=h~m8jY5a z9rU^~Zh=yawaEh2m@HSw#T0W7T30eC{msF;O~w6M7D7^-|)GsJ8)x!`#e&Stxp9jRiGrVIe-pACM8Cc#w(~V&HU)6aZ3h#yBiyGCT(fw#kidW#B9l+f|8+_vzfF= z8|`f?lKKmgFk*T&M|)NUw)I!!2N+7&EWNKFnp-dAm#h`zLY{4F;p%%WEi@3SbTMt~ z$}iP?b(l>x$T9eR`>j(6qzcplyOCH6(UR%rV(Dl(jRL_c+rEc>GOew7$4FlJBoun&P~V+PjudGRq&BUmvI!^hR~%Uu$b`*Z~`f2po$ zKv_XSbTo_udZ2u*^M%OluId$0ll|;M zrxg1RI^>ddKmC|CQ)6!;&`Qi%CBW2|CjD9sNb9}$oHSfv4D8I|>&0s7{*Ez{>g`_B zq_uqxHGP_ftX6vc29~J!SfK}m-URhhyI2nf#)l7#O{(~I9Eb(LGszF)? zD2B4Lbh%;7K~d0Zt`Sdpp_(Jb1_yQx?n&{~G5qR|E2T7)gNBZxoL8Dd;_D|3&oH%3 zBf>Wp_NTfU%`7o91~_dRM-za15`9WS^s!yt>ITmf6{h_&62IiSCMLafgkOEMXxQj1 z6WhUk8kS;Iv$yq?F$Ma83dXji-|VOAO*V7I;FyPrH$R|&M>wI)Ncce$}cQ<}ht!O!}D4%2h*jz{UjNVO%dvaz4I!=ZeyR~lL1}~>ERvMf5IT}*X_C!u@mgOrg!qAH_(HvDA|B(y1he*w8LoxV?sgiO z%3#$ws`_}nr`4LPiJI%}lSSM#R?HM|q3}ed69AU4=AlTJn55u;I$Tx4^j$Zqj?x)2 zXiOPP#VYV)?u42k!2hwsJ;2K!58Mk5BcrI*)ULG?<)5rH?0<`THA13kc}18f`GUcs z6D9A?eU^2xTcfl?FVxgz963pZuke~E#S)t(4Qtnrv5fjd);1MAZz@&(?lE2X{#75& z#FnQh&IT__S5C+8w_O3RCDaZHBxlZ5)c!3>^0r#9QeD(GH*p32NE&IxD9&$%8pSBv z7vfMCv1*G1u~k%_fnNLDn;L~p%-bV7-Zn2<%6=d#2)NXuBcHb~QkxnX)Wp&hhd{iQ z&;HGe|N37vW!>U`;XD84ss86&ey9B>;ASg4Z=mk@=_)AEwB*=Nt#3EBD}((``U#Pb zu~Tf5G?up4J`k+lqnxi544D=Wi$4&9tfGBV)XNvSeC#KxanP;V6%+s47tLmGxuXnL zrMy}e8D+vq$!0ePJq_*iHSB@O1q7@=A^AP#hW_t&tIE;jIjg_#IH7?uD&BO-b0g%m z??S=3Eb~CJ7f&#>bXMMwUL_+5(s+SyF+NV?=}eC=DEku_>dkK9#6bbrT4`LyKh53<@jEYI zKr>hPPXsJR=2MtS>a1ll=s6uK(eo3mb=P~VZzFrT@kh(`+x1GJp!}TH>;>*G_ImdK zt#=-NjP~;f!bwu@-yocWJ`KHX3nlXq9?^H<6w~!K!R^MOm!o-@Hh@i(3`8_Gst3o&3@vFrFH@@XDwx?d!ha1G{|MKh zb`;#^pySIkBFyEUXV4@LjDC1D-Mi!+D8|khAX;g`w2JQp%k};t(WCA_u&?-gCJQ~P zoyb~`Lf7It-*in@fn#^-<~D2h@9suXgkNg8Uq)Yh>lX+(J3mBm<`lfj^-K&IiXZe1 zu+?zxS*g)BCk28@=f5)t<+H%M!g3uBORVB6Dn9Zk9;6lD0~%=Ri!`@17LA9>ua!+I z3f3GWX1@ixPoO2{!i4mAmY6%c4pwGFv8+s2s&p$-^V6Ra+z3)48S@PTC+c6mSKX*J zXbB`0^z!2!`q zB}PmHDD}@4O&+}KK!T!%F*)D&9#eA!I4FHnHMwis;t1SSU%J!NI(v%m3Y#_&^tyKR zuh`U&%I#x&;C(U>s>nZQ(q0m-SitgQ-JheVA`(`yA0?*)hCt?tLpt9KIEk8Mv&hAo z<}X!gqZKSh!=9O$|2p@;2rf%@j?Rra)0Hh1rN`i;Y#yFcV@K7IpI*msoY-1pl+9B+ zb2TQDg!MlDWLI;s+^Uw_O-YVaCvC9_cmOgw{&$vqG2gKgURo>6&_bpK$ zoMdHX%c7%Gsj~l9gsoOQC{tM1g&T$TxSNkSQvO7SFC!Na#6lF?yjz8?v6XzI#W`<;x*D=kn=SX{t$1i@4Eu(EzYhd2qIBxtN-Ha}p zYv+ojPRuXR73|P+gBq(Q-6%EnKBjHu&M6+SEOc=56CfrATSJ!m@R)EJVIXSO=O+)M zD0Y2k=*0Su&s~k$rBzLGzmID&D2j|Uu5;8+PRKQwXBdMa(9%b0U$3yrM92CIeV1zY z-?#^mw_5Ybij=$@M$J9qdCcxYTtw4napj4Z^fYoVmMOU^ls`UHt%4>`h5UmpDjcK_ z&o+=8$e%{idwF>8KGGJ>rl|DzX`H10Zt=O|cqxUz_sPZNuq48iROrLr^r#NI2*0`& z9+ce)T*%)apQv9rx3gJ%ygL;HCCK>`gE)P^4OObc(sC2!WOQBEP;yeB=6MyAMC6pY zuXO6ojo5a7xV`7rs}>&ndRfRACnV7JkuL^b8c^1x#7`#4vU$UyHNqo9g3Q7!n#ONskKAqR}HDyFrpeAMVVM|L+wsGlFRS>1hXk+N<`$6sfVND$r2 zrdJw|2A>AXg4JKt;8$N}Lm%^=#u46(V=GNuVYH0FeJm_KLemg`i}|SU!2y)Ci)qle*SufMAovRb zQtkMPPpq~_?fp{SynlQuE#F7idq9O~?v7?wN&y{X7F{7Ll6J~(F3GuKEo3IdZI(F) zH5EYn+Rxm+aZKd>q}}o%Hlvzg^=od!a|1OrL9?BzT>i zM|sPr7)zpt!0BuxBL5zLdi!73kTCy?^70=UP5-y`^xrR9Q7XmAL1^0BR*eK4EOK>l z7C){0f_1)}mH=oTOA`oz_ydVcJ_=hIu3Rk!bAP3qoIG&;c4AAsl^8Kq2Tr1mdD7M% z-C8XMdw5I9nY8$L!KTtwKq+BPP-vR`+(|@pYm3C^GJJWpnmsb{Nt48yj*f0&HHhtr z*flnScNbHp)*u7#lBj6xU% z!WTre-kI@V9;yj`dlvGBy(Ti+8}tw8?fs*vRfmC@G@0b&_H_&|i*Zk8qgGOabYBc> z-(|d62nn&j4M($1IGv*<;9yJWnT}TGF|v%AX&N`P7qhQU%XtJ)fRsdZ|!#G_#}eG zKUUVrTgEz;4Q^u2@Tq=Ju_9`hFUsV##y#NK%aeejDmJWS9*n~5mMXuFr)HnYX|uOt zha>1mF`1yKY^3F(k!8~^pP|pRopEZN=q3RJkUzM%&LXOKH_*O3BUZjUcFt>@&r#6H zpQYW9M-OlOkQgpBpqJt>cO=e7W*qa~G|1^T&Z!`uIw_?9#f0b&8gTeh^lVCP3q0 zFMp-BdRqpM$z@dhPQl<8DG47apR1?6EuK&JN|(Iz&sbSW^`+Mc8+CaKBTKgE)2aem ziSXTmUAMje~kbbnsA&WDmbqI3`d>^r`R6Ke12xdtUc1!l#{z-Q9L}L1Z z82|WF?d=!OAa+@@V4`MY5Lk~`ZC4nabhVwOJL-#iGS9(P(aK-tajq>OM`Jv01B|O^NvyBdb?#$PX1$E zG33ooO|MzDCO#oCZ~0ZIESO0eZ4EYq;xtqtHjNL(v?E5Leb6RTi(x&R*DZvN$jc0z zIgs1|aS~xD(>diQfELYO(gVuZ+&S0U1Sr0KUU=8#{?%VS%z9J(tvr9Zd zTJEzfy6V-WBPfhW#baVz?MoO<79v2T;spv$o3C)2NXHROCP_*qQ3~fuhJ@q3S|Q-I z=AV)WMdT%i{2$Z3qfjajwLW^8xT`$jA!b1AW>=Iap=j}yAUbIfJ+xU~cxKUoY7B6P z4%zpWwFo4LwiY=uNoHJVDNuDc3^488udY==+1rR(j+=1CoWx7_wlW%R4Du%?&XH~z z#eKOv{L3AiUH|PyzS_eg(m|W-F7j@M}WgP<9Ff%>qzXKz%UPfpr2ojQF&9JInm>;l zBk(OY{T`5JYJ(QA1J)XcA3+DWLMfo~mfsS@tMrJ_a1iDE3JqxdJZ~jf>5<_ayGJ#S8T^EgoEm zZ7^@4hJoo>et|RcQeE=nS@l6B40I2eDC}^leNpCjR;h=l>rMGhe`Fak7OpE!udYAW zI=Fh@bUv2 ze)($5^~)FhILtbPThFQ9(0%`XmK_j(Kq;ad43J0z&<8@#v|CEu1?1_pz_$Pip@q@P+<#Ty4aXBHsKaPPI7;LOkY*uHb zhP+ogHfMzz{S0BwHOBn@Jt-}*QgR`9^9o_Lx}&D=K1{N7Phsn(Dt+0O%dgaGiV)er zxhCq(S**VYXsjA5(8c(vST&)9!6s>ro(n0v(1$eq6P4d3A#w9 z^KwxQMyooSrk5KLY>Y7YQzjy?c_chr1goV|8UEh0iZgR={p#wpA?uWAZW?iI-g_w0 zqIC$bxkkW0tGz?#NO$LALPJ@7SUM-v6Ff<@w;m#4+vR2YR*xE-CaM&;&Ftrrfxa-e zjJOm{X7CPFso|MKN)D-t)rteYWW$g6yj|QUVnLZdoQ+)iovK!w-g#xc7<;X!yb*sK z3~f&08EOG>obw)jh9@0Gb#J$BRh#=sWep7;0xfgOZK;Pv%f?}JDLzlV5~0EnFjy7> zG5u5Is(Us=V~G}Bz^v(#(;r~BT!c1=%)@RYwD(0dBQM0iPT$AZ`E9TY2+#!uLud2Mmf zrEfOh14Ix^fW6aT$erZlgc&kH+-y@s&#hp{!WiV*XrbX%0#-i$tH;R_P$Q>=S=<1`0?XlzN=zJLtsc#7Fto#i_~GVs%$hczVLV#|s+IDjOS&1N)#{KzPA^}_WKJpo9-3!rZX^Glo6tZ(7 z!vJ&`*#hmDmN`+@d$iSTP$Jsb;V6fGF&S(2EN2Gd62Ay5&U=ZHDeBwbjQi9`P{syU zL0&s6cNk9R@KC$yN#>|4x-7?#LPu;49rPZa`F-Q9Igt7_!poCuTK{lgT;oRI;c!)6 zghxE7m$HcU5-FVG$63Ca`9jcIrm2Zon;pfR_s8Eh6d7tnJG9irwETzB|4}O?e;Ogw zil_3C;|#m{?-kXn4YB`Z4BSzLZB4#v>c=Tpc+^tRn&ID8(dhsSXkT1MS0U#PRkHYv zx`u(7PC7YW-OMropd5r*2MmT7uJim=zjEe7K!p80Al%&7^V^|YXN6`rPhOoNPjk6< zME17spzglS;&0V{zWL878tuP0Nc6-*z>(7H91Q)DcG3M6Cj-g8G-1mBOGr$6`=;5x zB*I*5$+s~#fv~%FantH$6+e)n_d)-DMQc?s$b%+g4wTbp6B${g7^BhMSUmo@P}Rw> z)@%69diuLGdiR;JSKOKIMfHyxzflU7RG6?Tj6D!f{*+*In^xN54Ap~>T&wo~o*UO) z;?q5IPUE}u>;2S20 z-2=)dz=^ZF*1gs}+UiqGs9<3LJ_X7(o^$_putA7PB>i`l%m=c}@k}gei2)YCNxK$H zP8>*b!HfkGO8&7dmn~kN2v3(_&Sg`w>a0cTDLD5I*U*it8Ak6CJS#05e94yci5=I| zq*x%=zFWp%tiH^>@t^`^@+*t*@D;ix9=$C_RCf=k%_!z|@V@r0&$9W`1*m_xK(R(7>>8Ov=yI^Hxu|lNrZey8+2BBg9cbM|{&Km~&3pq7GK}*@5SR?}Fgdy`})cXezvm;1V9sQSI?-cF<^}o<$8Fel4?m_$#lb z1<&OH|LA@shY7E9@VPU>m^AK5$J-Ds$zla;hf-xiAA-pn1r6v=qgoZ|-qX?mkW;CN zDDjRIa_BURr(B$28VuH>1#u^l1SAn7IkY|)R~{E$NBwa5Wfm(7ulPI6`5N+8{X&1d zqvghbZ3LoQZ2WhaQ-I(1fwH?7vi35!%$6!NHzfwahx=Lh(-T0YIf zbfj@ETx=HQ^K_t0fYK(?{v2z=o$s8tn2lMO+j`MmC9O-P0!@UNRF`1BAe;d_^+^iR zvQup8c|A_hevwN&85<((_}Y0Us(Oh=h+atR;5qm4Q)?mXi`AHNvJ046BAr_S-=>R& zy7X380OR_{eHY~p;zOw0*G^TI!mdFJ~5@6v0UiRzw~giRg#N*5wOCFlAPM2kh^QlW-j zX0l%8{&gXoUo&1<7Cz+LFD7U`i}ebY3h8D7gS3oG7z@RMLO*b4FS+@t#7mw`!hNDL zsFDe1V0P2<{3dR*LAt}zFE?8Y9)t`<=|3%cr+pi@tG8~_f6VbW@vj3wt_=mQEM2gN zbj=8%O>2F0%$8yHaZNA%L$U#;0^NG)oC6Jx<*50pRM2jPl zBm4qStsog2w6@tEg?zh4EXEA4CCyh-k=sC~>ASyXwEQ$7I;ggd<3^?NPEIDZ=af;c zrfJZ#UZ!;*C*{F9;;@OZYx`&j58k5?xz)a&((EAK=fs{aJA<2H%cUpPGvvEoG zFCP~Fc`ii{pLUD@9{V|GtRgE^sr$vQdVA*pI}ulcBBK&dO5aBG^?>t<*pE+lHJ5F( zg!~ArbaXCCG~g!Gd1laCiRJi8EUjHJ`epwQQQ(s%pzq|YMp<%0^?x|Rew)8`i#(WD zHb?q*Bxh8ptqi{9w2iSnC=&W?>^65Up4d+3@~Kp;u~dIW3TRF$h%pH`zE+ zv};DH`Qdw`}Bhr{by@A5a(Tkigg}w}@n-8e>iZo6CIj-LFRN zahSj3_g{$KRD`ASW#`E9`Q{PZf!MG#f3#sMH=`CkQKTA;rfZU%1=R^;kNeyQ*ACVJ%-2ja4m-v!HmgdTa-IhMY~?qrgT>w+rD=s3RXt@UT2 zdJ_@5kwhfXj>lF{B)M@&oWxGmxEHD}wgv=}goCZXVC9O62C4s$UQEV|(#fwaH^b{9 zTM!4SxDCJYpqUu@LI)wvFw67BAd0qmYKOs8l@=-H%i{wV{3mVB(cJF|8^Pr+_*(l-$$UW+%l8;W>9qcDF}G4=3!i2B zA_;wrz57DCW$WX+bPa39g+oR)^69k;kxBwXLhA*1rWmbKzFf(5&!qvIlUhI<5!9*sVh@+azr zW(9)LHR=1)b|%~!t;s%<+KO>GGp&rjTpM$8W=Oj$iH%Ri}66wxG9EC zo>TEfzd?+IBv7gw$ZwO=3AoA7^U1Eb%uE+7MEBW4-8V{7zH-n^N3YSmp4cGlJMh*Y zFL>0V@h2m#_<`s*WqW=5Li4VaN#F^)djwdjpjnjds5`Bg(!Vz%GW;c6e}j18AM1{1 z?J86bsI!T-CjGMr-*IS%w5lS%X96|(1Cr$Ccn_Fm&`s%X4>J|a-U%^A_gK!GW|WrW ziujf;2(y^MM3g%vWv!|+3BHoLk%nRW`ZoCzH}eGRrsuI9Dpn1S3C{G*mkeq04c09^ zy)D}83kmx9TIX&H9@KZ>0~0FSBSSF^7Fki?)ZsbY_WmB}7f!83i-J6zBH!m1E(w2toR17M z_ctK!Tx8Tqm)?zO?d|G|8R^UHb#RpnTDy(OWjF#Z4igm-hH_lW!`Abi@EBu9>W$yt~nFSU`-m%`B&iU&6 z%1_fXI~}!n^{VXJdY#ppuPHw_wmc@Fv411GWaZRSP_#o*dJky&5FQ#SU9PhSMK0N* z{&;gE)2;<4osf?Zb4avBFn}FFz-j>k z`4hMQl~nqljL2iNEC0VEry-s4@b!B&_^W_q?L$T8Ygshh<@-U#%p@K<=v?KsHD7e`XKqG z%wYU{r|-r9z*AffA{r(d_BzE)2XdgiNfCrRaJmPGNdH-IIWt*%w=GeuP#=JQ5pI}T z>=lP$YVW=G?!ux&49k~D#(x3?tHMtqI)362T;W$dWFuZOTubav<~XVRNL?*z{j#`h z-ymGnsSO?5+ZE*+O-!Xlwy2UguI+i1-u@xn{Gg6V{B3cLA&P(M?VChHnb=8Z%(xKy z;U9R7lQwKH&=Z1EyMhv%wCxeqos`9$!Ov2qZ+m=EIT4hI1t%(rAEJn}J36@7sFsde zZ0rX=wKQyRiE$lmPs6h{k+{kSJqK?C(u~>C8`lRK50@B=4}>3EfwlRyKssQxL4V@9 z)M`+)g-0YydFGIQKVJ@X@13~H+-?0TI@kWUTQsA?ln*tJOJnv|Dqx#@o2hJ#Nlxj110 zRYR?1-~KnK?)SwdED_DlaKmk8&=vp;T#SNvYdBQQwx?EG@c)X)vu0N&EDf#x-DY|H z#qrX4Q19x!YuY`4;&aej!XWN>j(dQyB<~?lko~3S?;!aEyWi}}5YJwAeWwm?#GASO z){oBu`qp~CcVd@J_kZ;5CZ9J?@1JqoF?kL}_L9bqfHfhSYFaEFF;HBr=2wU{DNZ4#*^JcW^ek2tnU#l=N8Dv|{bPs5;Ued|o{T|X)=-b1ziKQG3w-PSDh#=DrvLErC zFmt%|(kaHO713SGrDoBKxRqeyCjoJ}Gf@9o1I^2i8g2HV__MQt;cS;{Oa4f7T&0HP zJU)bs4xyUj=m`-bkEaU0=UZ>yV2#V3-+>h{$l$j`>yx*|MZ)l=NYt7;If)aZ#oX?49(=q0Dp7FJ!K@2p1dvwc?5+6ejp{vIVgfW5wXtVHQ!x@d% z-d;cZUxp7Nzo})A5Nh|`cwU1xnZo^=XI!DZOh8o4gLq&d5WL!DgUKCK#R{+2w!O|I{48us(x(94D7>b%90XZI5d=4+zp1j=vxPu;0H~Lewy? zeFhE7GR56B^Ty;R6<%r-D6*-O5ge<|wvC9)xgv_6KzW;*-b3a|Uhp?Hg%IxP>in*M z2T^T{Oq=iwJS)%l#-Z4j7U!L&G@ZWNVD#&@lGjOlusFzMpq^8~=^U5Da(vd4_N@tx z3}vS_4yx|anD!{8Pbz~Xs*agxDM(ha$5K0*_(RY8iwfT`Euk2pX0unjm4{_5_MbNR zhOiFKJW7!OJKR|~n>5TSI(O4r)uD@Sl>I!9$^i0qKbTTT4N&E|aDjE*UKEuZAqWAbasaDf0upa+EOam3p}X zL8%Sh91_p9TO1`A`oIK8qp^o*0ZkGjslNulS-ydEHk!>Suq?HI*T*OE=5r&Q6E&GX zYd}qA!B#Iz=(of-2>Py7tQ13`S3;OMp$b3t)&Mr*P%Fy!^f$Bbi^bZcHFPY$l zu@OJ9)_*Lk9Ibz0EL`>LD_c0fK)80Cyyk|OlCVGw)k|?RvUw<8t(Ljc$nW&Ff}X%p zFFa33dYH4HGU7B`1yycZ6?3bJb9Wi0LQkYIx%z&Ml{#Hgr?9#mswpMeToN!pHi-ZX zhAP2xrUcTURkEeCiB~96wWxGgRCRx?w3&Pl@zVY(_Br9Ym8V+&8Dn3L8-1nI1kLru zTE7Gb`(#l5rXJFBQ4sq5l-$BK?aTIpF|U5nOD;jgH^kl`KxeJw7!~HmWYcU5?@V%j z(Fu13uKS*x?@i7TF?$M-SBR|Y5KNcf2clQF)dNVX8!}_uv7feOwaFP;?T)tHbS0;Q z1wnhRK+VY{G&dqrK$6^csk|4Bjf!ZqTDb04XS-&`XJL;7ln>yk-oT&4FLr@fA=Fxn z@{}BW=U0)at$4FvH4Oc{yi80hpI?l!U8g(bF%sM*2g>B;L2rAc&B?i7pDs&w7wu?3 zYdUy|b)WsNyQZCY>nr0<9W!xGfY{rmLCMf))vZFeD4BHE`5{3k6;-;TjZg`dDhAQM zWfst(a!4rj&v-D)$?-X}l%@E)CC~3ZlMCYQaGR?Yd@>VN??gZaFilM#$c|HYPWVNn zH`D$JPedt4_`wC2fXn|#nKi)BLXXUom zVHCXOp`O}zyvImc+>4Kv@-gM1{(j4p5K7OyF<%E6nUGxYd+`WM-|5NYs z{^A_}|AlWjevPU?nrq|7vbUXe>ivC<16ug z$i)Op540BAO@qLT&08$P9DWvV**j)}cCC^&1>+YkeuRfpT7ZW0ksR@19&Iyqz)%Q; zlTk>8+Q+YB#)GYsy}X{D)I>bSG#zP(Hkd8uYa7f zX7U&Ml&M#T_#nknRi@FDrPA8fS7fd;{VY95bD&5NAJTYD?RsQiwqG*!vV1Z`xdch{a`WaP(Xj`I=$ano0yMte^;H8x6BQ zo4eF%v)q=obaXYu8yTP(T^c9WcFZKt^7}&;M(BvnoZOx54Yj+g%a}S9Y*%q?f&D-5 zCHrm~>AsVbRoh1l61yL;9e@WoW$fhnf)UE*^1}Y3`f(f(A9VqY#J<8ehmm>5Bg>t! z!0uPqK#5w)N;t(%oO~oYA=NW2M~q6Y3kUuV6LM>a`6Q)ozdW6<Pu=BC%F(Mjg^ z>o*36W$RhWwgtOBYjCg~S?ySKIjkCu@0r|FhEPj?+GaIoOm#12^()~S&qtfdI? zkN|u5{Vm((X!h%!Sb5Ifyusb(dIM5FMlm$m8X?f?6xwMv?S-dM;iK+i>Kiu?fv^fK z^sqK5k1(hoTm44$z9hg-`1+`w&Zf-h0p0p)qXtdE{4{teDrW$hJ6<49Qnnx8k*ww9 zAt@-l5;DzGkmOIkz?ufy@ zZRV-Sg=ax@*~lx9(+zap?zNqT4Q{;75@|fWtI6Q-LCRf0fnfbZsL{6h#ZFpiSeDQL z?&a`&Z({u+$n9o@)xe$!IeCVcmKx=nr67#Fres&la_FTpT=x7(3VXekatJc{ax&+h z{6qC!WwRJAbxYNA7Ppz2WY-4>*CHZB_QCp=@GNO&VbC{tfHpN0^@5fTXpx>s_Op! z%Fli_6TW`hv$N#$fvko>=1|>lL8~?gK3eNsgU%mNaUX$plt$yHRH>3?$sv8$%=_n| ztJ{c|EdYQffcLcl6>XCg*i|*8H=M2^aH(S>M{2HRsi7`*@%%Bt>~1}e>c}o_ z2WLmHkp{9<1$;_FlDhYC^xjp?(DlieECEJoQGzvJCcH@`^{BdSFv7 z*H0K_&vwu3)%ll9i;(Q=8dI4CY#d@aMX5lHI$FB+5H`fg4N4&_%t8<9_bkfaNe9@M z(cx`2pB<{Ua&k}li2_w(1nm4amz=mklgS7H)0ryK&7|-IemK9w$~8pRh=f<>5kl zd;1XikwC7Z=Pv!yh`a?17>VM!-~*exz#=(YVcjmpCP1J_w$=m1DMt9#Dumd zN1rw?Ql>Ya`S|UF`pP@@R6_5z8<6#xkM@A}t z6cv5+pCjC#?Ere@kD1^U@{xDM0Ulc2TCLxwblv&a72Yx$ou4n$bu2OO@bnU*iLw@G zLGdZ7qdN0T85Z`TwZGoYfO2GU0sy8kCo=L;eQbQSv1umOucm*=7px@1+DU`woC zX4`MQ`W<&o4zWh85MJ9IpU0CtE+Ew1LrahNpqM{9)G(qr=^bQvpxkGfKJAXTk1lGq z1>Wa15`v2<%Zu=b{`@#C5PZ1gVEm8YXjtF1X2?Wo{x{z+tG&)k2(lmfW#n^|iz_Z0PzbfE#d`tTL3IFFTPb$1x zYxO>O#*w%>!scHDIcvq{A5ekPXhfEGwIMgyGNc)^nfnrX+f4!m?p%w^EWMTaSHk&U qjUOQy|0Cx4-') + $(".pytorch-call-to-action-links").children().first().before("') } @@ -91,7 +91,7 @@ {% endblock %} diff --git a/advanced_source/cpp_frontend.rst b/advanced_source/cpp_frontend.rst index 7952b4593..878b2859c 100644 --- a/advanced_source/cpp_frontend.rst +++ b/advanced_source/cpp_frontend.rst @@ -1,5 +1,5 @@ PyTorch C++ 프론트엔드 사용하기 -============================= +================================= **번역**: `유용환 `_ @@ -87,7 +87,7 @@ C++ 프론트엔드에 대해 자세히 살펴보세요. 것에 유의하세요.) 기본 애플리케이션 작성하기 ------------------------- +--------------------------- 먼저 최소한의 C++ 애플리케이션을 작성해 우리의 설정과 빌드 환경이 동일한지 확인하겠습니다. 먼저, C++ @@ -228,7 +228,7 @@ CMake가 해당 파일의 *위치* 를 찾을 수 있도록 하려면 ``cmake`` 제가 보기엔 항등 행렬인 것 같군요! 신경망 모델 정의하기 -------------------- +---------------------- 이제 기본적인 환경을 설정했으니, 이번 튜토리얼에서 훨씬 더 흥미로운 부분을 살펴봅시다. 먼저 C++ 프론트엔드에서 모듈을 @@ -237,7 +237,7 @@ CMake가 해당 파일의 *위치* 를 찾을 수 있도록 하려면 ``cmake`` 내장 모듈 라이브러리를 사용하여 완성도 있는 GAN을 구현하겠습니다. 모듈 API 기초 -^^^^^^^^^^^^^ +^^^^^^^^^^^^^^^ 파이썬 인터페이스와 마찬가지로, C++ 프론트엔드에 기반을 둔 신경망도 *모듈* 이라 불리는 재사용 가능한 빌딩 블록으로 구성되어 있습니다. 파이썬에 @@ -262,7 +262,7 @@ CMake가 해당 파일의 *위치* 를 찾을 수 있도록 하려면 ``cmake`` 이동시킵니다. 모듈 정의 및 매개변수 등록 -************************* +*************************** 이 내용을 코드로 구현하기 위해, 파이썬 인터페이스로 작성된 간단한 모듈 하나를 생각해 봅시다. @@ -310,7 +310,7 @@ C++에서는 ``register_parameter`` 메서드를 통해 텐서를 전달해야 전통적인 (그리하여 덜 마법적인) 방식이 제공됩니다. 서브모듈 등록 및 모듈 계층 구조 탐색 -********************************** +************************************* 매개변수 등록과 마찬가지 방법으로 서브모듈을 등록할 수 있습니다. 파이썬에서 서브모듈은 어떤 모듈의 속성으로 지정될 때 자동으로 @@ -457,7 +457,7 @@ C++에서 ``torch::nn::Linear`` 등의 모듈을 서브모듈로 등록하려면 있습니다. 순전파(forward) 모드로 네트워크 실행 -********************************** +************************************* 네트워크를 C++로 실행하기 위해서는, 우리가 정의한 ``forward()`` 메서드를 호출하기만 하면 됩니다. @@ -479,7 +479,7 @@ C++에서 ``torch::nn::Linear`` 등의 모듈을 서브모듈로 등록하려면 [ Variable[CPUFloatType]{2,5} ] 모듈 오너십 (Ownership) -********************** +************************ 이제 우리는 C++에서 모듈을 정의하고, 매개변수를 등록하고, 하위 모듈을 등록하고, ``parameters()`` 등의 메서드를 통해 모듈 계층을 탐색하고, @@ -659,7 +659,7 @@ API(``torch::save`` 및 ``torch::load``)는 모듈 holder(혹은 일반 이 API를 사용하겠습니다. DCGAN 모듈 정의하기 -^^^^^^^^^^^^^^^^ +^^^^^^^^^^^^^^^^^^^^^ 이제 이 글에서 해결하려는 머신러닝 태스크를 위한 모듈을 정의하는데 필요한 배경과 도입부 설명이 끝났습니다. 다시 상기하자면, 우리의 태스크는 @@ -697,7 +697,7 @@ GAN은 *생성기(generator)* 와 *판별기(discriminator)* 라는 기계입니다. 생성기 (Generator) 모듈 -******************** +************************* 먼저 일련의 전치된 (transposed) 2D 합성곱, 배치 정규화 및 ReLU 활성화 유닛으로 구성된 생성기 모듈을 정의하겠습니다. @@ -778,7 +778,7 @@ ReLU 활성화 유닛으로 구성된 생성기 모듈을 정의하겠습니다. 판별기(Discriminator) 모듈 -************************ +***************************** 판별기는 마찬가지로 합성곱, 배치 정규화 및 활성화의 연속입니다. 하지만 이번에 합성곱은 전치되지 않은 기본 @@ -823,7 +823,7 @@ API입니다. `Sequential` 을 사용하면 판별기는 대략 다음과 같습 데이터 불러오기 ------------- +----------------- 이제 생성기와 판별기 모델을 정의했으므로 이러한 모델을 학습시킬 데이터가 필요합니다. 파이썬과 마찬가지로 C++ 프론트엔드는 @@ -936,7 +936,7 @@ MNIST 데이터셋은 학습 바이너리 실행 위치를 기준으로 ``./mnis 즉, MNIST 데이터셋에서 데이터를 성공적으로 로드할 수 있습니다. 학습 루프 작성하기 ------------------ +-------------------- 이제 예제의 알고리즘 부분을 마무리하고 생성기와 판별기 사이에서 일어나는 섬세한 작용을 구현해 보겠습니다. 먼저 생성기와 판별기 각각을 위해 @@ -946,9 +946,9 @@ optimizer는 `Adam `_ 알고리즘을 구 .. code-block:: cpp torch::optim::Adam generator_optimizer( - generator->parameters(), torch::optim::AdamOptions(2e-4).beta1(0.5)); + generator->parameters(), torch::optim::AdamOptions(2e-4).betas(std::make_tuple(0.5, 0.5))); torch::optim::Adam discriminator_optimizer( - discriminator->parameters(), torch::optim::AdamOptions(5e-4).beta1(0.5)); + discriminator->parameters(), torch::optim::AdamOptions(5e-4).betas(std::make_tuple(0.5, 0.5))); .. note:: @@ -1058,7 +1058,7 @@ optimizer는 `Adam `_ 알고리즘을 구 ... GPU로 이동하기 --------------- +---------------- 이 스크립트는 CPU에서 잘 동작하지만, 합성곱 연산이 GPU에서 훨씬 빠르다는 것은 잘 알려진 사실입니다. 어떻게 학습을 GPU로 옮길 수 있을 지에 대해 빠르게 논의해 @@ -1141,7 +1141,7 @@ MNIST 데이터셋의 텐서처럼 우리가 직접 생성하지 않는 텐서 torch::Device device(torch::cuda::is_available() ? torch::kCUDA : torch::kCPU); 학습 상태 저장 및 복원하기 ------------------------- +----------------------------- 마지막으로 학습 스크립트에 추가해야 할 내용은 모델 매개변수 및 옵티마이저의 상태, 그리고 생성된 몇 개의 이미지 샘플을 @@ -1200,7 +1200,7 @@ C++ 프론트엔드는 개별 텐서뿐만 아니라 모델 및 옵티마이저 생성한 이미지 검사하기 --------------------- +-------------------------- 학습 스크립트가 완성되어 CPU에서든 GPU에서든 GAN을 훈련시킬 준비가 됐습니다. 학습 과정의 중간 출력을 검사하기 위해 @@ -1280,7 +1280,7 @@ C++ 프론트엔드는 개별 텐서뿐만 아니라 모델 및 옵티마이저 모델을 개선할 수 있나요? 결론 ------ +------ 이 튜토리얼을 통해 PyTorch C++ 프론트엔드에 대한 어느 정도 이해도가 생기셨기 바랍니다. 필연적으로 PyTorch 같은 머신러닝 라이브러리는 매우 다양하고 diff --git a/advanced_source/ddp_pipeline.py b/advanced_source/ddp_pipeline.py index 36f1d2641..93858443a 100644 --- a/advanced_source/ddp_pipeline.py +++ b/advanced_source/ddp_pipeline.py @@ -22,7 +22,7 @@ ###################################################################### # 모델 정의하기 -# ------------- +# --------------- # ###################################################################### @@ -77,7 +77,7 @@ def forward(self, x): # ``nn.TransformerEncoderLayer`` 의 절반은 한 GPU에 두고 # 나머지 절반은 다른 GPU에 있도록 모델을 분할합니다. 이를 위해서 ``Encoder`` 와 # ``Decoder`` 섹션을 분리된 모듈로 빼낸 다음, 원본 트랜스포머 모듈을 -# 나타내는 nn.Sequential을 빌드 합니다. +# 나타내는 ``nn.Sequential`` 을 빌드 합니다. if sys.platform == 'win32': @@ -122,7 +122,7 @@ def forward(self, inp): ###################################################################### # 학습을 위한 다중 프로세스 시작 -# ------------------------------ +# -------------------------------- # @@ -135,7 +135,7 @@ def run_worker(rank, world_size): ###################################################################### # 데이터 로드하고 배치 만들기 -# --------------------------- +# ----------------------------- # ###################################################################### @@ -149,16 +149,17 @@ def run_worker(rank, world_size): # 알파벳을 길이가 6인 4개의 시퀀스로 나눌 수 있습니다: # # .. math:: -# \begin{bmatrix} -# \text{A} & \text{B} & \text{C} & \ldots & \text{X} & \text{Y} & \text{Z} -# \end{bmatrix} -# \Rightarrow -# \begin{bmatrix} -# \begin{bmatrix}\text{A} \\ \text{B} \\ \text{C} \\ \text{D} \\ \text{E} \\ \text{F}\end{bmatrix} & -# \begin{bmatrix}\text{G} \\ \text{H} \\ \text{I} \\ \text{J} \\ \text{K} \\ \text{L}\end{bmatrix} & -# \begin{bmatrix}\text{M} \\ \text{N} \\ \text{O} \\ \text{P} \\ \text{Q} \\ \text{R}\end{bmatrix} & -# \begin{bmatrix}\text{S} \\ \text{T} \\ \text{U} \\ \text{V} \\ \text{W} \\ \text{X}\end{bmatrix} -# \end{bmatrix} +# +# \begin{bmatrix} +# \text{A} & \text{B} & \text{C} & \ldots & \text{X} & \text{Y} & \text{Z} +# \end{bmatrix} +# \Rightarrow +# \begin{bmatrix} +# \begin{bmatrix}\text{A} \\ \text{B} \\ \text{C} \\ \text{D} \\ \text{E} \\ \text{F}\end{bmatrix} & +# \begin{bmatrix}\text{G} \\ \text{H} \\ \text{I} \\ \text{J} \\ \text{K} \\ \text{L}\end{bmatrix} & +# \begin{bmatrix}\text{M} \\ \text{N} \\ \text{O} \\ \text{P} \\ \text{Q} \\ \text{R}\end{bmatrix} & +# \begin{bmatrix}\text{S} \\ \text{T} \\ \text{U} \\ \text{V} \\ \text{W} \\ \text{X}\end{bmatrix} +# \end{bmatrix} # # 이 열들은 모델에 의해서 독립적으로 취급되며, 이는 # ``G`` 와 ``F`` 의 의존성이 학습될 수 없다는 것을 의미하지만, 더 효율적인 @@ -190,11 +191,11 @@ def data_process(raw_text_iter): device = torch.device(2 * rank) def batchify(data, bsz, rank, world_size, is_train=False): - # Divide the dataset into bsz parts. + # Divide the dataset into ``bsz`` parts. nbatch = data.size(0) // bsz # Trim off any extra elements that wouldn't cleanly fit (remainders). data = data.narrow(0, 0, nbatch * bsz) - # Evenly divide the data across the bsz batches. + # Evenly divide the data across the ``bsz`` batches. data = data.view(bsz, -1).t().contiguous() # Divide the data across the ranks only for training data. if is_train: @@ -211,7 +212,7 @@ def batchify(data, bsz, rank, world_size, is_train=False): ###################################################################### # 입력과 타겟 시퀀스를 생성하기 위한 함수들 -# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ # @@ -240,7 +241,7 @@ def get_batch(source, i): ###################################################################### # 모델 규모와 파이프 초기화 -# ------------------------- +# --------------------------- # @@ -264,9 +265,9 @@ def get_batch(source, i): # In 'run_worker' ntokens = len(vocab) # the size of vocabulary emsize = 4096 # embedding dimension - nhid = 4096 # the dimension of the feedforward network model in nn.TransformerEncoder - nlayers = 8 # the number of nn.TransformerEncoderLayer in nn.TransformerEncoder - nhead = 16 # the number of heads in the multiheadattention models + nhid = 4096 # the dimension of the feedforward network model in ``nn.TransformerEncoder`` + nlayers = 8 # the number of ``nn.TransformerEncoderLayer`` in ``nn.TransformerEncoder`` + nhead = 16 # the number of heads in the Multihead Attention models dropout = 0.2 # the dropout value from torch.distributed import rpc @@ -332,7 +333,7 @@ def get_total_params(module: torch.nn.Module): ###################################################################### # 모델 실행하기 -# ------------- +# --------------- # @@ -435,7 +436,7 @@ def evaluate(eval_model, data_source): ###################################################################### # 평가 데이터셋으로 모델 평가하기 -# ------------------------------- +# --------------------------------- # # 평가 데이터셋에서의 결과를 확인하기 위해 최고의 모델을 적용합니다. @@ -454,7 +455,7 @@ def evaluate(eval_model, data_source): mp.spawn(run_worker, args=(world_size, ), nprocs=world_size, join=True) ###################################################################### # Output -# ------ +# -------- # diff --git a/advanced_source/dynamic_quantization_tutorial.py b/advanced_source/dynamic_quantization_tutorial.py index be0182a5d..3c9f2af99 100644 --- a/advanced_source/dynamic_quantization_tutorial.py +++ b/advanced_source/dynamic_quantization_tutorial.py @@ -72,7 +72,7 @@ def init_hidden(self, bsz): ###################################################################### # 2. 텍스트 데이터 불러오기 -# ------------------------ +# --------------------------- # # 다음으로, 단어 단위 언어 모델 예제의 `전처리 `_ # 과정을 따라 `Wikitext-2 데이터셋 `_ 을 `Corpus` 인스턴스에 불러옵니다. @@ -195,11 +195,11 @@ def tokenize(self, path): # 테스트 데이터셋 만들기 def batchify(data, bsz): - # 데이터셋을 bsz 부분으로 얼마나 깔끔하게 나눌 수 있는지 계산합니다. + # 데이터셋을 ``bsz`` 부분으로 얼마나 깔끔하게 나눌 수 있는지 계산합니다. nbatch = data.size(0) // bsz # 깔끔하게 맞지 않는 추가적인 부분(나머지들)을 잘라냅니다. data = data.narrow(0, 0, nbatch * bsz) - # 데이터에 대하여 bsz 배치들로 동등하게 나눕니다. + # 데이터에 대하여 ``bsz`` 묶음(batch)들로 동등하게 나눕니다. return data.view(bsz, -1).t().contiguous() test_data = batchify(corpus.test, eval_batch_size) diff --git a/advanced_source/generic_join.rst b/advanced_source/generic_join.rst index ec9ef5610..c083d0cfa 100644 --- a/advanced_source/generic_join.rst +++ b/advanced_source/generic_join.rst @@ -4,7 +4,7 @@ Distributed Training with Uneven Inputs Using the Join Context Manager **Author**\ : `Andrew Gu `_ .. note:: - |edit| View and edit this tutorial in `github `__. + |edit| View and edit this tutorial in `github `__. .. note:: ``Join`` is introduced in PyTorch 1.10 as a prototype feature. This API is subject to change. @@ -369,7 +369,7 @@ of inputs across all ranks. def join_hook(self, **kwargs) -> JoinHook: r""" Return a join hook that shadows the all-reduce in :meth:`__call__`. - + This join hook supports the following keyword arguments: sync_max_count (bool, optional): whether to synchronize the maximum count across all ranks once all ranks join; default is ``False``. @@ -442,9 +442,9 @@ Some key points to highlight: .. _Join: https://pytorch.org/docs/master/distributed.algorithms.join.html -.. _Getting Started with Distributed Data Parallel: https://pytorch.org/tutorials/intermediate/ddp_tutorial.html -.. _Getting Started with Distributed Data Parallel - Basic Use Case: https://pytorch.org/tutorials/intermediate/ddp_tutorial.html#basic-use-case -.. _Shard Optimizer States with ZeroRedundancyOptimizer: https://pytorch.org/tutorials/recipes/zero_redundancy_optimizer.html +.. _Getting Started with Distributed Data Parallel: https://tutorials.pytorch.kr/intermediate/ddp_tutorial.html +.. _Getting Started with Distributed Data Parallel - Basic Use Case: https://tutorials.pytorch.kr/intermediate/ddp_tutorial.html#basic-use-case +.. _Shard Optimizer States with ZeroRedundancyOptimizer: https://tutorials.pytorch.kr/recipes/zero_redundancy_optimizer.html .. _DistributedDataParallel: https://pytorch.org/docs/stable/generated/torch.nn.parallel.DistributedDataParallel.html .. _join(): https://pytorch.org/docs/stable/_modules/torch/nn/parallel/distributed.html#DistributedDataParallel.join .. _ZeroRedundancyOptimizer: https://pytorch.org/docs/stable/distributed.optim.html diff --git a/advanced_source/neural_style_tutorial.py b/advanced_source/neural_style_tutorial.py index 1403cebd9..8a24e8943 100644 --- a/advanced_source/neural_style_tutorial.py +++ b/advanced_source/neural_style_tutorial.py @@ -1,6 +1,6 @@ """ -PyTorch를 이용하여 뉴럴 변환(Neural Transfer) -============================= +PyTorch를 이용한 뉴럴 변환(Neural Transfer) +========================================================== **Author**: `Alexis Jacq `_ @@ -140,7 +140,7 @@ def imshow(tensor, title=None): # 손실 함수 # -------------- # 콘텐츠 손실(Content Loss) -# ~~~~~~~~~~~~ +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ # # Content 손실은 각 계층에 대한 Content 거리의 가중치 버전을 나타냅니다. # 이 함수는 입력 :math:`X` 를 처리하는 레이어 :math:`L` 의 특징 맵 :math:`F_{XL}` 을 가져와서 @@ -179,7 +179,7 @@ def forward(self, input): ###################################################################### # 스타일 손실(Style Loss) -# ~~~~~~~~~~ +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ # # Style 손실 모듈은 Content 손실 모듈과 유사하게 구현됩니다. # 네트워크에서 해당 계층의 Style 손실을 계산하는 역할을 합니다. @@ -256,7 +256,7 @@ def forward(self, input): cnn_normalization_mean = torch.tensor([0.485, 0.456, 0.406]).to(device) cnn_normalization_std = torch.tensor([0.229, 0.224, 0.225]).to(device) -# 입력 이미지를 정규화하는 모듈을 생성하여 쉽게 nn.Sequential에 넣을 수 있습니다. +# 입력 이미지를 정규화하는 모듈을 생성하여 쉽게 ``nn.Sequential`` 에 넣을 수 있습니다. class Normalization(nn.Module): def __init__(self, mean, std): super(Normalization, self).__init__() @@ -267,14 +267,14 @@ def __init__(self, mean, std): self.std = torch.tensor(std).view(-1, 1, 1) def forward(self, img): - # img 정규화 + # ``img`` 정규화 return (img - self.mean) / self.std ###################################################################### # ``Sequential`` 모듈은 순서가 있는 하위 모듈의 리스트가 포함됩니다. # 예를 들어, ``vgg19.features`` 은 올바른 순서로 정렬 된 -# 시퀀스(Conv2d, ReLU, MaxPool2d, Conv2d, ReLU…)가 포함되어 있습니다. +# 시퀀스( ``Conv2d``, ``ReLU``, ``MaxPool2d``, ``Conv2d``, ``ReLU``…)가 포함되어 있습니다. # Content 손실과 Style 손실 계층을 감지하는 합성곱 계층 바로 뒤에 추가해야합니다. # 이렇게 하기 위해서는 Content 손실과 Style 손실 모듈이 # 올바르게 삽입된 새로운 ``Sequential`` 모듈을 만들어야 합니다. @@ -295,8 +295,8 @@ def get_style_model_and_losses(cnn, normalization_mean, normalization_std, content_losses = [] style_losses = [] - # cnn이 nn.Sequential이라고 가정하고, - # 순차적으로 활성화되어야 하는 모듈에 새로운 nn.Sequential을 만듭니다. + # ``cnn`` 이 ``nn.Sequential`` 이라고 가정하고, + # 순차적으로 활성화되어야 하는 모듈에 새로운 ``nn.Sequential`` 을 만듭니다. model = nn.Sequential(normalization) i = 0 # conv를 볼 때마다 증가 @@ -306,7 +306,7 @@ def get_style_model_and_losses(cnn, normalization_mean, normalization_std, name = 'conv_{}'.format(i) elif isinstance(layer, nn.ReLU): name = 'relu_{}'.format(i) - # in-place 버전은 아래에 삽입한 Content 손실과 Style 손실와 잘 어울리지 않습니다. + # 아래에 추가한 ``ContentLoss`` 와 ``StyleLoss`` 는 in-place 버전에서는 잘 동작하지 않습니다. # 그래서 여기서는 out-of-place로 대체합니다. layer = nn.ReLU(inplace=False) elif isinstance(layer, nn.MaxPool2d): @@ -347,8 +347,11 @@ def get_style_model_and_losses(cnn, normalization_mean, normalization_std, # input_img = content_img.clone() -# 만약 화이트 노이즈(white noise)을 사용하려면 아래 주석을 제거하세요 -# input_img = torch.randn(content_img.data.size(), device=device) +# 만약 화이트 노이즈(white noise)을 사용하려면 아래 주석을 제거하세요: +# +# :: +# +# input_img = torch.randn(content_img.data.size(), device=device) # 그림에 원본 입력 이미지를 추가합니다. plt.figure() diff --git a/advanced_source/numpy_extensions_tutorial.py b/advanced_source/numpy_extensions_tutorial.py index 9475ff084..f08bf6a09 100644 --- a/advanced_source/numpy_extensions_tutorial.py +++ b/advanced_source/numpy_extensions_tutorial.py @@ -1,7 +1,8 @@ # -*- coding: utf-8 -*- """ -numpy 와 scipy 를 이용한 확장(Extensions) 만들기 +NumPy와 SciPy를 사용한 확장(Extension) 만들기 ===================================================== + **Author**: `Adam Paszke `_ **Updated by**: `Adam Dziedzic `_ @@ -11,7 +12,7 @@ 이번 튜토리얼에서는 두 가지 작업을 수행할 것입니다: 1. 매개 변수가 없는 신경망 계층(layer) 만들기 - - 이는 구현의 일부로 **numpy** 를 호출합니다. + - 이는 구현의 일부로 **NumPy** 를 호출합니다. 2. 학습 가능한 가중치가 있는 신경망 계층(layer) 만들기 - 이는 구현의 일부로 **Scipy** 를 호출합니다. @@ -26,7 +27,7 @@ # # 이 계층(layer)은 특별히 유용하거나 수학적으로 올바른 작업을 수행하지 않습니다. # -# 이름은 대충 BadFFTFunction으로 지었습니다. +# 이름은 대충 ``BadFFTFunction`` 으로 지었습니다. # # **계층(layer) 구현** @@ -46,7 +47,7 @@ def backward(ctx, grad_output): result = irfft2(numpy_go) return grad_output.new(result) -# 이 계층에는 매개 변수가 없으므로 nn.Module 클래스가 아닌 함수로 간단히 선언할 수 있습니다. +# 이 계층에는 매개 변수가 없으므로 ``nn.Module`` 클래스가 아닌 함수로 간단히 선언할 수 있습니다. def incorrect_fft(input): diff --git a/advanced_source/static_quantization_tutorial.rst b/advanced_source/static_quantization_tutorial.rst index 8bafc2999..fe24050e0 100644 --- a/advanced_source/static_quantization_tutorial.rst +++ b/advanced_source/static_quantization_tutorial.rst @@ -217,7 +217,7 @@ torch.ao.quantization.fuse_modules(m.conv, [str(idx), str(idx + 1)], inplace=True) 2. 헬퍼(Helper) 함수 --------------------- +---------------------- 다음으로 모델 평가를 위한 헬퍼 함수들을 만듭니다. 코드 대부분은 `여기 `_ 에서 가져왔습니다. @@ -302,7 +302,7 @@ 마지막 주요 설정 단계로서 학습과 테스트 데이터를 위한 DataLoader를 정의합니다. ImageNet 데이터 -^^^^^^^^^^^^^^^ +^^^^^^^^^^^^^^^^^ 전체 ImageNet Dataset을 이용해서 이 튜토리얼의 코드를 실행시키기 위해, 첫번째로 `ImageNet Data `_ 의 지시를 따라 ImageNet을 다운로드합니다. 다운로드한 파일의 압축을 'data_path'에 풉니다. @@ -398,7 +398,7 @@ ImageNet 데이터 이 값이 비교를 위한 기준이 될 것입니다. 다음으로 양자화된 모델을 봅시다. 4. 학습 후 정적 양자화(post-training static quantization) --------------------------------------------------------- +----------------------------------------------------------- 학습 후 정적 양자화는 동적 양자화처럼 가중치를 float에서 int로 변환하는 것뿐만 아니라 추가적인 단계도 수행합니다. 네트워크에 데이터 배치의 첫 번째 공급과 다른 활성값들의 @@ -459,7 +459,8 @@ x86 아키텍처에서 양자화를 위한 권장 설정을 그대로 쓰기만 per_channel_quantized_model = load_model(saved_model_dir + float_model_file) per_channel_quantized_model.eval() per_channel_quantized_model.fuse_model() - per_channel_quantized_model.qconfig = torch.ao.quantization.get_default_qconfig('fbgemm') + # 이전의 'fbgemm' 또한 여전히 사용 가능하지만, 'x86'을 기본으로 사용하는 것을 권장합니다. + per_channel_quantized_model.qconfig = torch.ao.quantization.get_default_qconfig('x86') print(per_channel_quantized_model.qconfig) torch.ao.quantization.prepare(per_channel_quantized_model, inplace=True) @@ -475,7 +476,7 @@ x86 아키텍처에서 양자화를 위한 권장 설정을 그대로 쓰기만 이제 양자화 자각 학습을 시도해 봅시다. 5. 양자화 자각 학습(Quantization-aware training) -------------------------------------------------- +--------------------------------------------------- 양자화 자각 학습(QAT)은 일반적으로 가장 높은 정확도를 제공하는 양자화 방법입니다. 모든 가중치화 활성값은 QAT로 인해 학습 도중에 순전파와 역전파를 도중 "가짜 양자화"됩니다. @@ -535,7 +536,8 @@ x86 아키텍처에서 양자화를 위한 권장 설정을 그대로 쓰기만 qat_model.fuse_model() optimizer = torch.optim.SGD(qat_model.parameters(), lr = 0.0001) - qat_model.qconfig = torch.ao.quantization.get_default_qat_qconfig('fbgemm') + # 이전의 'fbgemm' 또한 여전히 사용 가능하지만, 'x86'을 기본으로 사용하는 것을 권장합니다. + qat_model.qconfig = torch.ao.quantization.get_default_qat_qconfig('x86') 마지막으로 모델이 양자화 자각 학습을 준비하기 위해 ``prepare_qat`` 로 "가짜 양자화"를 수행합니다. diff --git a/advanced_source/super_resolution_with_onnxruntime.py b/advanced_source/super_resolution_with_onnxruntime.py index 7c2886342..5fdf00cf4 100644 --- a/advanced_source/super_resolution_with_onnxruntime.py +++ b/advanced_source/super_resolution_with_onnxruntime.py @@ -110,7 +110,7 @@ def _initialize_weights(self): # 특정 차원을 동적 차원으로 지정하지 않는 이상, ONNX로 변환된 그래프의 경우 입력값의 크기는 모든 차원에 대해 고정됩니다. # 예시에서는 모델이 항상 배치 사이즈 1을 사용하도록 변환하였지만, ``torch.onnx.export()`` 의 ``dynamic_axes`` 인자의 # 첫번째 차원은 동적 차원으로 지정합니다. 따라서 변환된 모델은 임의의 batch_size에 대해 [batch_size, 1, 224, 224] 사이즈 -# 입력값을 받을 수 있습니다. +# 입력값을 받을 수 있습니다. # # PyTorch의 변환 인터페이스에 대해 더 자세히 알고 싶다면 # `torch.onnx 문서 `__ 를 참고해주세요. @@ -137,8 +137,8 @@ def _initialize_weights(self): # # ONNX 런타임에서의 모델 결과값을 확인하기 전에 먼저 ONNX API를 사용해 ONNX 모델을 확인해보도록 하겠습니다. # 먼저, ``onnx.load("super_resolution.onnx")`` 는 저장된 모델을 읽어온 후 -# 머신러닝 모델을 취합하여 저장하고 있는 상위 파일 컨테이너인 onnx.ModelProto를 리턴합니다. -# onnx.ModelProto에 대해 더 자세한 것은 `onnx.proto 기술문서 `__ 에서 +# 머신러닝 모델을 취합하여 저장하고 있는 상위 파일 컨테이너인 ``onnx.ModelProto`` 를 리턴합니다. +# ``onnx.ModelProto`` 에 대해 더 자세한 것은 `onnx.proto 기술문서 `__ 에서 # 확인하실 수 있습니다. # ``onnx.checker.check_model(onnx_model)`` 는 모델의 구조를 확인하고 # 모델이 유효한 스키마(valid schema)를 가지고 있는지를 체크합니다. @@ -182,7 +182,7 @@ def to_numpy(tensor): ###################################################################### -# 이제 PyTorch와 ONNX 런타임에서 연산된 결과값이 서로 일치하는지 오차범위 (rtol=1e-03, atol=1e-05) +# 이제 PyTorch와 ONNX 런타임에서 연산된 결과값이 서로 일치하는지 오차범위( ``rtol=1e-03`` 와 ``atol=1e-05``) # 이내에서 확인해야 합니다. # 만약 결과가 일치하지 않는다면 ONNX 변환기에 문제가 있는 것이니 저희에게 알려주시기 바랍니다. # diff --git a/beginner_source/Intro_to_TorchScript_tutorial.py b/beginner_source/Intro_to_TorchScript_tutorial.py index 47da92ee6..832536a74 100644 --- a/beginner_source/Intro_to_TorchScript_tutorial.py +++ b/beginner_source/Intro_to_TorchScript_tutorial.py @@ -7,7 +7,7 @@ **번역**: `강준혁 `_ 이 튜토리얼은 C++와 같은 고성능 환경에서 실행될 수 있는 -PyTorch 모델(``nn.Module`` 의 하위클래스)의 중간 표현인 +PyTorch 모델( ``nn.Module`` 의 하위클래스)의 중간 표현인 TorchScript에 대한 소개입니다. 이 튜토리얼에서는 다음을 다룰 것입니다: @@ -144,7 +144,7 @@ def forward(self, x, h): ###################################################################### -# MyCell 클래스를 다시 정의했지만, 여기선 ``MyDecisionGate`` 를 정의했습니다. +# ``MyCell`` 클래스를 다시 정의했지만, 여기선 ``MyDecisionGate`` 를 정의했습니다. # 이 모듈은 **제어 흐름** 을 활용합니다. 제어 흐름은 루프와 ``if`` 명령문과 # 같은 것으로 구성됩니다. # @@ -271,7 +271,7 @@ def forward(self, x, h): ###################################################################### # ``.code`` 출력을 보면, ``if-else`` 분기가 어디에도 없다는 것을 알 수 있습니다! # 어째서일까요? 추적은 코드를 실행하고 *발생하는* 작업을 기록하며 정확하게 수행하는 -# 스크립트 모듈(ScriptModule)을 구성하는 일을 수행합니다. 불행하게도, 제어 흐름과 +# 스크립트 모듈( ``ScriptModule`` )을 구성하는 일을 수행합니다. 불행하게도, 제어 흐름과 # 같은 것들은 지워집니다. # # TorchScript에서 이 모듈을 어떻게 충실하게 나타낼 수 있을까요? Python 소스 코드를 diff --git a/beginner_source/audio_data_augmentation_tutorial.py b/beginner_source/audio_data_augmentation_tutorial.py deleted file mode 100644 index 5ffe96106..000000000 --- a/beginner_source/audio_data_augmentation_tutorial.py +++ /dev/null @@ -1,434 +0,0 @@ -# -*- coding: utf-8 -*- -""" -오디오 데이터 증강 -======================= - -*역자*: Lee Jong Bub - -``torchaudio`` 는 오디오 데이터를 증강시키는 다양한 방법들을 제공합니다. - -이 튜토리얼에서는 효과, 필터, -공간 임펄스 응답(RIR, Room Impulse Response)과 코덱을 적용하는 방법을 살펴보겠습니다. - -하단부에서는, 깨끗한 음성으로 부터 휴대폰 너머의 잡음이 낀 음성을 합성하겠습니다. -""" - -import torch -import torchaudio -import torchaudio.functional as F - -print(torch.__version__) -print(torchaudio.__version__) - -###################################################################### -# 준비 -# ----------- -# -# 먼저, 모듈을 불러오고 튜토리얼에 사용할 오디오 자료들을 다운로드합니다. -# - -import math - -from IPython.display import Audio -import matplotlib.pyplot as plt - -from torchaudio.utils import download_asset - -SAMPLE_WAV = download_asset("tutorial-assets/steam-train-whistle-daniel_simon.wav") -SAMPLE_RIR = download_asset("tutorial-assets/Lab41-SRI-VOiCES-rm1-impulse-mc01-stu-clo-8000hz.wav") -SAMPLE_SPEECH = download_asset("tutorial-assets/Lab41-SRI-VOiCES-src-sp0307-ch127535-sg0042-8000hz.wav") -SAMPLE_NOISE = download_asset("tutorial-assets/Lab41-SRI-VOiCES-rm1-babb-mc01-stu-clo-8000hz.wav") - - -###################################################################### -# 효과와 필터링 적용하기 -# ------------------------------ -# -# :py:func:`torchaudio.sox_effects` 는 ``sox`` 와 유사한 필터들을 -# 텐서 객체들과 파일 객체 오디오 소스들에 직접 적용 해줍니다. -# -# 이를 위해 두가지 함수가 사용됩니다: -# -# - :py:func:`torchaudio.sox_effects.apply_effects_tensor` 는 텐서에 -# 효과를 적용합니다. -# - :py:func:`torchaudio.sox_effects.apply_effects_file` 는 다른 오디오 소스들에 -# 효과를 적용합니다. -# -# 두 함수들은 효과의 정의를 ``List[List[str]]`` 형태로 받아들입니다. -# ``sox`` 와 작동하는 방법이 거의 유사합니다. 하지만, 한가지 유의점은 -# ``sox`` 는 자동으로 효과를 추가하지만, ``torchaudio`` 의 구현은 그렇지 않다는 점입니다. -# -# 사용 가능한 효과들의 목록을 알고싶다면, `the sox -# documentation `__ 을 참조해주세요. -# -# **Tip** 즉석으로 오디오 데이터 로드와 다시 샘플링 하고싶다면, -# 효과 ``"rate"`` 와 함께 :py:func:`torchaudio.sox_effects.apply_effects_file` 을 사용하세요. -# -# **Note** :py:func:`torchaudio.sox_effects.apply_effects_file` 는 파일 형태의 객체 또는 주소 형태의 객체를 받습니다. -# :py:func:`torchaudio.load` 와 유사하게, 오디오 포맷이 -# 파일 확장자나 헤더를 통해 추론될 수 없으면, -# 전달인자 ``format`` 을 주어, 오디오 소스의 포맷을 구체화 해줄 수 있습니다. -# -# **Note** 이 과정은 미분 불가능합니다. -# - -# 데이터를 불러옵니다. -waveform1, sample_rate1 = torchaudio.load(SAMPLE_WAV) - -# 효과들을 정의합니다. -effects = [ - ["lowpass", "-1", "300"], # 단극 저주파 통과 필터를 적용합니다. - ["speed", "0.8"], # 속도를 감소시킵니다. - # 이 부분은 샘플 레이트만 변경하기에, 이후에 - # 필수적으로 `rate` 효과를 기존 샘플 레이트로 주어야합니다. - ["rate", f"{sample_rate1}"], - ["reverb", "-w"], # 잔향은 약간의 극적인 느낌을 줍니다. -] - -# 효과들을 적용합니다. -waveform2, sample_rate2 = torchaudio.sox_effects.apply_effects_tensor(waveform1, sample_rate1, effects) - -print(waveform1.shape, sample_rate1) -print(waveform2.shape, sample_rate2) - -###################################################################### -# 효과가 적용되면, 프레임의 수와 채널의 수는 기존에 적용된 것들과 달라짐에 주의하세요. -# 이제 오디오를 들어봅시다. -# - -def plot_waveform(waveform, sample_rate, title="Waveform", xlim=None): - waveform = waveform.numpy() - - num_channels, num_frames = waveform.shape - time_axis = torch.arange(0, num_frames) / sample_rate - - figure, axes = plt.subplots(num_channels, 1) - if num_channels == 1: - axes = [axes] - for c in range(num_channels): - axes[c].plot(time_axis, waveform[c], linewidth=1) - axes[c].grid(True) - if num_channels > 1: - axes[c].set_ylabel(f"Channel {c+1}") - if xlim: - axes[c].set_xlim(xlim) - figure.suptitle(title) - plt.show(block=False) - -###################################################################### -# - -def plot_specgram(waveform, sample_rate, title="Spectrogram", xlim=None): - waveform = waveform.numpy() - - num_channels, _ = waveform.shape - - figure, axes = plt.subplots(num_channels, 1) - if num_channels == 1: - axes = [axes] - for c in range(num_channels): - axes[c].specgram(waveform[c], Fs=sample_rate) - if num_channels > 1: - axes[c].set_ylabel(f"Channel {c+1}") - if xlim: - axes[c].set_xlim(xlim) - figure.suptitle(title) - plt.show(block=False) - -###################################################################### -# 기존: -# ~~~~~~~~~ -# - -plot_waveform(waveform1, sample_rate1, title="Original", xlim=(-0.1, 3.2)) -plot_specgram(waveform1, sample_rate1, title="Original", xlim=(0, 3.04)) -Audio(waveform1, rate=sample_rate1) - -###################################################################### -# 효과 적용 후: -# ~~~~~~~~~~~~~~~~ -# - -plot_waveform(waveform2, sample_rate2, title="Effects Applied", xlim=(-0.1, 3.2)) -plot_specgram(waveform2, sample_rate2, title="Effects Applied", xlim=(0, 3.04)) -Audio(waveform2, rate=sample_rate2) - -###################################################################### -# 좀 더 극적으로 들리지 않나요? -# - -###################################################################### -# 방 잔향 모의 실험하기 -# ----------------------------- -# -# `Convolution -# reverb `__ 는 -# 깨끗한 오디오를 다른 환경에서 생성된 것처럼 만들어주는 기술입니다. -# -# 예를들어, 공간 임펄스 응답 (RIR)을 활용하여, 깨끗한 음성을 -# 마치 회의실에서 발음된 것처럼 만들 수 있습니다. -# -# 이 과정을 위해서, RIR 데이터가 필요합니다. 다음 데이터들은 VOiCES 데이터셋에서 왔습니다. -# 하지만, 직접 녹음할 수도 있습니다. - 직접 마이크를 켜시고, 박수를 치세요! -# - -rir_raw, sample_rate = torchaudio.load(SAMPLE_RIR) -plot_waveform(rir_raw, sample_rate, title="Room Impulse Response (raw)") -plot_specgram(rir_raw, sample_rate, title="Room Impulse Response (raw)") -Audio(rir_raw, rate=sample_rate) - -###################################################################### -# 먼저, RIR을 깨끗하게 만들어줘야합니다. 주요한 임펄스를 추출하고, -# 신호 전력을 정규화 합니다. 그리고 나서 시간축을 뒤집어 줍니다. -# - -rir = rir_raw[:, int(sample_rate * 1.01) : int(sample_rate * 1.3)] -rir = rir / torch.norm(rir, p=2) -RIR = torch.flip(rir, [1]) - -plot_waveform(rir, sample_rate, title="Room Impulse Response") - -###################################################################### -# 그 후, RIR 필터와 음성 신호를 합성곱 합니다. -# - -speech, _ = torchaudio.load(SAMPLE_SPEECH) - -speech_ = torch.nn.functional.pad(speech, (RIR.shape[1] - 1, 0)) -augmented = torch.nn.functional.conv1d(speech_[None, ...], RIR[None, ...])[0] - -###################################################################### -# 기존: -# ~~~~~~~~~ -# - -plot_waveform(speech, sample_rate, title="Original") -plot_specgram(speech, sample_rate, title="Original") -Audio(speech, rate=sample_rate) - -###################################################################### -# RIR 적용 후: -# ~~~~~~~~~~~~ -# - -plot_waveform(augmented, sample_rate, title="RIR Applied") -plot_specgram(augmented, sample_rate, title="RIR Applied") -Audio(augmented, rate=sample_rate) - - -###################################################################### -# 배경 소음 추가하기 -# ----------------------- -# -# 오디오 데이터에 소음을 추가하기 위해서, 간단히 소음 텐서를 오디오 데이터 텐서에 더할 수 있습니다. -# 소음의 정도를 조절하는 흔한 방법은 신호 대 잡음비 (SNR)를 바꾸는 것입니다. -# [`wikipedia `__] -# -# $$ \\mathrm{SNR} = \\frac{P_{signal}}{P_{noise}} $$ -# -# $$ \\mathrm{SNR_{dB}} = 10 \\log _{{10}} \\mathrm {SNR} $$ -# - -speech, _ = torchaudio.load(SAMPLE_SPEECH) -noise, _ = torchaudio.load(SAMPLE_NOISE) -noise = noise[:, : speech.shape[1]] - -speech_rms = speech.norm(p=2) -noise_rms = noise.norm(p=2) - -snr_dbs = [20, 10, 3] -noisy_speeches = [] -for snr_db in snr_dbs: - snr = 10 ** (snr_db / 20) - scale = snr * noise_rms / speech_rms - noisy_speeches.append((scale * speech + noise) / 2) - -###################################################################### -# 배경 잡음: -# ~~~~~~~~~~~~~~~~~ -# - -plot_waveform(noise, sample_rate, title="Background noise") -plot_specgram(noise, sample_rate, title="Background noise") -Audio(noise, rate=sample_rate) - -###################################################################### -# SNR 20 dB: -# ~~~~~~~~~~ -# - -snr_db, noisy_speech = snr_dbs[0], noisy_speeches[0] -plot_waveform(noisy_speech, sample_rate, title=f"SNR: {snr_db} [dB]") -plot_specgram(noisy_speech, sample_rate, title=f"SNR: {snr_db} [dB]") -Audio(noisy_speech, rate=sample_rate) - -###################################################################### -# SNR 10 dB: -# ~~~~~~~~~~ -# - -snr_db, noisy_speech = snr_dbs[1], noisy_speeches[1] -plot_waveform(noisy_speech, sample_rate, title=f"SNR: {snr_db} [dB]") -plot_specgram(noisy_speech, sample_rate, title=f"SNR: {snr_db} [dB]") -Audio(noisy_speech, rate=sample_rate) - -###################################################################### -# SNR 3 dB: -# ~~~~~~~~~ -# - -snr_db, noisy_speech = snr_dbs[2], noisy_speeches[2] -plot_waveform(noisy_speech, sample_rate, title=f"SNR: {snr_db} [dB]") -plot_specgram(noisy_speech, sample_rate, title=f"SNR: {snr_db} [dB]") -Audio(noisy_speech, rate=sample_rate) - - -###################################################################### -# 코덱을 텐서 객체에 적용하기 -# ------------------------------- -# -# :py:func:`torchaudio.functional.apply_codec` 는 텐서 오브젝트에 코덱을 적용합니다. -# -# **Note** 이 과정은 미분 불가능합니다. -# - - -waveform, sample_rate = torchaudio.load(SAMPLE_SPEECH) - -configs = [ - {"format": "wav", "encoding": "ULAW", "bits_per_sample": 8}, - {"format": "gsm"}, - {"format": "vorbis", "compression": -1}, -] -waveforms = [] -for param in configs: - augmented = F.apply_codec(waveform, sample_rate, **param) - waveforms.append(augmented) - -###################################################################### -# Original: -# ~~~~~~~~~ -# - -plot_waveform(waveform, sample_rate, title="Original") -plot_specgram(waveform, sample_rate, title="Original") -Audio(waveform, rate=sample_rate) - -###################################################################### -# 8 bit mu-law: -# ~~~~~~~~~~~~~ -# - -plot_waveform(waveforms[0], sample_rate, title="8 bit mu-law") -plot_specgram(waveforms[0], sample_rate, title="8 bit mu-law") -Audio(waveforms[0], rate=sample_rate) - -###################################################################### -# GSM-FR: -# ~~~~~~~ -# - -plot_waveform(waveforms[1], sample_rate, title="GSM-FR") -plot_specgram(waveforms[1], sample_rate, title="GSM-FR") -Audio(waveforms[1], rate=sample_rate) - -###################################################################### -# Vorbis: -# ~~~~~~~ -# - -plot_waveform(waveforms[2], sample_rate, title="Vorbis") -plot_specgram(waveforms[2], sample_rate, title="Vorbis") -Audio(waveforms[2], rate=sample_rate) - -###################################################################### -# 전화 녹음 모의 실험하기 -# --------------------------- -# -# 이전 기술들을 혼합하여, 반향 있는 방의 사람들이 이야기하는 배경에서 전화 통화하는 -# 것처럼 들리는 오디오를 모의 실험할 수 있습니다. -# - -sample_rate = 16000 -original_speech, sample_rate = torchaudio.load(SAMPLE_SPEECH) - -plot_specgram(original_speech, sample_rate, title="Original") - -# RIR 적용하기 -speech_ = torch.nn.functional.pad(original_speech, (RIR.shape[1] - 1, 0)) -rir_applied = torch.nn.functional.conv1d(speech_[None, ...], RIR[None, ...])[0] - -plot_specgram(rir_applied, sample_rate, title="RIR Applied") - -# 배경 잡음 추가하기 -# 잡음이 실제 환경에서 녹음되었기 때문에, 잡음이 환경의 음향 특징을 가지고 있다고 고려했습니다. -# 따라서, RIR 적용 후에 잡음을 추가했습니다 -noise, _ = torchaudio.load(SAMPLE_NOISE) -noise = noise[:, : rir_applied.shape[1]] - -snr_db = 8 -scale = (10 ** (snr_db / 20)) * noise.norm(p=2) / rir_applied.norm(p=2) -bg_added = (scale * rir_applied + noise) / 2 - -plot_specgram(bg_added, sample_rate, title="BG noise added") - -# 필터링을 적용하고 샘플 레이트 수정하기 -filtered, sample_rate2 = torchaudio.sox_effects.apply_effects_tensor( - bg_added, - sample_rate, - effects=[ - ["lowpass", "4000"], - [ - "compand", - "0.02,0.05", - "-60,-60,-30,-10,-20,-8,-5,-8,-2,-8", - "-8", - "-7", - "0.05", - ], - ["rate", "8000"], - ], -) - -plot_specgram(filtered, sample_rate2, title="Filtered") - -# 전화 코덱 적용하기 -codec_applied = F.apply_codec(filtered, sample_rate2, format="gsm") - -plot_specgram(codec_applied, sample_rate2, title="GSM Codec Applied") - - -###################################################################### -# 기존 음성: -# ~~~~~~~~~~~~~~~~ -# - -Audio(original_speech, rate=sample_rate) - -###################################################################### -# RIR 적용 후: -# ~~~~~~~~~~~~ -# - -Audio(rir_applied, rate=sample_rate) - -###################################################################### -# 배경 잡음 추가 후: -# ~~~~~~~~~~~~~~~~~~~~~~~ -# - -Audio(bg_added, rate=sample_rate) - -###################################################################### -# 필터링 적용 후: -# ~~~~~~~~~ -# - -Audio(filtered, rate=sample_rate2) - -###################################################################### -# 코덱 적용 후: -# ~~~~~~~~~~~~~ -# - -Audio(codec_applied, rate=sample_rate2) diff --git a/beginner_source/audio_data_augmentation_tutorial.rst b/beginner_source/audio_data_augmentation_tutorial.rst new file mode 100644 index 000000000..55ba024a5 --- /dev/null +++ b/beginner_source/audio_data_augmentation_tutorial.rst @@ -0,0 +1,10 @@ +Audio Data Augmentation +======================= + +This tutorial has been moved to https://pytorch.org/audio/stable/tutorials/audio_data_augmentation_tutorial.html + +It will redirect in 3 seconds. + +.. raw:: html + + diff --git a/beginner_source/audio_datasets_tutorial.py b/beginner_source/audio_datasets_tutorial.py deleted file mode 100644 index f08ed99e0..000000000 --- a/beginner_source/audio_datasets_tutorial.py +++ /dev/null @@ -1,87 +0,0 @@ -# -*- coding: utf-8 -*- -""" -Audio Datasets -============== - -``torchaudio`` provides easy access to common, publicly accessible -datasets. Please refer to the official documentation for the list of -available datasets. -""" - -# When running this tutorial in Google Colab, install the required packages -# with the following. -# !pip install torchaudio - -import torch -import torchaudio - -print(torch.__version__) -print(torchaudio.__version__) - -###################################################################### -# Preparing data and utility functions (skip this section) -# -------------------------------------------------------- -# - -# @title Prepare data and utility functions. {display-mode: "form"} -# @markdown -# @markdown You do not need to look into this cell. -# @markdown Just execute once and you are good to go. - -# ------------------------------------------------------------------------------- -# Preparation of data and helper functions. -# ------------------------------------------------------------------------------- -import multiprocessing -import os - -import matplotlib.pyplot as plt -from IPython.display import Audio, display - - -_SAMPLE_DIR = "_assets" -YESNO_DATASET_PATH = os.path.join(_SAMPLE_DIR, "yes_no") -os.makedirs(YESNO_DATASET_PATH, exist_ok=True) - - -def plot_specgram(waveform, sample_rate, title="Spectrogram", xlim=None): - waveform = waveform.numpy() - - num_channels, num_frames = waveform.shape - - figure, axes = plt.subplots(num_channels, 1) - if num_channels == 1: - axes = [axes] - for c in range(num_channels): - axes[c].specgram(waveform[c], Fs=sample_rate) - if num_channels > 1: - axes[c].set_ylabel(f"Channel {c+1}") - if xlim: - axes[c].set_xlim(xlim) - figure.suptitle(title) - plt.show(block=False) - - -def play_audio(waveform, sample_rate): - waveform = waveform.numpy() - - num_channels, num_frames = waveform.shape - if num_channels == 1: - display(Audio(waveform[0], rate=sample_rate)) - elif num_channels == 2: - display(Audio((waveform[0], waveform[1]), rate=sample_rate)) - else: - raise ValueError("Waveform with more than 2 channels are not supported.") - - -###################################################################### -# Here, we show how to use the -# :py:func:`torchaudio.datasets.YESNO` dataset. -# - - -dataset = torchaudio.datasets.YESNO(YESNO_DATASET_PATH, download=True) - -for i in [1, 3, 5]: - waveform, sample_rate, label = dataset[i] - plot_specgram(waveform, sample_rate, title=f"Sample {i}: {label}") - play_audio(waveform, sample_rate) diff --git a/beginner_source/audio_datasets_tutorial.rst b/beginner_source/audio_datasets_tutorial.rst new file mode 100644 index 000000000..6e9b4f4f4 --- /dev/null +++ b/beginner_source/audio_datasets_tutorial.rst @@ -0,0 +1,10 @@ +Audio Datasets +============== + +This tutorial has been moved to https://pytorch.org/audio/stable/tutorials/audio_datasets_tutorial.html + +It will redirect in 3 seconds. + +.. raw:: html + + diff --git a/beginner_source/audio_feature_augmentation_tutorial.py b/beginner_source/audio_feature_augmentation_tutorial.py deleted file mode 100644 index 3961dafbc..000000000 --- a/beginner_source/audio_feature_augmentation_tutorial.py +++ /dev/null @@ -1,168 +0,0 @@ -# -*- coding: utf-8 -*- -""" -Audio Feature Augmentation -========================== -""" - -# When running this tutorial in Google Colab, install the required packages -# with the following. -# !pip install torchaudio librosa - -import torch -import torchaudio -import torchaudio.transforms as T - -print(torch.__version__) -print(torchaudio.__version__) - -###################################################################### -# Preparing data and utility functions (skip this section) -# -------------------------------------------------------- -# - -# @title Prepare data and utility functions. {display-mode: "form"} -# @markdown -# @markdown You do not need to look into this cell. -# @markdown Just execute once and you are good to go. -# @markdown -# @markdown In this tutorial, we will use a speech data from [VOiCES dataset](https://iqtlabs.github.io/voices/), -# @markdown which is licensed under Creative Commos BY 4.0. - -# ------------------------------------------------------------------------------- -# Preparation of data and helper functions. -# ------------------------------------------------------------------------------- - -import os - -import librosa -import matplotlib.pyplot as plt -import requests - - -_SAMPLE_DIR = "_assets" - -SAMPLE_WAV_SPEECH_URL = "https://pytorch-tutorial-assets.s3.amazonaws.com/VOiCES_devkit/source-16k/train/sp0307/Lab41-SRI-VOiCES-src-sp0307-ch127535-sg0042.wav" # noqa: E501 -SAMPLE_WAV_SPEECH_PATH = os.path.join(_SAMPLE_DIR, "speech.wav") - -os.makedirs(_SAMPLE_DIR, exist_ok=True) - - -def _fetch_data(): - uri = [ - (SAMPLE_WAV_SPEECH_URL, SAMPLE_WAV_SPEECH_PATH), - ] - for url, path in uri: - with open(path, "wb") as file_: - file_.write(requests.get(url).content) - - -_fetch_data() - - -def _get_sample(path, resample=None): - effects = [["remix", "1"]] - if resample: - effects.extend( - [ - ["lowpass", f"{resample // 2}"], - ["rate", f"{resample}"], - ] - ) - return torchaudio.sox_effects.apply_effects_file(path, effects=effects) - - -def get_speech_sample(*, resample=None): - return _get_sample(SAMPLE_WAV_SPEECH_PATH, resample=resample) - - -def get_spectrogram( - n_fft=400, - win_len=None, - hop_len=None, - power=2.0, -): - waveform, _ = get_speech_sample() - spectrogram = T.Spectrogram( - n_fft=n_fft, - win_length=win_len, - hop_length=hop_len, - center=True, - pad_mode="reflect", - power=power, - ) - return spectrogram(waveform) - - -def plot_spectrogram(spec, title=None, ylabel="freq_bin", aspect="auto", xmax=None): - fig, axs = plt.subplots(1, 1) - axs.set_title(title or "Spectrogram (db)") - axs.set_ylabel(ylabel) - axs.set_xlabel("frame") - im = axs.imshow(librosa.power_to_db(spec), origin="lower", aspect=aspect) - if xmax: - axs.set_xlim((0, xmax)) - fig.colorbar(im, ax=axs) - plt.show(block=False) - - -###################################################################### -# SpecAugment -# ----------- -# -# `SpecAugment `__ -# is a popular spectrogram augmentation technique. -# -# ``torchaudio`` implements :py:func:`torchaudio.transforms.TimeStretch`, -# :py:func:`torchaudio.transforms.TimeMasking` and -# :py:func:`torchaudio.transforms.FrequencyMasking`. -# - -###################################################################### -# TimeStretch -# ----------- -# - - -spec = get_spectrogram(power=None) -stretch = T.TimeStretch() - -rate = 1.2 -spec_ = stretch(spec, rate) -plot_spectrogram(torch.abs(spec_[0]), title=f"Stretched x{rate}", aspect="equal", xmax=304) - -plot_spectrogram(torch.abs(spec[0]), title="Original", aspect="equal", xmax=304) - -rate = 0.9 -spec_ = stretch(spec, rate) -plot_spectrogram(torch.abs(spec_[0]), title=f"Stretched x{rate}", aspect="equal", xmax=304) - -###################################################################### -# TimeMasking -# ----------- -# - -torch.random.manual_seed(4) - -spec = get_spectrogram() -plot_spectrogram(spec[0], title="Original") - -masking = T.TimeMasking(time_mask_param=80) -spec = masking(spec) - -plot_spectrogram(spec[0], title="Masked along time axis") - -###################################################################### -# FrequencyMasking -# ---------------- -# - - -torch.random.manual_seed(4) - -spec = get_spectrogram() -plot_spectrogram(spec[0], title="Original") - -masking = T.FrequencyMasking(freq_mask_param=80) -spec = masking(spec) - -plot_spectrogram(spec[0], title="Masked along frequency axis") diff --git a/beginner_source/audio_feature_augmentation_tutorial.rst b/beginner_source/audio_feature_augmentation_tutorial.rst new file mode 100644 index 000000000..55d3811b3 --- /dev/null +++ b/beginner_source/audio_feature_augmentation_tutorial.rst @@ -0,0 +1,10 @@ +Audio Feature Augmentation +========================== + +This tutorial has been moved to https://pytorch.org/audio/stable/tutorials/audio_data_augmentation_tutorial.html + +It will redirect in 3 seconds. + +.. raw:: html + + diff --git a/beginner_source/audio_feature_extractions_tutorial.py b/beginner_source/audio_feature_extractions_tutorial.py deleted file mode 100644 index 822c00d97..000000000 --- a/beginner_source/audio_feature_extractions_tutorial.py +++ /dev/null @@ -1,457 +0,0 @@ -# -*- coding: utf-8 -*- -""" -Audio Feature Extractions -========================= - -``torchaudio`` implements feature extractions commonly used in the audio -domain. They are available in ``torchaudio.functional`` and -``torchaudio.transforms``. - -``functional`` implements features as standalone functions. -They are stateless. - -``transforms`` implements features as objects, -using implementations from ``functional`` and ``torch.nn.Module``. -They can be serialized using TorchScript. -""" - -import torch -import torchaudio -import torchaudio.functional as F -import torchaudio.transforms as T - -print(torch.__version__) -print(torchaudio.__version__) - -###################################################################### -# Preparation -# ----------- -# -# .. note:: -# -# When running this tutorial in Google Colab, install the required packages -# -# .. code:: -# -# !pip install librosa -# -from IPython.display import Audio -import librosa -import matplotlib.pyplot as plt -from torchaudio.utils import download_asset - -torch.random.manual_seed(0) - -SAMPLE_SPEECH = download_asset("tutorial-assets/Lab41-SRI-VOiCES-src-sp0307-ch127535-sg0042.wav") - - -def plot_waveform(waveform, sr, title="Waveform"): - waveform = waveform.numpy() - - num_channels, num_frames = waveform.shape - time_axis = torch.arange(0, num_frames) / sr - - figure, axes = plt.subplots(num_channels, 1) - axes.plot(time_axis, waveform[0], linewidth=1) - axes.grid(True) - figure.suptitle(title) - plt.show(block=False) - - -def plot_spectrogram(specgram, title=None, ylabel="freq_bin"): - fig, axs = plt.subplots(1, 1) - axs.set_title(title or "Spectrogram (db)") - axs.set_ylabel(ylabel) - axs.set_xlabel("frame") - im = axs.imshow(librosa.power_to_db(specgram), origin="lower", aspect="auto") - fig.colorbar(im, ax=axs) - plt.show(block=False) - - -def plot_fbank(fbank, title=None): - fig, axs = plt.subplots(1, 1) - axs.set_title(title or "Filter bank") - axs.imshow(fbank, aspect="auto") - axs.set_ylabel("frequency bin") - axs.set_xlabel("mel bin") - plt.show(block=False) - - -###################################################################### -# Overview of audio features -# -------------------------- -# -# The following diagram shows the relationship between common audio features -# and torchaudio APIs to generate them. -# -# .. image:: https://download.pytorch.org/torchaudio/tutorial-assets/torchaudio_feature_extractions.png -# -# For the complete list of available features, please refer to the -# documentation. -# - - -###################################################################### -# Spectrogram -# ----------- -# -# To get the frequency make-up of an audio signal as it varies with time, -# you can use :py:func:`torchaudio.transforms.Spectrogram`. -# - -SPEECH_WAVEFORM, SAMPLE_RATE = torchaudio.load(SAMPLE_SPEECH) - -plot_waveform(SPEECH_WAVEFORM, SAMPLE_RATE, title="Original waveform") -Audio(SPEECH_WAVEFORM.numpy(), rate=SAMPLE_RATE) - - -###################################################################### -# - -n_fft = 1024 -win_length = None -hop_length = 512 - -# Define transform -spectrogram = T.Spectrogram( - n_fft=n_fft, - win_length=win_length, - hop_length=hop_length, - center=True, - pad_mode="reflect", - power=2.0, -) - -###################################################################### -# - -# Perform transform -spec = spectrogram(SPEECH_WAVEFORM) - -###################################################################### -# - -plot_spectrogram(spec[0], title="torchaudio") - -###################################################################### -# GriffinLim -# ---------- -# -# To recover a waveform from a spectrogram, you can use ``GriffinLim``. -# - -torch.random.manual_seed(0) - -n_fft = 1024 -win_length = None -hop_length = 512 - -spec = T.Spectrogram( - n_fft=n_fft, - win_length=win_length, - hop_length=hop_length, -)(SPEECH_WAVEFORM) - -###################################################################### -# - -griffin_lim = T.GriffinLim( - n_fft=n_fft, - win_length=win_length, - hop_length=hop_length, -) - -###################################################################### -# - -reconstructed_waveform = griffin_lim(spec) - -###################################################################### -# - -plot_waveform(reconstructed_waveform, SAMPLE_RATE, title="Reconstructed") -Audio(reconstructed_waveform, rate=SAMPLE_RATE) - -###################################################################### -# Mel Filter Bank -# --------------- -# -# :py:func:`torchaudio.functional.melscale_fbanks` generates the filter bank -# for converting frequency bins to mel-scale bins. -# -# Since this function does not require input audio/features, there is no -# equivalent transform in :py:func:`torchaudio.transforms`. -# - -n_fft = 256 -n_mels = 64 -sample_rate = 6000 - -mel_filters = F.melscale_fbanks( - int(n_fft // 2 + 1), - n_mels=n_mels, - f_min=0.0, - f_max=sample_rate / 2.0, - sample_rate=sample_rate, - norm="slaney", -) - -###################################################################### -# - -plot_fbank(mel_filters, "Mel Filter Bank - torchaudio") - -###################################################################### -# Comparison against librosa -# ~~~~~~~~~~~~~~~~~~~~~~~~~~ -# -# For reference, here is the equivalent way to get the mel filter bank -# with ``librosa``. -# - -mel_filters_librosa = librosa.filters.mel( - sr=sample_rate, - n_fft=n_fft, - n_mels=n_mels, - fmin=0.0, - fmax=sample_rate / 2.0, - norm="slaney", - htk=True, -).T - -###################################################################### -# - -plot_fbank(mel_filters_librosa, "Mel Filter Bank - librosa") - -mse = torch.square(mel_filters - mel_filters_librosa).mean().item() -print("Mean Square Difference: ", mse) - -###################################################################### -# MelSpectrogram -# -------------- -# -# Generating a mel-scale spectrogram involves generating a spectrogram -# and performing mel-scale conversion. In ``torchaudio``, -# :py:func:`torchaudio.transforms.MelSpectrogram` provides -# this functionality. -# - -n_fft = 1024 -win_length = None -hop_length = 512 -n_mels = 128 - -mel_spectrogram = T.MelSpectrogram( - sample_rate=sample_rate, - n_fft=n_fft, - win_length=win_length, - hop_length=hop_length, - center=True, - pad_mode="reflect", - power=2.0, - norm="slaney", - onesided=True, - n_mels=n_mels, - mel_scale="htk", -) - -melspec = mel_spectrogram(SPEECH_WAVEFORM) - -###################################################################### -# - -plot_spectrogram(melspec[0], title="MelSpectrogram - torchaudio", ylabel="mel freq") - -###################################################################### -# Comparison against librosa -# ~~~~~~~~~~~~~~~~~~~~~~~~~~ -# -# For reference, here is the equivalent means of generating mel-scale -# spectrograms with ``librosa``. -# - -melspec_librosa = librosa.feature.melspectrogram( - y=SPEECH_WAVEFORM.numpy()[0], - sr=sample_rate, - n_fft=n_fft, - hop_length=hop_length, - win_length=win_length, - center=True, - pad_mode="reflect", - power=2.0, - n_mels=n_mels, - norm="slaney", - htk=True, -) - -###################################################################### -# - -plot_spectrogram(melspec_librosa, title="MelSpectrogram - librosa", ylabel="mel freq") - -mse = torch.square(melspec - melspec_librosa).mean().item() -print("Mean Square Difference: ", mse) - -###################################################################### -# MFCC -# ---- -# - -n_fft = 2048 -win_length = None -hop_length = 512 -n_mels = 256 -n_mfcc = 256 - -mfcc_transform = T.MFCC( - sample_rate=sample_rate, - n_mfcc=n_mfcc, - melkwargs={ - "n_fft": n_fft, - "n_mels": n_mels, - "hop_length": hop_length, - "mel_scale": "htk", - }, -) - -mfcc = mfcc_transform(SPEECH_WAVEFORM) - -###################################################################### -# - -plot_spectrogram(mfcc[0]) - -###################################################################### -# Comparison against librosa -# ~~~~~~~~~~~~~~~~~~~~~~~~~~ -# - -melspec = librosa.feature.melspectrogram( - y=SPEECH_WAVEFORM.numpy()[0], - sr=sample_rate, - n_fft=n_fft, - win_length=win_length, - hop_length=hop_length, - n_mels=n_mels, - htk=True, - norm=None, -) - -mfcc_librosa = librosa.feature.mfcc( - S=librosa.core.spectrum.power_to_db(melspec), - n_mfcc=n_mfcc, - dct_type=2, - norm="ortho", -) - -###################################################################### -# - -plot_spectrogram(mfcc_librosa) - -mse = torch.square(mfcc - mfcc_librosa).mean().item() -print("Mean Square Difference: ", mse) - -###################################################################### -# LFCC -# ---- -# - -n_fft = 2048 -win_length = None -hop_length = 512 -n_lfcc = 256 - -lfcc_transform = T.LFCC( - sample_rate=sample_rate, - n_lfcc=n_lfcc, - speckwargs={ - "n_fft": n_fft, - "win_length": win_length, - "hop_length": hop_length, - }, -) - -lfcc = lfcc_transform(SPEECH_WAVEFORM) -plot_spectrogram(lfcc[0]) - -###################################################################### -# Pitch -# ----- -# - -pitch = F.detect_pitch_frequency(SPEECH_WAVEFORM, SAMPLE_RATE) - -###################################################################### -# - -def plot_pitch(waveform, sr, pitch): - figure, axis = plt.subplots(1, 1) - axis.set_title("Pitch Feature") - axis.grid(True) - - end_time = waveform.shape[1] / sr - time_axis = torch.linspace(0, end_time, waveform.shape[1]) - axis.plot(time_axis, waveform[0], linewidth=1, color="gray", alpha=0.3) - - axis2 = axis.twinx() - time_axis = torch.linspace(0, end_time, pitch.shape[1]) - axis2.plot(time_axis, pitch[0], linewidth=2, label="Pitch", color="green") - - axis2.legend(loc=0) - plt.show(block=False) - - -plot_pitch(SPEECH_WAVEFORM, SAMPLE_RATE, pitch) - -###################################################################### -# Kaldi Pitch (beta) -# ------------------ -# -# Kaldi Pitch feature [1] is a pitch detection mechanism tuned for automatic -# speech recognition (ASR) applications. This is a beta feature in ``torchaudio``, -# and it is available as :py:func:`torchaudio.functional.compute_kaldi_pitch`. -# -# 1. A pitch extraction algorithm tuned for automatic speech recognition -# -# Ghahremani, B. BabaAli, D. Povey, K. Riedhammer, J. Trmal and S. -# Khudanpur -# -# 2014 IEEE International Conference on Acoustics, Speech and Signal -# Processing (ICASSP), Florence, 2014, pp. 2494-2498, doi: -# 10.1109/ICASSP.2014.6854049. -# [`abstract `__], -# [`paper `__] -# - -pitch_feature = F.compute_kaldi_pitch(SPEECH_WAVEFORM, SAMPLE_RATE) -pitch, nfcc = pitch_feature[..., 0], pitch_feature[..., 1] - -###################################################################### -# - -def plot_kaldi_pitch(waveform, sr, pitch, nfcc): - _, axis = plt.subplots(1, 1) - axis.set_title("Kaldi Pitch Feature") - axis.grid(True) - - end_time = waveform.shape[1] / sr - time_axis = torch.linspace(0, end_time, waveform.shape[1]) - axis.plot(time_axis, waveform[0], linewidth=1, color="gray", alpha=0.3) - - time_axis = torch.linspace(0, end_time, pitch.shape[1]) - ln1 = axis.plot(time_axis, pitch[0], linewidth=2, label="Pitch", color="green") - axis.set_ylim((-1.3, 1.3)) - - axis2 = axis.twinx() - time_axis = torch.linspace(0, end_time, nfcc.shape[1]) - ln2 = axis2.plot(time_axis, nfcc[0], linewidth=2, label="NFCC", color="blue", linestyle="--") - - lns = ln1 + ln2 - labels = [l.get_label() for l in lns] - axis.legend(lns, labels, loc=0) - plt.show(block=False) - - -plot_kaldi_pitch(SPEECH_WAVEFORM, SAMPLE_RATE, pitch, nfcc) diff --git a/beginner_source/audio_feature_extractions_tutorial.rst b/beginner_source/audio_feature_extractions_tutorial.rst new file mode 100644 index 000000000..a2a8da4ab --- /dev/null +++ b/beginner_source/audio_feature_extractions_tutorial.rst @@ -0,0 +1,10 @@ +Audio Feature Extractions +========================= + +This tutorial has been moved to https://pytorch.org/audio/stable/tutorials/audio_feature_extractions_tutorial.html + +It will redirect in 3 seconds. + +.. raw:: html + + diff --git a/beginner_source/audio_io_tutorial.py b/beginner_source/audio_io_tutorial.py deleted file mode 100644 index 4917f1b10..000000000 --- a/beginner_source/audio_io_tutorial.py +++ /dev/null @@ -1,385 +0,0 @@ -# -*- coding: utf-8 -*- -""" -Audio I/O -========= - -This tutorial shows how to use TorchAudio's basic I/O API to load audio files -into PyTorch's Tensor object, and save Tensor objects to audio files. -""" - -import torch -import torchaudio - -print(torch.__version__) -print(torchaudio.__version__) - -###################################################################### -# Preparation -# ----------- -# -# First, we import the modules and download the audio assets we use in this tutorial. -# -# .. note:: -# When running this tutorial in Google Colab, install the required packages -# with the following: -# -# .. code:: -# -# !pip install boto3 - -import io -import os -import tarfile -import tempfile - -import boto3 -import matplotlib.pyplot as plt -import requests -from botocore import UNSIGNED -from botocore.config import Config -from IPython.display import Audio -from torchaudio.utils import download_asset - -SAMPLE_GSM = download_asset("tutorial-assets/steam-train-whistle-daniel_simon.gsm") -SAMPLE_WAV = download_asset("tutorial-assets/Lab41-SRI-VOiCES-src-sp0307-ch127535-sg0042.wav") -SAMPLE_WAV_8000 = download_asset("tutorial-assets/Lab41-SRI-VOiCES-src-sp0307-ch127535-sg0042-8000hz.wav") - - - -###################################################################### -# Querying audio metadata -# ----------------------- -# -# Function :py:func:`torchaudio.info` fetches audio metadata. -# You can provide a path-like object or file-like object. -# - -metadata = torchaudio.info(SAMPLE_WAV) -print(metadata) - -###################################################################### -# Where -# -# - ``sample_rate`` is the sampling rate of the audio -# - ``num_channels`` is the number of channels -# - ``num_frames`` is the number of frames per channel -# - ``bits_per_sample`` is bit depth -# - ``encoding`` is the sample coding format -# -# ``encoding`` can take on one of the following values: -# -# - ``"PCM_S"``: Signed integer linear PCM -# - ``"PCM_U"``: Unsigned integer linear PCM -# - ``"PCM_F"``: Floating point linear PCM -# - ``"FLAC"``: Flac, `Free Lossless Audio -# Codec `__ -# - ``"ULAW"``: Mu-law, -# [`wikipedia `__] -# - ``"ALAW"``: A-law -# [`wikipedia `__] -# - ``"MP3"`` : MP3, MPEG-1 Audio Layer III -# - ``"VORBIS"``: OGG Vorbis [`xiph.org `__] -# - ``"AMR_NB"``: Adaptive Multi-Rate -# [`wikipedia `__] -# - ``"AMR_WB"``: Adaptive Multi-Rate Wideband -# [`wikipedia `__] -# - ``"OPUS"``: Opus [`opus-codec.org `__] -# - ``"GSM"``: GSM-FR -# [`wikipedia `__] -# - ``"HTK"``: Single channel 16-bit PCM -# - ``"UNKNOWN"`` None of above -# - -###################################################################### -# **Note** -# -# - ``bits_per_sample`` can be ``0`` for formats with compression and/or -# variable bit rate (such as MP3). -# - ``num_frames`` can be ``0`` for GSM-FR format. -# - -metadata = torchaudio.info(SAMPLE_GSM) -print(metadata) - - -###################################################################### -# Querying file-like object -# ------------------------- -# -# :py:func:`torchaudio.info` works on file-like objects. -# - -url = "https://download.pytorch.org/torchaudio/tutorial-assets/steam-train-whistle-daniel_simon.wav" -with requests.get(url, stream=True) as response: - metadata = torchaudio.info(response.raw) -print(metadata) - -###################################################################### -# .. note:: -# -# When passing a file-like object, ``info`` does not read -# all of the underlying data; rather, it reads only a portion -# of the data from the beginning. -# Therefore, for a given audio format, it may not be able to retrieve the -# correct metadata, including the format itself. In such case, you -# can pass ``format`` argument to specify the format of the audio. - -###################################################################### -# Loading audio data -# ------------------ -# -# To load audio data, you can use :py:func:`torchaudio.load`. -# -# This function accepts a path-like object or file-like object as input. -# -# The returned value is a tuple of waveform (``Tensor``) and sample rate -# (``int``). -# -# By default, the resulting tensor object has ``dtype=torch.float32`` and -# its value range is ``[-1.0, 1.0]``. -# -# For the list of supported format, please refer to `the torchaudio -# documentation `__. -# - -waveform, sample_rate = torchaudio.load(SAMPLE_WAV) - - -###################################################################### -# -def plot_waveform(waveform, sample_rate): - waveform = waveform.numpy() - - num_channels, num_frames = waveform.shape - time_axis = torch.arange(0, num_frames) / sample_rate - - figure, axes = plt.subplots(num_channels, 1) - if num_channels == 1: - axes = [axes] - for c in range(num_channels): - axes[c].plot(time_axis, waveform[c], linewidth=1) - axes[c].grid(True) - if num_channels > 1: - axes[c].set_ylabel(f"Channel {c+1}") - figure.suptitle("waveform") - plt.show(block=False) - - -###################################################################### -# -plot_waveform(waveform, sample_rate) - - -###################################################################### -# -def plot_specgram(waveform, sample_rate, title="Spectrogram"): - waveform = waveform.numpy() - - num_channels, num_frames = waveform.shape - - figure, axes = plt.subplots(num_channels, 1) - if num_channels == 1: - axes = [axes] - for c in range(num_channels): - axes[c].specgram(waveform[c], Fs=sample_rate) - if num_channels > 1: - axes[c].set_ylabel(f"Channel {c+1}") - figure.suptitle(title) - plt.show(block=False) - - -###################################################################### -# -plot_specgram(waveform, sample_rate) - - -###################################################################### -# -Audio(waveform.numpy()[0], rate=sample_rate) - -###################################################################### -# Loading from file-like object -# ----------------------------- -# -# The I/O functions support file-like objects. -# This allows for fetching and decoding audio data from locations -# within and beyond the local file system. -# The following examples illustrate this. -# - -###################################################################### -# - -# Load audio data as HTTP request -url = "https://download.pytorch.org/torchaudio/tutorial-assets/Lab41-SRI-VOiCES-src-sp0307-ch127535-sg0042.wav" -with requests.get(url, stream=True) as response: - waveform, sample_rate = torchaudio.load(response.raw) -plot_specgram(waveform, sample_rate, title="HTTP datasource") - -###################################################################### -# - -# Load audio from tar file -tar_path = download_asset("tutorial-assets/VOiCES_devkit.tar.gz") -tar_item = "VOiCES_devkit/source-16k/train/sp0307/Lab41-SRI-VOiCES-src-sp0307-ch127535-sg0042.wav" -with tarfile.open(tar_path, mode="r") as tarfile_: - fileobj = tarfile_.extractfile(tar_item) - waveform, sample_rate = torchaudio.load(fileobj) -plot_specgram(waveform, sample_rate, title="TAR file") - -###################################################################### -# - -# Load audio from S3 -bucket = "pytorch-tutorial-assets" -key = "VOiCES_devkit/source-16k/train/sp0307/Lab41-SRI-VOiCES-src-sp0307-ch127535-sg0042.wav" -client = boto3.client("s3", config=Config(signature_version=UNSIGNED)) -response = client.get_object(Bucket=bucket, Key=key) -waveform, sample_rate = torchaudio.load(response["Body"]) -plot_specgram(waveform, sample_rate, title="From S3") - - -###################################################################### -# Tips on slicing -# --------------- -# -# Providing ``num_frames`` and ``frame_offset`` arguments restricts -# decoding to the corresponding segment of the input. -# -# The same result can be achieved using vanilla Tensor slicing, -# (i.e. ``waveform[:, frame_offset:frame_offset+num_frames]``). However, -# providing ``num_frames`` and ``frame_offset`` arguments is more -# efficient. -# -# This is because the function will end data acquisition and decoding -# once it finishes decoding the requested frames. This is advantageous -# when the audio data are transferred via network as the data transfer will -# stop as soon as the necessary amount of data is fetched. -# -# The following example illustrates this. -# - -# Illustration of two different decoding methods. -# The first one will fetch all the data and decode them, while -# the second one will stop fetching data once it completes decoding. -# The resulting waveforms are identical. - -frame_offset, num_frames = 16000, 16000 # Fetch and decode the 1 - 2 seconds - -url = "https://download.pytorch.org/torchaudio/tutorial-assets/Lab41-SRI-VOiCES-src-sp0307-ch127535-sg0042.wav" -print("Fetching all the data...") -with requests.get(url, stream=True) as response: - waveform1, sample_rate1 = torchaudio.load(response.raw) - waveform1 = waveform1[:, frame_offset : frame_offset + num_frames] - print(f" - Fetched {response.raw.tell()} bytes") - -print("Fetching until the requested frames are available...") -with requests.get(url, stream=True) as response: - waveform2, sample_rate2 = torchaudio.load(response.raw, frame_offset=frame_offset, num_frames=num_frames) - print(f" - Fetched {response.raw.tell()} bytes") - -print("Checking the resulting waveform ... ", end="") -assert (waveform1 == waveform2).all() -print("matched!") - -###################################################################### -# Saving audio to file -# -------------------- -# -# To save audio data in formats interpretable by common applications, -# you can use :py:func:`torchaudio.save`. -# -# This function accepts a path-like object or file-like object. -# -# When passing a file-like object, you also need to provide argument ``format`` -# so that the function knows which format it should use. In the -# case of a path-like object, the function will infer the format from -# the extension. If you are saving to a file without an extension, you need -# to provide argument ``format``. -# -# When saving WAV-formatted data, the default encoding for ``float32`` Tensor -# is 32-bit floating-point PCM. You can provide arguments ``encoding`` and -# ``bits_per_sample`` to change this behavior. For example, to save data -# in 16-bit signed integer PCM, you can do the following. -# -# .. note:: -# -# Saving data in encodings with a lower bit depth reduces the -# resulting file size but also precision. -# - -waveform, sample_rate = torchaudio.load(SAMPLE_WAV) - - -###################################################################### -# - -def inspect_file(path): - print("-" * 10) - print("Source:", path) - print("-" * 10) - print(f" - File size: {os.path.getsize(path)} bytes") - print(f" - {torchaudio.info(path)}") - print() - -###################################################################### -# -# Save without any encoding option. -# The function will pick up the encoding which -# the provided data fit -with tempfile.TemporaryDirectory() as tempdir: - path = f"{tempdir}/save_example_default.wav" - torchaudio.save(path, waveform, sample_rate) - inspect_file(path) - -###################################################################### -# -# Save as 16-bit signed integer Linear PCM -# The resulting file occupies half the storage but loses precision -with tempfile.TemporaryDirectory() as tempdir: - path = f"{tempdir}/save_example_PCM_S16.wav" - torchaudio.save(path, waveform, sample_rate, encoding="PCM_S", bits_per_sample=16) - inspect_file(path) - - -###################################################################### -# :py:func:`torchaudio.save` can also handle other formats. -# To name a few: -# - -formats = [ - "flac", - "vorbis", - "sph", - "amb", - "amr-nb", - "gsm", -] - -###################################################################### -# -waveform, sample_rate = torchaudio.load(SAMPLE_WAV_8000) -with tempfile.TemporaryDirectory() as tempdir: - for format in formats: - path = f"{tempdir}/save_example.{format}" - torchaudio.save(path, waveform, sample_rate, format=format) - inspect_file(path) - -###################################################################### -# Saving to file-like object -# -------------------------- -# -# Similar to the other I/O functions, you can save audio to file-like -# objects. When saving to a file-like object, argument ``format`` is -# required. -# - - -waveform, sample_rate = torchaudio.load(SAMPLE_WAV) - -# Saving to bytes buffer -buffer_ = io.BytesIO() -torchaudio.save(buffer_, waveform, sample_rate, format="wav") - -buffer_.seek(0) -print(buffer_.read(16)) diff --git a/beginner_source/audio_io_tutorial.rst b/beginner_source/audio_io_tutorial.rst new file mode 100644 index 000000000..3263ad93a --- /dev/null +++ b/beginner_source/audio_io_tutorial.rst @@ -0,0 +1,10 @@ +Audio I/O +========= + +This tutorial has been moved to https://pytorch.org/audio/stable/tutorials/audio_io_tutorial.html + +It will redirect in 3 seconds. + +.. raw:: html + + diff --git a/beginner_source/audio_resampling_tutorial.py b/beginner_source/audio_resampling_tutorial.py deleted file mode 100644 index 3ffd73998..000000000 --- a/beginner_source/audio_resampling_tutorial.py +++ /dev/null @@ -1,476 +0,0 @@ -# -*- coding: utf-8 -*- -""" -Audio Resampling -================ - -This tutorial shows how to use torchaudio's resampling API. -""" - -import torch -import torchaudio -import torchaudio.functional as F -import torchaudio.transforms as T - -print(torch.__version__) -print(torchaudio.__version__) - -###################################################################### -# Preparation -# ----------- -# -# First, we import the modules and define the helper functions. -# -# .. note:: -# When running this tutorial in Google Colab, install the required packages -# with the following. -# -# .. code:: -# -# !pip install librosa - -import math -import time - -import librosa -import matplotlib.pyplot as plt -import pandas as pd -from IPython.display import Audio, display - -pd.set_option('display.max_rows', None) -pd.set_option('display.max_columns', None) - -DEFAULT_OFFSET = 201 - - -def _get_log_freq(sample_rate, max_sweep_rate, offset): - """Get freqs evenly spaced out in log-scale, between [0, max_sweep_rate // 2] - - offset is used to avoid negative infinity `log(offset + x)`. - - """ - start, stop = math.log(offset), math.log(offset + max_sweep_rate // 2) - return torch.exp(torch.linspace(start, stop, sample_rate, dtype=torch.double)) - offset - - -def _get_inverse_log_freq(freq, sample_rate, offset): - """Find the time where the given frequency is given by _get_log_freq""" - half = sample_rate // 2 - return sample_rate * (math.log(1 + freq / offset) / math.log(1 + half / offset)) - - -def _get_freq_ticks(sample_rate, offset, f_max): - # Given the original sample rate used for generating the sweep, - # find the x-axis value where the log-scale major frequency values fall in - time, freq = [], [] - for exp in range(2, 5): - for v in range(1, 10): - f = v * 10**exp - if f < sample_rate // 2: - t = _get_inverse_log_freq(f, sample_rate, offset) / sample_rate - time.append(t) - freq.append(f) - t_max = _get_inverse_log_freq(f_max, sample_rate, offset) / sample_rate - time.append(t_max) - freq.append(f_max) - return time, freq - - -def get_sine_sweep(sample_rate, offset=DEFAULT_OFFSET): - max_sweep_rate = sample_rate - freq = _get_log_freq(sample_rate, max_sweep_rate, offset) - delta = 2 * math.pi * freq / sample_rate - cummulative = torch.cumsum(delta, dim=0) - signal = torch.sin(cummulative).unsqueeze(dim=0) - return signal - - -def plot_sweep( - waveform, - sample_rate, - title, - max_sweep_rate=48000, - offset=DEFAULT_OFFSET, -): - x_ticks = [100, 500, 1000, 5000, 10000, 20000, max_sweep_rate // 2] - y_ticks = [1000, 5000, 10000, 20000, sample_rate // 2] - - time, freq = _get_freq_ticks(max_sweep_rate, offset, sample_rate // 2) - freq_x = [f if f in x_ticks and f <= max_sweep_rate // 2 else None for f in freq] - freq_y = [f for f in freq if f in y_ticks and 1000 <= f <= sample_rate // 2] - - figure, axis = plt.subplots(1, 1) - _, _, _, cax = axis.specgram(waveform[0].numpy(), Fs=sample_rate) - plt.xticks(time, freq_x) - plt.yticks(freq_y, freq_y) - axis.set_xlabel("Original Signal Frequency (Hz, log scale)") - axis.set_ylabel("Waveform Frequency (Hz)") - axis.xaxis.grid(True, alpha=0.67) - axis.yaxis.grid(True, alpha=0.67) - figure.suptitle(f"{title} (sample rate: {sample_rate} Hz)") - plt.colorbar(cax) - plt.show(block=True) - - -###################################################################### -# Resampling Overview -# ------------------- -# -# To resample an audio waveform from one freqeuncy to another, you can use -# :py:func:`torchaudio.transforms.Resample` or -# :py:func:`torchaudio.functional.resample`. -# ``transforms.Resample`` precomputes and caches the kernel used for resampling, -# while ``functional.resample`` computes it on the fly, so using -# ``torchaudio.transforms.Resample`` will result in a speedup when resampling -# multiple waveforms using the same parameters (see Benchmarking section). -# -# Both resampling methods use `bandlimited sinc -# interpolation `__ to compute -# signal values at arbitrary time steps. The implementation involves -# convolution, so we can take advantage of GPU / multithreading for -# performance improvements. -# -# .. note:: -# -# When using resampling in multiple subprocesses, such as data loading -# with multiple worker processes, your application might create more -# threads than your system can handle efficiently. -# Setting ``torch.set_num_threads(1)`` might help in this case. -# -# Because a finite number of samples can only represent a finite number of -# frequencies, resampling does not produce perfect results, and a variety -# of parameters can be used to control for its quality and computational -# speed. We demonstrate these properties through resampling a logarithmic -# sine sweep, which is a sine wave that increases exponentially in -# frequency over time. -# -# The spectrograms below show the frequency representation of the signal, -# where the x-axis corresponds to the frequency of the original -# waveform (in log scale), y-axis the frequency of the -# plotted waveform, and color intensity the amplitude. -# - -sample_rate = 48000 -waveform = get_sine_sweep(sample_rate) - -plot_sweep(waveform, sample_rate, title="Original Waveform") -Audio(waveform.numpy()[0], rate=sample_rate) - -###################################################################### -# -# Now we resample (downsample) it. -# -# We see that in the spectrogram of the resampled waveform, there is an -# artifact, which was not present in the original waveform. - -resample_rate = 32000 -resampler = T.Resample(sample_rate, resample_rate, dtype=waveform.dtype) -resampled_waveform = resampler(waveform) - -plot_sweep(resampled_waveform, resample_rate, title="Resampled Waveform") -Audio(resampled_waveform.numpy()[0], rate=resample_rate) - -###################################################################### -# Controling resampling quality with parameters -# --------------------------------------------- -# -# Lowpass filter width -# ~~~~~~~~~~~~~~~~~~~~ -# -# Because the filter used for interpolation extends infinitely, the -# ``lowpass_filter_width`` parameter is used to control for the width of -# the filter to use to window the interpolation. It is also referred to as -# the number of zero crossings, since the interpolation passes through -# zero at every time unit. Using a larger ``lowpass_filter_width`` -# provides a sharper, more precise filter, but is more computationally -# expensive. -# - -sample_rate = 48000 -resample_rate = 32000 - -resampled_waveform = F.resample(waveform, sample_rate, resample_rate, lowpass_filter_width=6) -plot_sweep(resampled_waveform, resample_rate, title="lowpass_filter_width=6") - -###################################################################### -# - -resampled_waveform = F.resample(waveform, sample_rate, resample_rate, lowpass_filter_width=128) -plot_sweep(resampled_waveform, resample_rate, title="lowpass_filter_width=128") - -###################################################################### -# Rolloff -# ~~~~~~~ -# -# The ``rolloff`` parameter is represented as a fraction of the Nyquist -# frequency, which is the maximal frequency representable by a given -# finite sample rate. ``rolloff`` determines the lowpass filter cutoff and -# controls the degree of aliasing, which takes place when frequencies -# higher than the Nyquist are mapped to lower frequencies. A lower rolloff -# will therefore reduce the amount of aliasing, but it will also reduce -# some of the higher frequencies. -# - - -sample_rate = 48000 -resample_rate = 32000 - -resampled_waveform = F.resample(waveform, sample_rate, resample_rate, rolloff=0.99) -plot_sweep(resampled_waveform, resample_rate, title="rolloff=0.99") - -###################################################################### -# - -resampled_waveform = F.resample(waveform, sample_rate, resample_rate, rolloff=0.8) -plot_sweep(resampled_waveform, resample_rate, title="rolloff=0.8") - - -###################################################################### -# Window function -# ~~~~~~~~~~~~~~~ -# -# By default, ``torchaudio``’s resample uses the Hann window filter, which is -# a weighted cosine function. It additionally supports the Kaiser window, -# which is a near optimal window function that contains an additional -# ``beta`` parameter that allows for the design of the smoothness of the -# filter and width of impulse. This can be controlled using the -# ``resampling_method`` parameter. -# - - -sample_rate = 48000 -resample_rate = 32000 - -resampled_waveform = F.resample(waveform, sample_rate, resample_rate, resampling_method="sinc_interpolation") -plot_sweep(resampled_waveform, resample_rate, title="Hann Window Default") - -###################################################################### -# - -resampled_waveform = F.resample(waveform, sample_rate, resample_rate, resampling_method="kaiser_window") -plot_sweep(resampled_waveform, resample_rate, title="Kaiser Window Default") - - -###################################################################### -# Comparison against librosa -# -------------------------- -# -# ``torchaudio``’s resample function can be used to produce results similar to -# that of librosa (resampy)’s kaiser window resampling, with some noise -# - -sample_rate = 48000 -resample_rate = 32000 - -###################################################################### -# kaiser_best -# ~~~~~~~~~~~ -# -resampled_waveform = F.resample( - waveform, - sample_rate, - resample_rate, - lowpass_filter_width=64, - rolloff=0.9475937167399596, - resampling_method="kaiser_window", - beta=14.769656459379492, -) -plot_sweep(resampled_waveform, resample_rate, title="Kaiser Window Best (torchaudio)") - -###################################################################### -# - -librosa_resampled_waveform = torch.from_numpy( - librosa.resample(waveform.squeeze().numpy(), orig_sr=sample_rate, target_sr=resample_rate, res_type="kaiser_best") -).unsqueeze(0) -plot_sweep(librosa_resampled_waveform, resample_rate, title="Kaiser Window Best (librosa)") - -###################################################################### -# - -mse = torch.square(resampled_waveform - librosa_resampled_waveform).mean().item() -print("torchaudio and librosa kaiser best MSE:", mse) - -###################################################################### -# kaiser_fast -# ~~~~~~~~~~~ -# -resampled_waveform = F.resample( - waveform, - sample_rate, - resample_rate, - lowpass_filter_width=16, - rolloff=0.85, - resampling_method="kaiser_window", - beta=8.555504641634386, -) -plot_sweep(resampled_waveform, resample_rate, title="Kaiser Window Fast (torchaudio)") - -###################################################################### -# - -librosa_resampled_waveform = torch.from_numpy( - librosa.resample(waveform.squeeze().numpy(), orig_sr=sample_rate, target_sr=resample_rate, res_type="kaiser_fast") -).unsqueeze(0) -plot_sweep(librosa_resampled_waveform, resample_rate, title="Kaiser Window Fast (librosa)") - -###################################################################### -# - -mse = torch.square(resampled_waveform - librosa_resampled_waveform).mean().item() -print("torchaudio and librosa kaiser fast MSE:", mse) - -###################################################################### -# Performance Benchmarking -# ------------------------ -# -# Below are benchmarks for downsampling and upsampling waveforms between -# two pairs of sampling rates. We demonstrate the performance implications -# that the ``lowpass_filter_wdith``, window type, and sample rates can -# have. Additionally, we provide a comparison against ``librosa``\ ’s -# ``kaiser_best`` and ``kaiser_fast`` using their corresponding parameters -# in ``torchaudio``. -# -# To elaborate on the results: -# -# - a larger ``lowpass_filter_width`` results in a larger resampling kernel, -# and therefore increases computation time for both the kernel computation -# and convolution -# - using ``kaiser_window`` results in longer computation times than the default -# ``sinc_interpolation`` because it is more complex to compute the intermediate -# window values - a large GCD between the sample and resample rate will result -# in a simplification that allows for a smaller kernel and faster kernel computation. -# - - -def benchmark_resample( - method, - waveform, - sample_rate, - resample_rate, - lowpass_filter_width=6, - rolloff=0.99, - resampling_method="sinc_interpolation", - beta=None, - librosa_type=None, - iters=5, -): - if method == "functional": - begin = time.monotonic() - for _ in range(iters): - F.resample( - waveform, - sample_rate, - resample_rate, - lowpass_filter_width=lowpass_filter_width, - rolloff=rolloff, - resampling_method=resampling_method, - ) - elapsed = time.monotonic() - begin - return elapsed / iters - elif method == "transforms": - resampler = T.Resample( - sample_rate, - resample_rate, - lowpass_filter_width=lowpass_filter_width, - rolloff=rolloff, - resampling_method=resampling_method, - dtype=waveform.dtype, - ) - begin = time.monotonic() - for _ in range(iters): - resampler(waveform) - elapsed = time.monotonic() - begin - return elapsed / iters - elif method == "librosa": - waveform_np = waveform.squeeze().numpy() - begin = time.monotonic() - for _ in range(iters): - librosa.resample(waveform_np, orig_sr=sample_rate, target_sr=resample_rate, res_type=librosa_type) - elapsed = time.monotonic() - begin - return elapsed / iters - - -###################################################################### -# - -configs = { - "downsample (48 -> 44.1 kHz)": [48000, 44100], - "downsample (16 -> 8 kHz)": [16000, 8000], - "upsample (44.1 -> 48 kHz)": [44100, 48000], - "upsample (8 -> 16 kHz)": [8000, 16000], -} - -for label in configs: - times, rows = [], [] - sample_rate = configs[label][0] - resample_rate = configs[label][1] - waveform = get_sine_sweep(sample_rate) - - # sinc 64 zero-crossings - f_time = benchmark_resample("functional", waveform, sample_rate, resample_rate, lowpass_filter_width=64) - t_time = benchmark_resample("transforms", waveform, sample_rate, resample_rate, lowpass_filter_width=64) - times.append([None, 1000 * f_time, 1000 * t_time]) - rows.append("sinc (width 64)") - - # sinc 6 zero-crossings - f_time = benchmark_resample("functional", waveform, sample_rate, resample_rate, lowpass_filter_width=16) - t_time = benchmark_resample("transforms", waveform, sample_rate, resample_rate, lowpass_filter_width=16) - times.append([None, 1000 * f_time, 1000 * t_time]) - rows.append("sinc (width 16)") - - # kaiser best - lib_time = benchmark_resample("librosa", waveform, sample_rate, resample_rate, librosa_type="kaiser_best") - f_time = benchmark_resample( - "functional", - waveform, - sample_rate, - resample_rate, - lowpass_filter_width=64, - rolloff=0.9475937167399596, - resampling_method="kaiser_window", - beta=14.769656459379492, - ) - t_time = benchmark_resample( - "transforms", - waveform, - sample_rate, - resample_rate, - lowpass_filter_width=64, - rolloff=0.9475937167399596, - resampling_method="kaiser_window", - beta=14.769656459379492, - ) - times.append([1000 * lib_time, 1000 * f_time, 1000 * t_time]) - rows.append("kaiser_best") - - # kaiser fast - lib_time = benchmark_resample("librosa", waveform, sample_rate, resample_rate, librosa_type="kaiser_fast") - f_time = benchmark_resample( - "functional", - waveform, - sample_rate, - resample_rate, - lowpass_filter_width=16, - rolloff=0.85, - resampling_method="kaiser_window", - beta=8.555504641634386, - ) - t_time = benchmark_resample( - "transforms", - waveform, - sample_rate, - resample_rate, - lowpass_filter_width=16, - rolloff=0.85, - resampling_method="kaiser_window", - beta=8.555504641634386, - ) - times.append([1000 * lib_time, 1000 * f_time, 1000 * t_time]) - rows.append("kaiser_fast") - - df = pd.DataFrame(times, columns=["librosa", "functional", "transforms"], index=rows) - df.columns = pd.MultiIndex.from_product([[f"{label} time (ms)"], df.columns]) - - print(f"torchaudio: {torchaudio.__version__}") - print(f"librosa: {librosa.__version__}") - display(df.round(2)) diff --git a/beginner_source/audio_resampling_tutorial.rst b/beginner_source/audio_resampling_tutorial.rst new file mode 100644 index 000000000..01210830e --- /dev/null +++ b/beginner_source/audio_resampling_tutorial.rst @@ -0,0 +1,9 @@ +Audio Resampling +================ + +This tutorial has been moved to `a new location `_ +You will be redirected in 3 seconds. + +.. raw:: html + + diff --git a/beginner_source/basics/autogradqs_tutorial.py b/beginner_source/basics/autogradqs_tutorial.py index 2ea815895..a9f162aad 100644 --- a/beginner_source/basics/autogradqs_tutorial.py +++ b/beginner_source/basics/autogradqs_tutorial.py @@ -117,9 +117,7 @@ ###################################################################### # 변화도 추적을 멈춰야 하는 이유들은 다음과 같습니다: -# - 신경망의 일부 매개변수를 **고정된 매개변수(frozen parameter)**\ 로 표시합니다. 이는 -# `사전 학습된 신경망을 미세조정 `__ -# 할 때 매우 일반적인 시나리오입니다. +# - 신경망의 일부 매개변수를 **고정된 매개변수(frozen parameter)**\ 로 표시합니다. # - 변화도를 추적하지 않는 텐서의 연산이 더 효율적이기 때문에, 순전파 단계만 수행할 때 # **연산 속도가 향상됩니다.** diff --git a/beginner_source/basics/buildmodel_tutorial.py b/beginner_source/basics/buildmodel_tutorial.py index e6dc90bc5..ae80f4e10 100644 --- a/beginner_source/basics/buildmodel_tutorial.py +++ b/beginner_source/basics/buildmodel_tutorial.py @@ -32,11 +32,17 @@ # 학습을 위한 장치 얻기 # ------------------------------------------------------------------------------------------ # -# 가능한 경우 GPU와 같은 하드웨어 가속기에서 모델을 학습하려고 합니다. -# `torch.cuda `_ 를 사용할 수 있는지 -# 확인하고 그렇지 않으면 CPU를 계속 사용합니다. - -device = "cuda" if torch.cuda.is_available() else "cpu" +# 가능한 경우 GPU 또는 MPS와 같은 하드웨어 가속기에서 모델을 학습하려고 합니다. +# `torch.cuda `_ 또는 `torch.backends.mps `_ +# 가 사용 가능한지 확인해보고, 그렇지 않으면 CPU를 계속 사용합니다. + +device = ( + "cuda" + if torch.cuda.is_available() + else "mps" + if torch.backends.mps.is_available() + else "cpu" +) print(f"Using {device} device") ############################################## @@ -48,7 +54,7 @@ class NeuralNetwork(nn.Module): def __init__(self): - super(NeuralNetwork, self).__init__() + super().__init__() self.flatten = nn.Flatten() self.linear_relu_stack = nn.Sequential( nn.Linear(28*28, 512), diff --git a/beginner_source/basics/intro.py b/beginner_source/basics/intro.py index 4207bcccd..2aae4d0bb 100644 --- a/beginner_source/basics/intro.py +++ b/beginner_source/basics/intro.py @@ -36,7 +36,7 @@ 다음의 두 가지 방법으로 이 튜토리얼을 실행해볼 수 있습니다: -- **클라우드**: 시작하기 가장 쉬운 방법입니다! 각 섹션의 맨 위에는 "Run in Microsoft Learn" 링크가 있으며, 이 링크는 완전히 호스팅되는 환경에서 Microsoft Learn의 노트북을 엽니다. +- **클라우드**: 시작하기 가장 쉬운 방법입니다! 각 섹션의 맨 위에는 "Run in Microsoft Learn" 링크와 "Run in Google Colab" 링크가 있으며, 이 링크들은 각각 완전히 호스팅되는 환경에서 Microsoft Learn 또는 Google Colab의 노트북을 엽니다. - **로컬**: 먼저 로컬 컴퓨터에 PyTorch와 TorchVision을 설치해야 합니다 (`설치 방법 `_). 노트북을 내려받거나 코드를 원하는 IDE에 복사하세요. diff --git a/beginner_source/basics/optimization_tutorial.py b/beginner_source/basics/optimization_tutorial.py index b459bf649..67ade9650 100644 --- a/beginner_source/basics/optimization_tutorial.py +++ b/beginner_source/basics/optimization_tutorial.py @@ -13,7 +13,7 @@ ========================================================================== 이제 모델과 데이터가 준비되었으니, 데이터에 매개변수를 최적화하여 모델을 학습하고, 검증하고, 테스트할 차례입니다. -모델을 학습하는 과정은 반복적인 과정을 거칩니다; (*에폭(epoch)*\ 이라고 부르는) 각 반복 단계에서 모델은 출력을 추측하고, +모델을 학습하는 과정은 반복적인 과정을 거칩니다; 각 반복 단계에서 모델은 출력을 추측하고, 추측과 정답 사이의 오류(\ *손실(loss)*\ )를 계산하고, (`이전 장 `_\ 에서 본 것처럼) 매개변수에 대한 오류의 도함수(derivative)를 수집한 뒤, 경사하강법을 사용하여 이 파라미터들을 **최적화(optimize)**\ 합니다. 이 과정에 대한 자세한 설명은 `3Blue1Brown의 역전파 `__ 영상을 참고하세요. @@ -160,7 +160,7 @@ def train_loop(dataloader, model, loss_fn, optimizer): optimizer.step() if batch % 100 == 0: - loss, current = loss.item(), batch * len(X) + loss, current = loss.item(), (batch + 1) * len(X) print(f"loss: {loss:>7f} [{current:>5d}/{size:>5d}]") diff --git a/beginner_source/basics/quickstart_tutorial.py b/beginner_source/basics/quickstart_tutorial.py index c7f7d9a33..fb36ac1b2 100644 --- a/beginner_source/basics/quickstart_tutorial.py +++ b/beginner_source/basics/quickstart_tutorial.py @@ -82,10 +82,16 @@ # ------------------------------------------------------------------------------------------ # PyTorch에서 신경망 모델은 `nn.Module `_ 을 # 상속받는 클래스(class)를 생성하여 정의합니다. ``__init__`` 함수에서 신경망의 계층(layer)들을 정의하고 ``forward`` 함수에서 -# 신경망에 데이터를 어떻게 전달할지 지정합니다. 가능한 경우 GPU로 신경망을 이동시켜 연산을 가속(accelerate)합니다. - -# 학습에 사용할 CPU나 GPU 장치를 얻습니다. -device = "cuda" if torch.cuda.is_available() else "cpu" +# 신경망에 데이터를 어떻게 전달할지 지정합니다. 가능한 경우 GPU 또는 MPS로 신경망을 이동시켜 연산을 가속(accelerate)합니다. + +# 학습에 사용할 CPU나 GPU, MPS 장치를 얻습니다. +device = ( + "cuda" + if torch.cuda.is_available() + else "mps" + if torch.backends.mps.is_available() + else "cpu" +) print(f"Using {device} device") # 모델을 정의합니다. @@ -148,7 +154,7 @@ def train(dataloader, model, loss_fn, optimizer): optimizer.step() if batch % 100 == 0: - loss, current = loss.item(), batch * len(X) + loss, current = loss.item(), (batch + 1) * len(X) print(f"loss: {loss:>7f} [{current:>5d}/{size:>5d}]") ############################################################################## @@ -205,7 +211,7 @@ def test(dataloader, model, loss_fn): # # 모델을 불러오는 과정에는 모델 구조를 다시 만들고 상태 사전을 모델에 불러오는 과정이 포함됩니다. -model = NeuralNetwork() +model = NeuralNetwork().to(device) model.load_state_dict(torch.load("model.pth")) ############################################################# @@ -227,6 +233,7 @@ def test(dataloader, model, loss_fn): model.eval() x, y = test_data[0][0], test_data[0][1] with torch.no_grad(): + x = x.to(device) pred = model(x) predicted, actual = classes[pred[0].argmax(0)], classes[y] print(f'Predicted: "{predicted}", Actual: "{actual}"') diff --git a/beginner_source/basics/saveloadrun_tutorial.py b/beginner_source/basics/saveloadrun_tutorial.py index e94c0119d..cff004d14 100644 --- a/beginner_source/basics/saveloadrun_tutorial.py +++ b/beginner_source/basics/saveloadrun_tutorial.py @@ -26,14 +26,14 @@ # PyTorch 모델은 학습한 매개변수를 ``state_dict``\ 라고 불리는 내부 상태 사전(internal state dictionary)에 저장합니다. # 이 상태 값들은 ``torch.save`` 메소드를 사용하여 저장(persist)할 수 있습니다: -model = models.vgg16(pretrained=True) +model = models.vgg16(weights='IMAGENET1K_V1') torch.save(model.state_dict(), 'model_weights.pth') ########################## # 모델 가중치를 불러오기 위해서는, 먼저 동일한 모델의 인스턴스(instance)를 생성한 다음에 ``load_state_dict()`` 메소드를 사용하여 # 매개변수들을 불러옵니다. -model = models.vgg16() # 기본 가중치를 불러오지 않으므로 pretrained=True를 지정하지 않습니다. +model = models.vgg16() # 여기서는 ``weights`` 를 지정하지 않았으므로, 학습되지 않은 모델을 생성합니다. model.load_state_dict(torch.load('model_weights.pth')) model.eval() diff --git a/beginner_source/basics/tensorqs_tutorial.py b/beginner_source/basics/tensorqs_tutorial.py index 9dff0dfe4..04c4fd50c 100644 --- a/beginner_source/basics/tensorqs_tutorial.py +++ b/beginner_source/basics/tensorqs_tutorial.py @@ -142,6 +142,7 @@ # **산술 연산(Arithmetic operations)** # 두 텐서 간의 행렬 곱(matrix multiplication)을 계산합니다. y1, y2, y3은 모두 같은 값을 갖습니다. +# ``tensor.T`` 는 텐서의 전치(transpose)를 반환합니다. y1 = tensor @ tensor.T y2 = tensor.matmul(tensor.T) diff --git a/beginner_source/bettertransformer_tutorial.rst b/beginner_source/bettertransformer_tutorial.rst index 10302331b..96249d886 100644 --- a/beginner_source/bettertransformer_tutorial.rst +++ b/beginner_source/bettertransformer_tutorial.rst @@ -18,7 +18,7 @@ been updated to use the core library modules to benefit from fastpath accelerati Better Transformer offers two types of acceleration: -* Native multihead attention implementation for CPU and GPU to improvee overall execution efficiency. +* Native multihead attention (MHA) implementation for CPU and GPU to improve overall execution efficiency. * Exploiting sparsity in NLP inference. Because of variable input lengths, input tokens may contain a large number of padding tokens for which processing may be skipped, delivering significant speedups. @@ -124,6 +124,7 @@ Finally, we set the benchmark iteration count: 2.1 Run and benchmark inference on CPU with and without BT fastpath (native MHA only) We run the model on CPU, and collect profile information: + * The first run uses traditional ("slow path") execution. * The second run enables BT fastpath execution by putting the model in inference mode using `model.eval()` and disables gradient collection with `torch.no_grad()`. @@ -167,6 +168,7 @@ We disable the BT sparsity: We run the model on DEVICE, and collect profile information for native MHA execution on DEVICE: + * The first run uses traditional ("slow path") execution. * The second run enables BT fastpath execution by putting the model in inference mode using `model.eval()` and disables gradient collection with `torch.no_grad()`. diff --git a/beginner_source/blitz/autograd_tutorial.py b/beginner_source/blitz/autograd_tutorial.py index 9948320f5..e377ce5c4 100644 --- a/beginner_source/blitz/autograd_tutorial.py +++ b/beginner_source/blitz/autograd_tutorial.py @@ -268,10 +268,6 @@ # 이러한 매개변수의 변화도가 필요하지 않다는 것을 미리 알고 있으면, 신경망 모델의 일부를 "고정(freeze)"하는 것이 유용합니다. # (이렇게 하면 autograd 연산량을 줄임으로써 성능 상의 이득을 제공합니다.) # -# DAG에서 제외하는 것이 중요한 또 다른 일반적인 사례(usecase)는 -# `미리 학습된 모델을 미세조정 `__ -# 하는 경우입니다. -# # 미세조정(finetuning)을 하는 과정에서, 새로운 정답(label)을 예측할 수 있도록 모델의 대부분을 고정한 뒤 일반적으로 분류 계층(classifier layer)만 변경합니다. # 이를 설명하기 위해 간단한 예제를 살펴보겠습니다. 이전과 마찬가지로 이미 학습된 resnet18 모델을 불러온 뒤 모든 매개변수를 고정합니다. diff --git a/beginner_source/blitz/neural_networks_tutorial.py b/beginner_source/blitz/neural_networks_tutorial.py index 8b86d65c7..bed2d8e4c 100644 --- a/beginner_source/blitz/neural_networks_tutorial.py +++ b/beginner_source/blitz/neural_networks_tutorial.py @@ -130,7 +130,7 @@ def forward(self, x): # - 신경망의 가중치를 갱신하는 것 # # 손실 함수 (Loss Function) -# ------------------------- +# --------------------------- # 손실 함수는 (output, target)을 한 쌍(pair)의 입력으로 받아, 출력(output)이 # 정답(target)으로부터 얼마나 멀리 떨어져 있는지 추정하는 값을 계산합니다. # @@ -172,7 +172,7 @@ def forward(self, x): ######################################################################## # 역전파(Backprop) -# ---------------- +# ------------------ # 오차(error)를 역전파하기 위해서는 ``loss.backward()`` 만 해주면 됩니다. # 기존에 계산된 변화도의 값을 누적 시키고 싶지 않다면 기존에 계산된 변화도를 0으로 만드는 # 작업이 필요합니다. @@ -210,7 +210,10 @@ def forward(self, x): # 실제로 많이 사용되는 가장 단순한 갱신 규칙은 확률적 경사하강법(SGD; Stochastic # Gradient Descent)입니다: # -# ``새로운 가중치(weight) = 가중치(weight) - 학습률(learning rate) * 변화도(gradient)`` +# .. code:: python +# +# # 새로운 가중치 = 가중치 - 학습률 * 변화도 +# weight = weight - learning_rate * gradient # # 간단한 Python 코드로 이를 구현해볼 수 있습니다: # @@ -223,18 +226,21 @@ def forward(self, x): # 신경망을 구성할 때 SGD, Nesterov-SGD, Adam, RMSProp 등과 같은 다양한 갱신 규칙을 # 사용하고 싶을 수 있습니다. 이를 위해서 ``torch.optim`` 라는 작은 패키지에 이러한 # 방법들을 모두 구현해두었습니다. 사용법은 매우 간단합니다: - -import torch.optim as optim - -# Optimizer를 생성합니다. -optimizer = optim.SGD(net.parameters(), lr=0.01) - -# 학습 과정(training loop)은 다음과 같습니다: -optimizer.zero_grad() # 변화도 버퍼를 0으로 -output = net(input) -loss = criterion(output, target) -loss.backward() -optimizer.step() # 업데이트 진행 +# +# .. code:: python +# +# import torch.optim as optim +# +# # Optimizer를 생성합니다. +# optimizer = optim.SGD(net.parameters(), lr=0.01) +# +# # 학습 과정(training loop)은 다음과 같습니다: +# optimizer.zero_grad() # 변화도 버퍼를 0으로 +# output = net(input) +# loss = criterion(output, target) +# loss.backward() +# optimizer.step() # 업데이트 진행 +# ############################################################### diff --git a/beginner_source/chatbot_tutorial.py b/beginner_source/chatbot_tutorial.py index e16489ede..6244d8536 100644 --- a/beginner_source/chatbot_tutorial.py +++ b/beginner_source/chatbot_tutorial.py @@ -35,28 +35,28 @@ # :align: center # :alt: bot # -# .. code:: python -# -# > hello? (안녕하세요?) -# Bot: hello . (안녕하세요.) -# > where am I? (여긴 어디죠?) -# Bot: you re in a hospital . (병원입니다.) -# > who are you? (당신은 누구시죠?) -# Bot: i m a lawyer . (변호사입니다.) -# > how are you doing? (어떻게 지내세요?) -# Bot: i m fine . (잘 지냅니다.) -# > are you my friend? (당신은 제 친구인가요?) -# Bot: no . (아뇨.) -# > you're under arrest (당신을 체포하겠습니다) -# Bot: i m trying to help you ! (난 당신을 도우려 하는 겁니다!) -# > i'm just kidding (농담이었어요) -# Bot: i m sorry . (미안하네요.) -# > where are you from? (어디서 오셨어요?) -# Bot: san francisco . (샌프란시스코요.) -# > it's time for me to leave (전 이제 가봐야겠네요) -# Bot: i know . (알겠습니다.) -# > goodbye (안녕히 계세요) -# Bot: goodbye . (안녕히 가세요.) +# .. code-block:: python +# +# > hello? (안녕하세요?) +# Bot: hello . (안녕하세요.) +# > where am I? (여긴 어디죠?) +# Bot: you re in a hospital . (병원입니다.) +# > who are you? (당신은 누구시죠?) +# Bot: i m a lawyer . (변호사입니다.) +# > how are you doing? (어떻게 지내세요?) +# Bot: i m fine . (잘 지냅니다.) +# > are you my friend? (당신은 제 친구인가요?) +# Bot: no . (아뇨.) +# > you're under arrest (당신을 체포하겠습니다) +# Bot: i m trying to help you ! (난 당신을 도우려 하는 겁니다!) +# > i'm just kidding (농담이었어요) +# Bot: i m sorry . (미안하네요.) +# > where are you from? (어디서 오셨어요?) +# Bot: san francisco . (샌프란시스코요.) +# > it's time for me to leave (전 이제 가봐야겠네요) +# Bot: i know . (알겠습니다.) +# > goodbye (안녕히 계세요) +# Bot: goodbye . (안녕히 가세요.) # # **이 튜토리얼의 핵심 내용** # @@ -86,7 +86,7 @@ ###################################################################### # 준비 단계 -# --------- +# ----------- # # 시작에 앞서, `여기 `__ 에서 # ZIP 파일 형태의 데이터를 내려받고, 현재 디렉토리 아래에 ``data/`` 라는 @@ -123,7 +123,7 @@ ###################################################################### # 데이터 읽기 & 전처리하기 -# ------------------------ +# -------------------------- # # 다음 단계는 데이터 파일의 형식을 재조정한 후, 우리가 작업하기 편한 # 구조로 읽어들이는 것입니다. @@ -159,21 +159,21 @@ def printLines(file, n=10): ###################################################################### # 원하는 형식의 데이터 파일로 만들기 -# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ # # 편의를 위해 데이터의 형식을 원하는 형태로 만들려고 합니다. 각 줄에 # *질의 문장* 과 *응답 문장* 의 쌍이 탭으로 구분되어 있게끔 하는 것입니다. # -# 다음의 함수를 통해 *utterances.jsonl* 원본 데이터 파일을 파싱하려 +# 다음의 함수를 통해 ``utterances.jsonl`` 원본 데이터 파일을 파싱하려 # 합니다. # # - ``loadLines`` 는 파일에 포함된 대사를 변환하여 항목(대사 ID ``lineID``, # 인물 ID ``characterID``, 영화 ID ``movieID``, 인물 ``character``, 대사 # 내용 ``text``)에 대한 사전 형태로 변환합니다 # - ``loadConversations`` 는 ``loadLines`` 를 통해 읽어들인 -# 대사(``lines``)의 항목(``fields``)를 *movie_conversations.txt* 에 나와 +# 대사( ``lines`` )의 항목( ``fields`` )를 *movie_conversations.txt* 에 나와 # 있는 내용에 맞춰 대화 형태로 묶습니다 -# - ``extractSentencePairs`` 는 대화(``conversations``)에서 문장 쌍을 +# - ``extractSentencePairs`` 는 대화( ``conversations`` )에서 문장 쌍을 # 추출합니다 # @@ -220,12 +220,12 @@ def extractSentencePairs(conversations): ###################################################################### -# 이제 이 함수들을 호출하여 새로운 파일인 *formatted_utterances.jsonl* 를 +# 이제 이 함수들을 호출하여 새로운 파일인 ``formatted_movie_lines.txt`` 를 # 만듭니다. # # 새 파일에 대한 경로를 정의합니다 -datafile = os.path.join(corpus, "formatted_utterances.jsonl") +datafile = os.path.join(corpus, "formatted_movie_lines.txt") delimiter = '\t' # 구분자에 대해 unescape 함수를 호출합니다 @@ -252,7 +252,7 @@ def extractSentencePairs(conversations): ###################################################################### # 데이터 읽고 정리하기 -# ~~~~~~~~~~~~~~~~~~~~ +# ~~~~~~~~~~~~~~~~~~~~~~ # # 다음에 해야 할 일은 어휘집을 만들고, 질의/응답 문장 쌍을 메모리로 # 읽어들이는 것입니다. @@ -265,9 +265,9 @@ def extractSentencePairs(conversations): # 이를 위해 우리는 ``Voc`` 라는 클래스를 만들어 단어에서 인덱스로의 # 매핑, 인덱스에서 단어로의 역 매핑, 각 단어의 등장 횟수, 전체 단어 수 # 등을 관리하려 합니다. 이 클래스는 어휘집에 새로운 단어를 추가하는 -# 메서드(``addWord``), 문장에 등장하는 모든 단어를 추가하는 -# 메서드(``addSentence``), 그리고 자주 등장하지 않는 단어를 정리하는 -# 메서드(``trim``)를 제공합니다. 단어를 정리하는 내용에 대해서는 뒤에서 +# 메서드( ``addWord``), 문장에 등장하는 모든 단어를 추가하는 +# 메서드( ``addSentence``), 그리고 자주 등장하지 않는 단어를 정리하는 +# 메서드( ``trim``)를 제공합니다. 단어를 정리하는 내용에 대해서는 뒤에서 # 좀 더 자세히 살펴보겠습니다. # @@ -332,7 +332,7 @@ def trim(self, min_count): # 합니다. 다음에는 모든 글자를 소문자로 변환하고, 알파벳도 아니고 기본적인 # 문장 부호도 아닌 글자는 제거합니다(정규화, ``normalizeString``). # 마지막으로는 학습할 때의 편의성을 위해서, 길이가 일정 기준을 초과하는, -# 즉 ``MAX_LENGTH`` 보다 긴 문장을 제거합니다(``filterPairs``). +# 즉 ``MAX_LENGTH`` 보다 긴 문장을 제거합니다( ``filterPairs``). # MAX_LENGTH = 10 # 고려할 문장의 최대 길이 @@ -369,7 +369,7 @@ def filterPair(p): # EOS 토큰을 위해 입력 시퀀스의 마지막 단어를 보존해야 합니다 return len(p[0].split(' ')) < MAX_LENGTH and len(p[1].split(' ')) < MAX_LENGTH -# 조건식 filterPair에 따라 pairs를 필터링합니다 +# 조건식 ``filterPair`` 에 따라 pairs를 필터링합니다 def filterPairs(pairs): return [pair for pair in pairs if filterPair(pair)] @@ -446,7 +446,7 @@ def trimRareWords(voc, pairs, MIN_COUNT): ###################################################################### # 모델을 위한 데이터 준비하기 -# --------------------------- +# ----------------------------- # # 상당한 노력을 기울여 데이터를 전처리하고, 잘 정리하여 어휘집 객체와 # 문장 쌍의 리스트 형태로 만들어두긴 했지만, 결국 우리가 만들 모델에서 @@ -464,7 +464,7 @@ def trimRareWords(voc, pairs, MIN_COUNT): # 점에 유의해야 한다는 것을 뜻합니다. 같은 배치 안에서 크기가 다른 # 문장을 처리하기 위해서는 배치용 입력 텐서의 모양을 *(max_length, # batch_size)* 로 맞춰야 합니다. 이때 *max_length* 보다 짧은 문장에 -# 대해서는 *EOS 토큰* 뒤에 제로 토큰을 덧붙이면 됩니다. +# 대해서는 *EOS_token* 뒤에 제로 토큰을 덧붙이면 됩니다. # # 영어로 된 문장을 텐서로 변환하기 위해 단순히 그에 대응하는 인덱스를 # 사용하고(``indexesFromSentence``) 제로 토큰을 패딩한다고 해봅시다. @@ -489,7 +489,7 @@ def trimRareWords(voc, pairs, MIN_COUNT): # ``outputVar`` 함수는 ``inputVar`` 와 비슷한 작업을 수행하지만, ``lengths`` # 텐서를 반환하는 대신에 이진 마스크로 구성된 텐서와 목표 문장의 최대 # 길이를 같이 반환합니다. 이진 마스크 텐서는 출력에 해당하는 목표 텐서와 -# 그 모양이 같지만, 패딩 토큰(*PAD_token*)에 해당하는 경우에는 값이 0이며 +# 그 모양이 같지만, 패딩 토큰( *PAD_token* )에 해당하는 경우에는 값이 0이며 # 나머지 경우의 값은 1입니다. # # ``batch2TrainData`` 는 단순히 여러 쌍을 입력으로 받아서, 앞서 설명한 @@ -558,10 +558,10 @@ def batch2TrainData(voc, pair_batch): ###################################################################### # 모델 정의하기 -# ------------- +# --------------- # # Seq2Seq 모델 -# ~~~~~~~~~~~~ +# ~~~~~~~~~~~~~~ # # 우리 챗봇의 두뇌에 해당하는 모델은 sequence-to-sequence (seq2seq) # 모델입니다. seq2seq 모델의 목표는 가변 길이 시퀀스를 입력으로 받고, @@ -588,7 +588,7 @@ def batch2TrainData(voc, pair_batch): ###################################################################### # 인코더 -# ~~~~~~ +# ~~~~~~~ # # 인코더 RNN은 입력 시퀀스를 토큰 단위로(예를 들어, 단어 단위로) 한번에 # 하나씩 살펴보며 진행합니다. 그리고 각 단계마다 "출력" 벡터와 "은닉 @@ -627,7 +627,7 @@ def batch2TrainData(voc, pair_batch): # ``nn.utils.rnn.pack_padded_sequence`` 와 # ``nn.utils.rnn.pad_packed_sequence`` 를 통해 수행할 수 있습니다. # -# **계산 그래프:** +# **연산 그래프:** # # 1) 단어 인덱스를 임베딩으로 변환합니다. # 2) RNN 모듈을 위한 패딩된 배치 시퀀스를 패킹합니다. @@ -788,7 +788,7 @@ def forward(self, hidden, encoder_outputs): # 제공하려 합니다. 이는 임베딩된 단어 텐서와 GRU 출력의 모양이 둘 다 # *(1, batch_size, hidden_size)* 라는 의미입니다. # -# **계산 그래프:** +# **연산 그래프:** # # 1) 현재의 입력 단어에 대한 임베딩을 구합니다. # 2) 무방향 GRU로 포워드 패스를 수행합니다. @@ -861,10 +861,10 @@ def forward(self, input_step, last_hidden, encoder_outputs): ###################################################################### # 학습 프로시저 정의하기 -# ---------------------- +# ------------------------ # # Masked loss -# ~~~~~~~~~~~ +# ~~~~~~~~~~~~~ # # 우리는 패딩된 시퀀스 배치를 다루기 때문에 손실을 계산할 때 단순히 텐서의 # 모든 원소를 고려할 수는 없습니다. 우리는 ``maskNLLLoss`` 를 정의하여 @@ -883,7 +883,7 @@ def maskNLLLoss(inp, target, mask): ###################################################################### # 한 번의 학습 단계 -# ~~~~~~~~~~~~~~~~~ +# ~~~~~~~~~~~~~~~~~~~ # # ``train`` 함수에 학습을 한 단계(입력 배치 한 개에 대한) 진행하는 알고리즘이 # 나와 있습니다. @@ -927,7 +927,7 @@ def maskNLLLoss(inp, target, mask): # # .. warning:: # -# PyTorch의 RNN 모듈(``RNN``, ``LSTM``, ``GRU``)은 전체 입력 시퀀스(또는 +# PyTorch의 RNN 모듈( ``RNN``, ``LSTM``, ``GRU`` )은 전체 입력 시퀀스(또는 # 시퀀스의 배치)를 단순히 넣어주기만 하면 다른 비순환 레이어처럼 사용할 수 # 있습니다. 우리는 ``encoder`` 에서 ``GRU`` 레이어를 이런 식으로 사용합니다. # 그 안이 실제로 어떻게 되어 있는지를 살펴보면, 매 시간 단계마다 은닉 상태를 @@ -951,7 +951,7 @@ def train(input_variable, lengths, target_variable, mask, max_target_len, encode input_variable = input_variable.to(device) target_variable = target_variable.to(device) mask = mask.to(device) - # Lengths for rnn packing should always be on the cpu + # RNN 패킹의 길이는 항상 CPU에 위치해야 합니다 lengths = lengths.to("cpu") # 변수를 초기화합니다 @@ -1016,7 +1016,7 @@ def train(input_variable, lengths, target_variable, mask, max_target_len, encode ###################################################################### # 학습 단계 -# ~~~~~~~~~ +# ~~~~~~~~~~~ # # 이제 마지막으로 전체 학습 프로시저와 데이터를 하나로 엮을 때가 # 되었습니다. ``trainIters`` 함수는 주어진 모델, optimizer, 데이터 등을 @@ -1025,7 +1025,7 @@ def train(input_variable, lengths, target_variable, mask, max_target_len, encode # 함수에 옮겨 놓았기 때문입니다. # # 한 가지 주의할 점은 우리가 모델을 저장하려 할 때, 인코더와 디코더의 -# state_dicts (매개변수), optimizer의 state_dicts, 손실, 진행 단계 수 +# ``state_dicts`` (매개변수), optimizer의 ``state_dicts``, 손실, 진행 단계 수 # 등을 tarball로 만들어 저장한다는 점입니다. 모델을 이러한 방식으로 # 저장하면 checkpoint에 대해 아주 높은 수준의 유연성을 확보할 수 있게 # 됩니다. Checkpoint를 불러오고 나면, 우리는 모델 매개변수를 이용하여 @@ -1083,13 +1083,13 @@ def trainIters(model_name, voc, pairs, encoder, decoder, encoder_optimizer, deco ###################################################################### # 평가 정의하기 -# ------------- +# --------------- # # 모델을 학습시키고 나면 직접 봇과 대화를 나눠보고 싶어질 것입니다. 그러려면 # 먼저 모델이 인코딩된 입력을 어떻게 디코딩할지를 정의해줘야 합니다. # # 탐욕적 디코딩 -# ~~~~~~~~~~~~~ +# ~~~~~~~~~~~~~~~ # # 탐욕적 디코딩(Greedy decoding)은 우리가 학습 단계에서 teacher forcing을 # 적용하지 않았을 때 사용한 디코딩 방법입니다. 달리 말하면, 각 단계에 대해 @@ -1098,12 +1098,12 @@ def trainIters(model_name, voc, pairs, encoder, decoder, encoder_optimizer, deco # # 우리는 탐욕적 디코딩 연산을 수행할 수 있도록 ``GreedySearchDecoder`` # 클래스를 만들었습니다. 수행 과정에서 이 클래스의 인스턴스는 모양이 -# *(input_seq length, 1)* 인 입력 시퀀스(``input_seq``), 조종할 입력 -# 길이(``input_length``) 텐서, 그리고 응답 문장 길이의 제한을 나타내는 +# *(input_seq length, 1)* 인 입력 시퀀스( ``input_seq`` ), 조종할 입력 +# 길이( ``input_length`` ) 텐서, 그리고 응답 문장 길이의 제한을 나타내는 # ``max_length`` 를 입력으로 받습니다. 입력 시퀀서는 다음과 같은 계산 그래프에 # 의해 평가됩니다. # -# **계산 그래프:** +# **연산 그래프:** # # 1) 인코더 모델로 입력을 포워드 패스합니다. # 2) 인코더의 마지막 은닉 레이어가 디코더의 첫 번째 은닉 레이어의 입력이 되도록 준비합니다. @@ -1150,7 +1150,7 @@ def forward(self, input_seq, input_length, max_length): ###################################################################### # 내 텍스트 평가하기 -# ~~~~~~~~~~~~~~~~~~ +# ~~~~~~~~~~~~~~~~~~~~ # # 이제 디코딩 모델을 정의했으니, 문자열로 된 입력 시퀀스를 평가하는 함수를 # 작성해볼 수 있을 것입니다. ``evaluate`` 함수에 입력 시퀀스를 낮은 @@ -1231,8 +1231,8 @@ def evaluateInput(encoder, decoder, searcher, voc): # 모델을 설정합니다 model_name = 'cb_model' attn_model = 'dot' -#attn_model = 'general' -#attn_model = 'concat' +#``attn_model = 'general'`` +#``attn_model = 'concat'`` hidden_size = 500 encoder_n_layers = 2 decoder_n_layers = 2 @@ -1242,12 +1242,17 @@ def evaluateInput(encoder, decoder, searcher, voc): # 불러올 checkpoint를 설정합니다. 처음부터 시작할 때는 None으로 둡니다. loadFilename = None checkpoint_iter = 4000 -#loadFilename = os.path.join(save_dir, model_name, corpus_name, -# '{}-{}_{}'.format(encoder_n_layers, decoder_n_layers, hidden_size), -# '{}_checkpoint.tar'.format(checkpoint_iter)) +############################################################# +# checkpoint로부터 불러오는 샘플 코드: +# +# .. code-block:: python +# +# loadFilename = os.path.join(save_dir, model_name, corpus_name, +# '{}-{}_{}'.format(encoder_n_layers, decoder_n_layers, hidden_size), +# '{}_checkpoint.tar'.format(checkpoint_iter)) -# loadFilename이 제공되는 경우에는 모델을 불러옵니다 +# ``loadFilename`` 이 존재하는 경우에는 모델을 불러옵니다 if loadFilename: # 모델을 학습할 때와 같은 기기에서 불러오는 경우 checkpoint = torch.load(loadFilename) @@ -1309,7 +1314,7 @@ def evaluateInput(encoder, decoder, searcher, voc): encoder_optimizer.load_state_dict(encoder_optimizer_sd) decoder_optimizer.load_state_dict(decoder_optimizer_sd) -# cuda가 있다면 cuda를 설정합니다 +# CUDA가 있으면 CUDA를 설정합니다 for state in encoder_optimizer.state.values(): for k, v in state.items(): if isinstance(v, torch.Tensor): @@ -1329,12 +1334,12 @@ def evaluateInput(encoder, decoder, searcher, voc): ###################################################################### # 평가 수행하기 -# ~~~~~~~~~~~~~ +# ~~~~~~~~~~~~~~~ # # 여러분의 모델과 채팅을 해보고 싶다면 다음 블록을 수행하면 됩니다. # -# Dropout 레이어를 평가 모드로 설정합니다 +# Dropout 레이어를 평가( ``eval`` ) 모드로 설정합니다 encoder.eval() decoder.eval() @@ -1347,7 +1352,7 @@ def evaluateInput(encoder, decoder, searcher, voc): ###################################################################### # 맺음말 -# ------ +# -------- # # 이번 튜토리얼을 이것으로 마무리하겠습니다. 축하합니다! 여러분은 이제 생성 # 챗봇 모델을 만들기 위한 기초 지식을 습득했습니다. 만약 좀 더 관심이 있다면 diff --git a/beginner_source/colab.rst b/beginner_source/colab.rst index 4b301e250..07bf5fbcc 100644 --- a/beginner_source/colab.rst +++ b/beginner_source/colab.rst @@ -1,5 +1,29 @@ +Google Colab에서 튜토리얼 실행하기 +===================================== + +Google Colab에서 튜토리얼을 실행할 때, 튜토리얼이 제대로 동작하기 +위해서 충족해야 하는 추가적인 추가 요구 사항과 종속성(dependancy)이 +있을 수 있습니다. 이 섹션에서는 Google Colab에서 파이토치(PyTorch) 튜토리얼을 +성공적으로 실행하기 위해 다양한 설정을 구성하는 방법에 대해 설명합니다. + +Google Colab의 PyTorch 버전 +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +공개(release)된지 얼마되지 않은 PyTorch 버전을 사용하는 튜토리얼을 실행하는 경우, +해당 버전이 아직 Google Colab에 반영되지 않았을 수 있습니다. +필요한 ``torch`` 와 호환되는 도메인 라이브러리(domain library)가 설치되어 있는지 +확인하려면 ``!pip list`` 를 실행하세요. + +만약 필요한 PyTorch 버전보다 낮은 버전이 설치되어 있는 경우, +다음 명령어를 실행하여 제거한 뒤, 다시 설치하세요: + +.. code-block:: python + !pip3 uninstall --yes torch torchaudio torchvision torchtext torchdata + !pip3 install torch torchaudio torchvision torchtext torchdata + + Colab에서 Google Drive의 튜토리얼 데이터 사용하기 -==================================================== +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 사용자가 Google Colab에서 튜토리얼과 관련된 노트북을 열 수 있도록 하는 새로운 기능이 튜토리얼에 추가되었습니다. 이 때, 보다 복잡한 튜토리얼을 실행하려면 diff --git a/beginner_source/data_loading_tutorial.py b/beginner_source/data_loading_tutorial.py index 2f3c7e198..12c0f9f1d 100644 --- a/beginner_source/data_loading_tutorial.py +++ b/beginner_source/data_loading_tutorial.py @@ -83,6 +83,7 @@ # 이미지와 랜드마크(landmark)를 보여주는 간단한 함수를 작성해보고, # 실제로 적용해보겠습니다. # + def show_landmarks(image, landmarks): """Show image with landmarks""" """ 랜드마크(landmark)와 이미지를 보여줍니다. """ @@ -123,7 +124,7 @@ class FaceLandmarksDataset(Dataset): def __init__(self, csv_file, root_dir, transform=None): """ - Args: + Arguments: csv_file (string): csv 파일의 경로 root_dir (string): 모든 이미지가 존재하는 디렉토리 경로 transform (callable, optional): 샘플에 적용될 Optional transform @@ -181,7 +182,7 @@ def __getitem__(self, idx): ###################################################################### # Transforms -# ---------- +# ------------ # # 위에서 볼 수 있었던 한가지 문제점은 샘플들이 다 같은 크기가 아니라는 것입니다. # 대부분의 신경망(neural networks)은 고정된 크기의 이미지라고 가정합니다. @@ -291,7 +292,7 @@ def __call__(self, sample): # .. note:: # 위 예시에서, `RandomCrop` 은 외부 라이브러리의 난수 생성기(random number generator; 이 경우, Numpy의 `np.random.int` )를 # 사용하고 있습니다. 이는 `DataLoader` 가 예상치 못한 동작을 하도록 할 수 있습니다. -# (https://pytorch.org/docs/stable/notes/faq.html#my-data-loader-workers-return-identical-random-numbers 를 참고하세요) +# (`여기 `_ 를 참고하세요) # 실제 상황에서는 `torch.randint` 와 같은 PyTorch가 제공하는 난수 생성기를 사용하는 것이 안전합니다. ###################################################################### @@ -398,7 +399,7 @@ def show_landmarks_batch(sample_batched): plt.title('Batch from dataloader') # Windows를 사용 중이라면, 다음 줄의 주석을 제거하고 for 반복문을 들여쓰기합니다. -# "num_workers"를 0으로 변경해야 할 수도 있습니다. +# ``num_workers`` 를 0으로 변경해야 할 수도 있습니다. # if __name__ == '__main__': for i_batch, sample_batched in enumerate(dataloader): @@ -416,7 +417,7 @@ def show_landmarks_batch(sample_batched): ###################################################################### # Afterword: torchvision -# ---------------------- +# ------------------------ # # 이번 튜토리얼에서는, 데이터셋 작성과 사용, 전이(transforms), 데이터를 불러오는 방법에 대해서 알아봤습니다. # ``torchvision`` 패키지는 몇몇의 일반적인 데이터셋과 전이(transforms)들을 제공합니다. @@ -438,21 +439,21 @@ def show_landmarks_batch(sample_batched): # 비슷하게, ``RandomHorizontalFlip`` , ``Scale`` 과 같이 ``PIL.Image`` 에서 작동하는 # 일반적인 전이(transforms)도 사용가능합니다. 이와 같이 데이터로더(dataloader)를 사용할 수 있습니다: :: # -# import torch -# from torchvision import transforms, datasets -# -# data_transform = transforms.Compose([ -# transforms.RandomSizedCrop(224), -# transforms.RandomHorizontalFlip(), -# transforms.ToTensor(), -# transforms.Normalize(mean=[0.485, 0.456, 0.406], -# std=[0.229, 0.224, 0.225]) -# ]) -# hymenoptera_dataset = datasets.ImageFolder(root='hymenoptera_data/train', -# transform=data_transform) -# dataset_loader = torch.utils.data.DataLoader(hymenoptera_dataset, -# batch_size=4, shuffle=True, -# num_workers=4) +# import torch +# from torchvision import transforms, datasets +# +# data_transform = transforms.Compose([ +# transforms.RandomSizedCrop(224), +# transforms.RandomHorizontalFlip(), +# transforms.ToTensor(), +# transforms.Normalize(mean=[0.485, 0.456, 0.406], +# std=[0.229, 0.224, 0.225]) +# ]) +# hymenoptera_dataset = datasets.ImageFolder(root='hymenoptera_data/train', +# transform=data_transform) +# dataset_loader = torch.utils.data.DataLoader(hymenoptera_dataset, +# batch_size=4, shuffle=True, +# num_workers=4) # # training code에 대한 예시를 알고 싶다면, # :doc:`transfer_learning_tutorial` 문서를 참고해주세요 diff --git a/beginner_source/dcgan_faces_tutorial.py b/beginner_source/dcgan_faces_tutorial.py index 0a2961bdb..624e47b3f 100644 --- a/beginner_source/dcgan_faces_tutorial.py +++ b/beginner_source/dcgan_faces_tutorial.py @@ -1,10 +1,10 @@ # -*- coding: utf-8 -*- """ DCGAN 튜토리얼 -============== +================ **저자**: `Nathan Inkawhich `_ - **번역**: `조민성 `_ +**번역**: `조민성 `_ """ @@ -121,22 +121,22 @@ # # 몇 가지 설정값들을 정의해봅시다: # -# - **dataroot** - 데이터셋 폴더의 경로입니다. 데이터셋에 관한건 다음 섹션에서 +# - ``dataroot`` - 데이터셋 폴더의 경로입니다. 데이터셋에 관한건 다음 섹션에서 # 더 자세히 설명하겠습니다. -# - **workers** - DataLoader에서 데이터를 불러올 때 사용할 쓰레드의 개수입니다. -# - **batch_size** - 학습에 사용할 배치 크기입니다. DCGAN에서는 128을 사용했습니다. -# - **image_size** - 학습에 사용되는 이미지의 크기입니다. +# - ``workers`` - DataLoader에서 데이터를 불러올 때 사용할 쓰레드의 개수입니다. +# - ``batch_size`` - 학습에 사용할 배치 크기입니다. DCGAN에서는 128을 사용했습니다. +# - ``image_size`` - 학습에 사용되는 이미지의 크기입니다. # 본 문서에서는 64x64의 크기를 기본으로 하나, 만일 다른 크기의 이미지를 사용한다면 # D와 G의 구조 역시 변경되어야 합니다. 더 자세한 정보를 위해선 # `이곳 `__ 을 확인해 보세요. -# - **nc** - 입력 이미지의 색 채널개수입니다. RGB 이미지이기 때문에 3으로 설정합니다. -# - **nz** - 잠재공간 벡터의 원소들 개수입니다. -# - **ngf** - 생성자를 통과할때 만들어질 특징 데이터의 채널개수입니다. -# - **ndf** - 구분자를 통과할때 만들어질 특징 데이터의 채널개수입니다. -# - **num_epochs** - 학습시킬 에폭 수입니다. 오래 학습시키는 것이 대부분 좋은 결과를 보이지만, 당연히도 시간이 오래걸리는 것이 단점입니다. -# - **lr** - 모델의 학습률입니다. DCGAN에서 사용된대로 0.0002로 설정합니다. -# - **beta1** - Adam 옵티마이저에서 사용할 beta1 하이퍼파라미터 값입니다. 역시나 논문에서 사용한대로 0.5로 설정했습니다. -# - **ngpu** - 사용가능한 GPU의 번호입니다. 0으로 두면 CPU에서 학습하고, 0보다 큰 수로 설정하면 각 숫자가 가리키는 GPU로 학습시킵니다. +# - ``nc`` - 입력 이미지의 색 채널개수입니다. RGB 이미지이기 때문에 3으로 설정합니다. +# - ``nz`` - 잠재공간 벡터의 원소들 개수입니다. +# - ``ngf`` - 생성자를 통과할때 만들어질 특징 데이터의 채널개수입니다. +# - ``ndf`` - 구분자를 통과할때 만들어질 특징 데이터의 채널개수입니다. +# - ``num_epochs`` - 학습시킬 에폭 수입니다. 오래 학습시키는 것이 대부분 좋은 결과를 보이지만, 당연히도 시간이 오래걸리는 것이 단점입니다. +# - ``lr`` - 모델의 학습률입니다. DCGAN에서 사용된대로 0.0002로 설정합니다. +# - ``beta1`` - Adam 옵티마이저에서 사용할 beta1 하이퍼파라미터 값입니다. 역시나 논문에서 사용한대로 0.5로 설정했습니다. +# - ``ngpu`` - 사용가능한 GPU의 번호입니다. 0으로 두면 CPU에서 학습하고, 0보다 큰 수로 설정하면 각 숫자가 가리키는 GPU로 학습시킵니다. # # 데이터셋의 경로 @@ -183,10 +183,10 @@ # 본 튜토리얼에서 사용할 데이터는 `Celeb-A Faces # dataset `__ 로, 해당 링크를 이용하거나 `Google # Drive `__ 에서 데이터를 받을 수 있습니다. -# 데이터를 받으면 *img_align_celeba.zip* 라는 파일을 보게될 겁니다. 다운로드가 끝나면 -# *celeba* 이라는 폴더를 새로 만들고, 해당 폴더에 해당 zip 파일을 압축해제 해주시면 됩니다. -# 압축 해제 후, 위에서 정의한 *dataroot* 변수에 방금 만든 *celeba* 폴더의 경로를 넣어주세요. -# 위의 작업이 끝나면 *celeba* 폴더의 구조는 다음과 같아야 합니다: +# 데이터를 받으면 ``img_align_celeba.zip`` 라는 파일을 보게될 겁니다. 다운로드가 끝나면 +# ``celeba`` 이라는 폴더를 새로 만들고, 해당 폴더에 해당 zip 파일을 압축해제 해주시면 됩니다. +# 압축 해제 후, 위에서 정의한 ``dataroot`` 변수에 방금 만든 ``celeba`` 폴더의 경로를 넣어주세요. +# 위의 작업이 끝나면 ``celeba`` 폴더의 구조는 다음과 같아야 합니다: # # :: # @@ -198,9 +198,10 @@ # -> 537394.jpg # ... # -# 이 과정들은 프로그램이 정상적으로 구동하기 위해서는 중요한 부분입니다. 이때 celeba 폴더안에 다시 폴더를 두는 이유는, -# ImageFolder 클래스가 데이터셋의 최상위 폴더에 서브폴더를 요구하기 때문입니다. -# 이제 데이터셋과 DataLoader의 설정을 끝냈습니다. +# 이 과정들은 프로그램이 정상적으로 구동하기 위해서는 중요한 부분입니다. +# 이때 ``celeba`` 폴더 안에 다시 폴더를 두는 이유는, +# ``ImageFolder`` 클래스가 데이터셋의 최상위 폴더에 서브폴더를 요구하기 때문입니다. +# 이제 ``Dataset`` 과 ``DataLoader`` 의 설정을 끝냈습니다. # 최종적으로 학습 데이터들을 시각화해봅시다. # @@ -230,23 +231,23 @@ ###################################################################### # 구현 -# ---- +# ------ # # 모델의 설정값들과 데이터들이 준비되었기 때문에, 드디어 모델의 구현으로 # 들어갈 수 있을 것 같습니다. 먼저 가중치 초기화에 대해 이야기 해보고, # 순서대로 생성자, 구분자, 손실 함수, 학습 방법들을 알아보겠습니다. # # 가중치 초기화 -# ~~~~~~~~~~~~~~ +# ~~~~~~~~~~~~~~~~ # -# DCGAN 논문에서는, 평균이 0이고 분산이 0.02인 정규분포을 이용해, -# 구분자와 생성자 모두 무작위 초기화를 진행하는 것이 좋다고 합니다. +# DCGAN 논문에서는, 평균이 0( ``mean=0`` )이고 분산이 0.02( ``stdev=0.02`` )인 +# 정규분포을 시용해, 구분자와 생성자 모두 무작위 초기화를 진행하는 것이 좋다고 합니다. # ``weights_init`` 함수는 매개변수로 모델을 입력받아, # 모든 합성곱 계층, 전치 합성곱 계층, 배치 정규화 계층을, 위에서 말한 조건대로 # 가중치들을 다시 초기화 시킵니다. 이 함수는 모델이 만들어지자 마자 바로 적용을 # 시키게 됩니다. -# netG와 netD에 적용시킬 커스텀 가중치 초기화 함수 +# ``netG`` 와 ``netD`` 에 적용시킬 커스텀 가중치 초기화 함수 def weights_init(m): classname = m.__class__.__name__ if classname.find('Conv') != -1: @@ -258,7 +259,7 @@ def weights_init(m): ###################################################################### # 생성자 -# ~~~~~~ +# ~~~~~~~~ # # 생성자 :math:`G` 는 잠재 공간 벡터 :math:`z` 를, 데이터 공간으로 # 변환시키도록 설계되었습니다. 우리에게 데이터라 함은 이미지이기 때문에, @@ -275,9 +276,9 @@ def weights_init(m): # .. figure:: /_static/img/dcgan_generator.png # :alt: dcgan_generator # -# 우리가 설정값 섹션에서 정의한 값들이 (*nz*, *ngf*, 그리고 -# *nc*) 생성자 모델 아키텍쳐에 어떻게 영향을 끼치는지 주목해주세요. *nz* 는 z 입력 벡터의 -# 길이, *ngf* 는 생성자를 통과하는 특징 데이터의 크기, 그리고 *nc* 는 출력 이미지의 +# 우리가 설정값 섹션에서 정의한 값들이 (``nz``, ``ngf``, 그리고 +# ``nc``) 생성자 모델 아키텍쳐에 어떻게 영향을 끼치는지 주목해주세요. ``nz`` 는 z 입력 벡터의 +# 길이, ``ngf`` 는 생성자를 통과하는 특징 데이터의 크기, 그리고 ``nc`` 는 출력 이미지의 # 채널 개수입니다 (RGB 이미지이기 때문에 3으로 설정을 했습니다). # 아래는 생성자의 코드입니다. # @@ -293,22 +294,22 @@ def __init__(self, ngpu): nn.ConvTranspose2d( nz, ngf * 8, 4, 1, 0, bias=False), nn.BatchNorm2d(ngf * 8), nn.ReLU(True), - # 위의 계층을 통과한 데이터의 크기. (ngf*8) x 4 x 4 + # 위의 계층을 통과한 데이터의 크기. ``(ngf*8) x 4 x 4`` nn.ConvTranspose2d(ngf * 8, ngf * 4, 4, 2, 1, bias=False), nn.BatchNorm2d(ngf * 4), nn.ReLU(True), - # 위의 계층을 통과한 데이터의 크기. (ngf*4) x 8 x 8 + # 위의 계층을 통과한 데이터의 크기. ``(ngf*4) x 8 x 8`` nn.ConvTranspose2d( ngf * 4, ngf * 2, 4, 2, 1, bias=False), nn.BatchNorm2d(ngf * 2), nn.ReLU(True), - # 위의 계층을 통과한 데이터의 크기. (ngf*2) x 16 x 16 + # 위의 계층을 통과한 데이터의 크기. ``(ngf*2) x 16 x 16`` nn.ConvTranspose2d( ngf * 2, ngf, 4, 2, 1, bias=False), nn.BatchNorm2d(ngf), nn.ReLU(True), - # 위의 계층을 통과한 데이터의 크기. (ngf) x 32 x 32 + # 위의 계층을 통과한 데이터의 크기. ``(ngf) x 32 x 32`` nn.ConvTranspose2d( ngf, nc, 4, 2, 1, bias=False), nn.Tanh() - # 위의 계층을 통과한 데이터의 크기. (nc) x 64 x 64 + # 위의 계층을 통과한 데이터의 크기. ``(nc) x 64 x 64`` ) def forward(self, input): @@ -324,12 +325,12 @@ def forward(self, input): # 생성자를 만듭니다 netG = Generator(ngpu).to(device) -# 필요한 경우 multi-gpu를 설정 해주세요 +# 필요한 경우 multi-GPU를 설정 해주세요 if (device.type == 'cuda') and (ngpu > 1): netG = nn.DataParallel(netG, list(range(ngpu))) -# 모든 가중치의 평균을 0, 분산을 0.02로 초기화 하기 위해 -# weight_init 함수를 적용시킵니다 +# 모든 가중치의 평균을 0( ``mean=0`` ), 분산을 0.02( ``stdev=0.02`` )로 초기화하기 위해 +# ``weight_init`` 함수를 적용시킵니다 netG.apply(weights_init) # 모델의 구조를 출력합니다 @@ -340,7 +341,7 @@ def forward(self, input): # 구분자 # ~~~~~~ # -# 앞서 언급했듯, 구분자 :math:`D`는 입력 이미지가 진짜 이미지인지 (혹은 반대로 가짜 이미지인지) +# 앞서 언급했듯, 구분자 :math:`D` 는 입력 이미지가 진짜 이미지인지 (혹은 반대로 가짜 이미지인지) # 판별하는 전통적인 이진 분류 신경망으로 볼 수 있습니다. 이때 :math:`D` 는 # 3x64x64 이미지를 입력받아, Conv2d, BatchNorm2d, 그리고 LeakyReLU 계층을 통과시켜 # 데이터를 가공시키고, 마지막 출력에서 Sigmoid 함수를 이용하여 @@ -362,22 +363,22 @@ def __init__(self, ngpu): super(Discriminator, self).__init__() self.ngpu = ngpu self.main = nn.Sequential( - # 입력 데이터의 크기는 (nc) x 64 x 64 입니다 + # 입력 데이터의 크기는 ``(nc) x 64 x 64`` 입니다 nn.Conv2d(nc, ndf, 4, 2, 1, bias=False), nn.LeakyReLU(0.2, inplace=True), - # 위의 계층을 통과한 데이터의 크기. (ndf) x 32 x 32 + # 위의 계층을 통과한 데이터의 크기. ``(ndf) x 32 x 32`` nn.Conv2d(ndf, ndf * 2, 4, 2, 1, bias=False), nn.BatchNorm2d(ndf * 2), nn.LeakyReLU(0.2, inplace=True), - # 위의 계층을 통과한 데이터의 크기. (ndf*2) x 16 x 16 + # 위의 계층을 통과한 데이터의 크기. ``(ndf*2) x 16 x 16`` nn.Conv2d(ndf * 2, ndf * 4, 4, 2, 1, bias=False), nn.BatchNorm2d(ndf * 4), nn.LeakyReLU(0.2, inplace=True), - # 위의 계층을 통과한 데이터의 크기. (ndf*4) x 8 x 8 + # 위의 계층을 통과한 데이터의 크기. ``(ndf*4) x 8 x 8`` nn.Conv2d(ndf * 4, ndf * 8, 4, 2, 1, bias=False), nn.BatchNorm2d(ndf * 8), nn.LeakyReLU(0.2, inplace=True), - # 위의 계층을 통과한 데이터의 크기. (ndf*8) x 4 x 4 + # 위의 계층을 통과한 데이터의 크기. ``(ndf*8) x 4 x 4`` nn.Conv2d(ndf * 8, 1, 4, 1, 0, bias=False), nn.Sigmoid() ) @@ -394,12 +395,12 @@ def forward(self, input): # 구분자를 만듭니다 netD = Discriminator(ngpu).to(device) -# 필요한 경우 multi-gpu를 설정 해주세요 +# 필요한 경우 multi-GPU를 설정 해주세요 if (device.type == 'cuda') and (ngpu > 1): netD = nn.DataParallel(netD, list(range(ngpu))) -# 모든 가중치의 평균을 0, 분산을 0.02로 초기화 하기 위해 -# weight_init 함수를 적용시킵니다 +# 모든 가중치의 평균을 0( ``mean=0`` ), 분산을 0.02( ``stdev=0.02`` )로 초기화하기 위해 +# ``weight_init`` 함수를 적용시킵니다 netD.apply(weights_init) # 모델의 구조를 출력합니다 @@ -412,7 +413,7 @@ def forward(self, input): # # :math:`D` 와 :math:`G` 의 설정을 끝냈으니, 이제 손실함수와 옵티마이저를 정하여 # 학습을 구체화시킬 시간입니다. 손실함수로는 Binary Cross Entropy loss -# (`BCELoss `__) +# (`BCELoss `__) # 를 사용할겁니다. 해당함수는 아래의 식으로 파이토치에 구현되어 있습니다: # # .. math:: \ell(x, y) = L = \{l_1,\dots,l_N\}^\top, \quad l_n = - \left[ y_n \cdot \log x_n + (1 - y_n) \cdot \log (1 - x_n) \right] @@ -434,7 +435,7 @@ def forward(self, input): # 입력하면, 그 출력값을 기반으로 생성자의 상태를 확인 할 수 있습니다. # -# BCELoss 함수의 인스턴스를 생성합니다 +# ``BCELoss`` 함수의 인스턴스를 초기화합니다 criterion = nn.BCELoss() # 생성자의 학습상태를 확인할 잠재 공간 벡터를 생성합니다 @@ -457,10 +458,10 @@ def forward(self, input): # 실제 모델을 학습시키는 방법을 알아보겠습니다. 주의를 기울일 것은, GAN을 학습시키는 건 # 관례적인 기술들의 집합이기 때문에, 잘못된 하이퍼파라미터의 설정은 # 모델의 학습을 망가뜨릴 수 있습니다. 무엇이 잘못되었는지 알아내는 것 조차도 힘들죠. -# 그러한 이유로, 본 튜토리얼에서는 Goodfellow의 논문에서 서술된 Algorithm 1을 기반으로, -# `ganhacks `__ 에서 사용된 몇가지 괜찮은 테크닉들을 +# 그러한 이유로, 본 튜토리얼에서는 `Goodfellow’s paper `__ +# 에서 서술된 Algorithm 1을 기반으로, `ganhacks `__ 에서 사용된 몇가지 괜찮은 테크닉들을 # 더할 것입니다. 앞서 몇번 설명했지만, 우리의 의도는 “진짜 혹은 가짜 이미지를 구성”하고, -# :math:`logD(G(z))` 를 최대화하는 G의 목적함수를 최적화 시키는 겁니다. 학습과정은 크게 두가지로 나눕니다. +# :math:`log(D(G(z)))` 를 최대화하는 G의 목적함수를 최적화 시키는 겁니다. 학습과정은 크게 두가지로 나눕니다. # Part 1은 구분자를, Part 2는 생성자를 업데이트하는 과정입니다. # # **Part 1 - 구분자의 학습** @@ -468,7 +469,7 @@ def forward(self, input): # 구분자의 목적은 주어진 입력값이 진짜인지 가짜인지 판별하는 것임을 상기합시다. # Goodfellow의 말을 빌리자면, 구분자는 “변화도(gradient)를 상승(ascending)시키며 훈련”하게 됩니다. # 실전적으로 얘기하면, :math:`log(D(x)) + log(1-D(G(z)))` 를 최대화시키는 것과 같습니다. -# ganhacks에서 미니 배치(mini-batch)를 분리하여 사용한 개념을 가져와서, +# `ganhacks `__ 에서 미니 배치(mini-batch)를 분리하여 사용한 개념을 가져와서, # 우리 역시 두가지 스텝으로 분리해 계산을 해보겠습니다. 먼저, # 진짜 데이터들로만 이루어진 배치를 만들어 :math:`D` 에 통과시킵니다. 그 출력값으로 (:math:`log(D(x))`) 의 손실값을 계산하고, # 역전파 과정에서의 변화도들을 계산합니다. 여기까지가 첫번째 스텝입니다. 두번째 스텝에서는, 오로지 가짜 데이터들로만 @@ -484,11 +485,11 @@ def forward(self, input): # 위해서는 : Part 1에서 한대로 구분자를 이용해 생성자의 출력값을 판별해주고, *진짜 라벨값* 을 이용해 G의 손실값을 구해줍니다. # 그러면 구해진 손실값으로 변화도를 구하고, 최종적으로는 옵티마이저를 이용해 G의 가중치들을 업데이트시켜주면 됩니다. # 언뜻 볼때는, 생성자가 만들어낸 *가짜* 이미지에 *진짜* 라벨을 사용하는것이 직관적으로 위배가 될테지만, 이렇게 라벨을 -# 바꿈으로써 :math:`log(x)` 라는 BCELoss의 일부분을 사용할 수 있게 합니다 (앞서 우리는 BCELoss에서 라벨을 이용해 원하는 로그 계산 +# 바꿈으로써 :math:`log(x)` 라는 ``BCELoss`` 의 일부분을 사용할 수 있게 합니다 (앞서 우리는 BCELoss에서 라벨을 이용해 원하는 로그 계산 # 요소를 고를 수 있음을 알아봤습니다). # # 마무리로 G의 훈련 상태를 알아보기 위하여, 몇가지 통계적인 수치들과, fixed_noise를 통과시킨 -# 결과를 화면에 출력하는 코드를 추가하겠습니다. 이때 통계적인 수치들이라 함은 : +# 결과를 화면에 출력하는 코드를 추가하겠습니다. 이때 통계적인 수치들이라 함은: # # - **Loss_D** - 진짜 데이터와 가짜 데이터들 모두에서 구해진 손실값. (:math:`log(D(x)) + log(1 - D(G(z)))`). # - **Loss_G** - 생성자의 손실값. :math:`log(D(G(z)))` @@ -589,7 +590,7 @@ def forward(self, input): ###################################################################### # 결과 -# ---- +# ------ # # 결과를 알아봅시다. 이 섹션에서는 총 세가지를 확인할겁니다. # 첫번째는 G와 D의 손실값들이 어떻게 변했는가, 두번째는 매 에폭마다 @@ -653,7 +654,7 @@ def forward(self, input): ###################################################################### # 이제 어디로 여행을 떠나볼까요? -# ------------------------------ +# -------------------------------- # # 드디어 DCGAN이 끝났습니다! 하지만 더 알아볼 것들이 많이 남아있죠. # 무엇을 더 시도해볼 수 있을까요? diff --git a/beginner_source/ddp_series_fault_tolerance.rst b/beginner_source/ddp_series_fault_tolerance.rst index e141b4a7f..2bc63d7ec 100644 --- a/beginner_source/ddp_series_fault_tolerance.rst +++ b/beginner_source/ddp_series_fault_tolerance.rst @@ -42,8 +42,8 @@ Follow along with the video below or on `youtube `__. +- You don't need to set environment variables or explicitly pass the ``rank`` and ``world_size``; ``torchrun`` assigns this along with several other `environment variables `__. - No need to call ``mp.spawn`` in your script; you only need a generic ``main()`` entrypoint, and launch the script with ``torchrun``. This way the same script can be run in non-distributed as well as single-node and multinode setups. - Gracefully restarting training from the last saved training snapshot @@ -117,7 +117,7 @@ Process group initialization - os.environ["MASTER_PORT"] = "12355" - init_process_group(backend="nccl", rank=rank, world_size=world_size) + init_process_group(backend="nccl") - + torch.cuda.set_device(int(os.environ["LOCAL_RANK"])) Use Torchrun-provided env variables ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/beginner_source/ddp_series_multigpu.rst b/beginner_source/ddp_series_multigpu.rst index baf92d8f8..5d25bfa62 100644 --- a/beginner_source/ddp_series_multigpu.rst +++ b/beginner_source/ddp_series_multigpu.rst @@ -15,13 +15,13 @@ Authors: `Suraj Subramanian `__ - How to migrate a single-GPU training script to multi-GPU via DDP - Setting up the distributed process group - Saving and loading models in a distributed setup - + .. grid:: 1 .. grid-item:: :octicon:`code-square;1.0em;` View the code used in this tutorial on `GitHub `__ - + .. grid-item-card:: :octicon:`list-unordered;1em;` Prerequisites * High-level overview of `how DDP works `__ @@ -40,11 +40,11 @@ In the `previous tutorial `__, we got a high-level overv In this tutorial, we start with a single-GPU training script and migrate that to running it on 4 GPUs on a single node. Along the way, we will talk through important concepts in distributed training while implementing them in our code. -.. note:: - If your model contains any ``BatchNorm`` layer, it needs to be converted to ``SyncBatchNorm`` to sync the running stats of ``BatchNorm`` +.. note:: + If your model contains any ``BatchNorm`` layers, it needs to be converted to ``SyncBatchNorm`` to sync the running stats of ``BatchNorm`` layers across replicas. - Use the helper function + Use the helper function `torch.nn.SyncBatchNorm.convert_sync_batchnorm(model) `__ to convert all ``BatchNorm`` layers in the model to ``SyncBatchNorm``. @@ -57,7 +57,7 @@ Imports ~~~~~~~ - ``torch.multiprocessing`` is a PyTorch wrapper around Python's native multiprocessing -- The dsitributed process group contains all the processes that can +- The distributed process group contains all the processes that can communicate and synchronize with each other. .. code:: diff @@ -65,7 +65,7 @@ Imports import torch import torch.nn.functional as F from utils import MyTrainDataset - + + import torch.multiprocessing as mp + from torch.utils.data.distributed import DistributedSampler + from torch.nn.parallel import DistributedDataParallel as DDP @@ -83,6 +83,8 @@ Constructing the process group initializes the distributed process group. - Read more about `choosing a DDP backend `__ +- `set_device `__ + sets the default GPU for each process. This is important to prevent hangs or excessive memory utilization on `GPU:0` .. code:: diff @@ -95,6 +97,7 @@ Constructing the process group + os.environ["MASTER_ADDR"] = "localhost" + os.environ["MASTER_PORT"] = "12355" + init_process_group(backend="nccl", rank=rank, world_size=world_size) + + torch.cuda.set_device(rank) Constructing the DDP model @@ -123,7 +126,7 @@ Distributing input data + sampler=DistributedSampler(train_dataset), ) -- Calling the ``set_epoch()`` method on the ``DistributedSampler`` at the beginning of each epoch is necessary to make shuffling work +- Calling the ``set_epoch()`` method on the ``DistributedSampler`` at the beginning of each epoch is necessary to make shuffling work properly across multiple epochs. Otherwise, the same ordering will be used in each epoch. .. code:: diff @@ -138,10 +141,10 @@ Distributing input data Saving model checkpoints ~~~~~~~~~~~~~~~~~~~~~~~~ -- We only need to save model checkpoints from one process. Without this +- We only need to save model checkpoints from one process. Without this condition, each process would save its copy of the identical mode. Read more on saving and loading models with - DDP `here `__ + DDP `here `__ .. code:: diff @@ -156,7 +159,7 @@ Saving model checkpoints .. warning:: `Collective calls `__ are functions that run on all the distributed processes, and they are used to gather certain states or values to a specific process. Collective calls require all ranks to run the collective code. - In this example, `_save_checkpoint` should not have any collective calls because it is only run on the ``rank:0`` process. + In this example, `_save_checkpoint` should not have any collective calls because it is only run on the ``rank:0`` process. If you need to make any collective calls, it should be before the ``if self.gpu_id == 0`` check. @@ -167,7 +170,7 @@ Running the distributed training job ``world_size``. - ``rank`` is auto-allocated by DDP when calling `mp.spawn `__. -- ``world_size`` is the number of processes across the training job. For GPU training, +- ``world_size`` is the number of processes across the training job. For GPU training, this corresponds to the number of GPUs in use, and each process works on a dedicated GPU. .. code:: diff @@ -177,11 +180,11 @@ Running the distributed training job + ddp_setup(rank, world_size) dataset, model, optimizer = load_train_objs() train_data = prepare_dataloader(dataset, batch_size=32) - - trainer = Trainer(model, dataset, optimizer, device, save_every) - + trainer = Trainer(model, dataset, optimizer, rank, save_every) + - trainer = Trainer(model, train_data, optimizer, device, save_every) + + trainer = Trainer(model, train_data, optimizer, rank, save_every) trainer.train(total_epochs) + destroy_process_group() - + if __name__ == "__main__": import sys total_epochs = int(sys.argv[1]) @@ -198,6 +201,6 @@ Further Reading - `Fault Tolerant distributed training `__ (next tutorial in this series) - `Intro to DDP `__ (previous tutorial in this series) -- `Getting Started with DDP `__ +- `Getting Started with DDP `__ - `Process Group initialization `__ diff --git a/beginner_source/ddp_series_theory.rst b/beginner_source/ddp_series_theory.rst index 8e4935231..963f8f9c6 100644 --- a/beginner_source/ddp_series_theory.rst +++ b/beginner_source/ddp_series_theory.rst @@ -19,7 +19,7 @@ Authors: `Suraj Subramanian `__ .. grid-item-card:: :octicon:`list-unordered;1em;` Prerequisites - * Familiarity with `basic non-distributed training `__ in PyTorch + * Familiarity with `basic non-distributed training `__ in PyTorch Follow along with the video below or on `youtube `__. @@ -40,7 +40,7 @@ algorithm `__ +`DataParallel `__ is an older approach to data parallelism. DP is trivially simple (with just one extra line of code) but it is much less performant. DDP improves upon the architecture in a few ways: @@ -54,8 +54,8 @@ DDP improves upon the architecture in a few ways: | | machines | +---------------------------------------+------------------------------+ | Slower; uses multithreading on a | Faster (no GIL contention) | -| single process and runs into GIL | because it uses | -| contention | multiprocessing | +| single process and runs into Global | because it uses | +| Interpreter Lock (GIL) contention | multiprocessing | +---------------------------------------+------------------------------+ Further Reading diff --git a/beginner_source/deploy_seq2seq_hybrid_frontend_tutorial.py b/beginner_source/deploy_seq2seq_hybrid_frontend_tutorial.py index 0812992f0..17b7d663b 100644 --- a/beginner_source/deploy_seq2seq_hybrid_frontend_tutorial.py +++ b/beginner_source/deploy_seq2seq_hybrid_frontend_tutorial.py @@ -39,7 +39,7 @@ # the Python runtime. # # The API for converting eager-mode PyTorch programs into TorchScript is -# found in the torch.jit module. This module has two core modalities for +# found in the ``torch.jit`` module. This module has two core modalities for # converting an eager-mode model to a TorchScript graph representation: # **tracing** and **scripting**. The ``torch.jit.trace`` function takes a # module or function and a set of example inputs. It then runs the example @@ -74,18 +74,18 @@ ###################################################################### -# Acknowledgements +# Acknowledgments # ---------------- # # This tutorial was inspired by the following sources: # -# 1) Yuan-Kuei Wu’s pytorch-chatbot implementation: +# 1) Yuan-Kuei Wu's pytorch-chatbot implementation: # https://github.com/ywk991112/pytorch-chatbot # -# 2) Sean Robertson’s practical-pytorch seq2seq-translation example: +# 2) Sean Robertson's practical-pytorch seq2seq-translation example: # https://github.com/spro/practical-pytorch/tree/master/seq2seq-translation # -# 3) FloydHub’s Cornell Movie Corpus preprocessing code: +# 3) FloydHub's Cornell Movie Corpus preprocessing code: # https://github.com/floydhub/textutil-preprocess-cornell-movie-corpus # @@ -290,7 +290,7 @@ def __init__(self, hidden_size, embedding, n_layers=1, dropout=0): self.hidden_size = hidden_size self.embedding = embedding - # Initialize GRU; the input_size and hidden_size params are both set to 'hidden_size' + # Initialize GRU; the ``input_size`` and ``hidden_size`` parameters are both set to 'hidden_size' # because our input size is a word embedding with number of features == hidden_size self.gru = nn.GRU(hidden_size, hidden_size, n_layers, dropout=(0 if n_layers == 1 else dropout), bidirectional=True) @@ -525,7 +525,7 @@ def forward(self, input_step, last_hidden, encoder_outputs): # we can use function type annotations as introduced in `PEP # 3107 `__. In addition, # it is possible to declare arguments of different types using -# MyPy-style type annotations (see +# Mypy-style type annotations (see # `doc `__). # # @@ -618,7 +618,7 @@ def evaluate(searcher, voc, sentence, max_length=MAX_LENGTH): return decoded_words -# Evaluate inputs from user input (stdin) +# Evaluate inputs from user input (``stdin``) def evaluateInput(searcher, voc): input_sentence = '' while(1): @@ -638,7 +638,7 @@ def evaluateInput(searcher, voc): except KeyError: print("Error: Encountered unknown word.") -# Normalize input sentence and call evaluate() +# Normalize input sentence and call ``evaluate()`` def evaluateExample(sentence, searcher, voc): print("> " + sentence) # Normalize sentence @@ -653,7 +653,7 @@ def evaluateExample(sentence, searcher, voc): # Load Pretrained Parameters # -------------------------- # -# Ok, its time to load our model! +# No, let's load our model! # # Use hosted model # ~~~~~~~~~~~~~~~~ @@ -671,7 +671,7 @@ def evaluateExample(sentence, searcher, voc): # Use your own model # ~~~~~~~~~~~~~~~~~~ # -# To load your own pre-trained model: +# To load your own pretrained model: # # 1) Set the ``loadFilename`` variable to the path to the checkpoint file # that you wish to load. Note that if you followed the convention for @@ -691,9 +691,9 @@ def evaluateExample(sentence, searcher, voc): # ~~~~~~~~~~~~~~~~~~~~~~ # # Notice that we initialize and load parameters into our encoder and -# decoder models as usual. If you are using tracing mode(`torch.jit.trace`) -# for some part of your models, you must call .to(device) to set the device -# options of the models and .eval() to set the dropout layers to test mode +# decoder models as usual. If you are using tracing mode(``torch.jit.trace``) +# for some part of your models, you must call ``.to(device)`` to set the device +# options of the models and ``.eval()`` to set the dropout layers to test mode # **before** tracing the models. `TracedModule` objects do not inherit the # ``to`` or ``eval`` methods. Since in this tutorial we are only using # scripting instead of tracing, we only need to do this before we do @@ -706,7 +706,7 @@ def evaluateExample(sentence, searcher, voc): # Configure models model_name = 'cb_model' attn_model = 'dot' -#attn_model = 'general' +#attn_model = 'general'`` #attn_model = 'concat' hidden_size = 500 encoder_n_layers = 2 @@ -717,7 +717,13 @@ def evaluateExample(sentence, searcher, voc): # If you're loading your own model # Set checkpoint to load from checkpoint_iter = 4000 -# loadFilename = os.path.join(save_dir, model_name, corpus_name, + +############################################################# +# Sample code to load from a checkpoint: +# +# .. code-block:: python +# +# loadFilename = os.path.join(save_dir, model_name, corpus_name, # '{}-{}_{}'.format(encoder_n_layers, decoder_n_layers, hidden_size), # '{}_checkpoint.tar'.format(checkpoint_iter)) @@ -743,13 +749,13 @@ def evaluateExample(sentence, searcher, voc): # Initialize encoder & decoder models encoder = EncoderRNN(hidden_size, embedding, encoder_n_layers, dropout) decoder = LuongAttnDecoderRNN(attn_model, embedding, hidden_size, voc.num_words, decoder_n_layers, dropout) -# Load trained model params +# Load trained model parameters encoder.load_state_dict(encoder_sd) decoder.load_state_dict(decoder_sd) # Use appropriate device encoder = encoder.to(device) decoder = decoder.to(device) -# Set dropout layers to eval mode +# Set dropout layers to ``eval`` mode encoder.eval() decoder.eval() print('Models built and ready to go!') @@ -794,7 +800,7 @@ def evaluateExample(sentence, searcher, voc): # data-dependent control flow. In the case of scripting, we do necessary # language changes to make sure the implementation complies with # TorchScript. We initialize the scripted searcher the same way that we -# would initialize an un-scripted variant. +# would initialize an unscripted variant. # ### Compile the whole greedy search model to TorchScript model @@ -847,7 +853,7 @@ def evaluateExample(sentence, searcher, voc): # Use appropriate device scripted_searcher.to(device) -# Set dropout layers to eval mode +# Set dropout layers to ``eval`` mode scripted_searcher.eval() # Evaluate examples @@ -855,8 +861,8 @@ def evaluateExample(sentence, searcher, voc): for s in sentences: evaluateExample(s, scripted_searcher, voc) -# Evaluate your input -#evaluateInput(traced_encoder, traced_decoder, scripted_searcher, voc) +# Evaluate your input by running +# ``evaluateInput(traced_encoder, traced_decoder, scripted_searcher, voc)`` ###################################################################### diff --git a/beginner_source/dist_overview.rst b/beginner_source/dist_overview.rst index 11b32a363..542eb31e0 100644 --- a/beginner_source/dist_overview.rst +++ b/beginner_source/dist_overview.rst @@ -3,7 +3,7 @@ PyTorch Distributed Overview **Author**: `Shen Li `_ .. note:: - |edit| View and edit this tutorial in `github `__. + |edit| View and edit this tutorial in `github `__. This is the overview page for the ``torch.distributed`` package. The goal of this page is to categorize documents into different topics and briefly @@ -126,7 +126,7 @@ DDP materials are listed below: described in the `Single-Machine Model Parallel Best Practices <../intermediate/model_parallel_tutorial.html>`__ tutorial. -3. The `Launching and configuring distributed data parallel applications `__ +3. The `Launching and configuring distributed data parallel applications `__ document shows how to use the DDP launching script. 4. The `Shard Optimizer States With ZeroRedundancyOptimizer <../recipes/zero_redundancy_optimizer.html>`__ recipe demonstrates how `ZeroRedundancyOptimizer `__ diff --git a/beginner_source/fgsm_tutorial.py b/beginner_source/fgsm_tutorial.py index 2257a24a1..45c72972f 100644 --- a/beginner_source/fgsm_tutorial.py +++ b/beginner_source/fgsm_tutorial.py @@ -13,15 +13,15 @@ 이 튜토리얼은 ML 모델들의 보안 취약점에 대한 인식을 높이고, 요즘 화두가 되고있는 적대적 머신 러닝에 대한 통찰력을 제공할 것입니다. 이미지에 눈치챌 수 없는 작은 변화(perturbation)를 추가하면 모델 성능이 크게 달라질 수 있다는 사실에 놀랄 수 있습니다. 이번 튜토리얼에서는 이미지 분류기의 예제를 통해 위 내용에 대해 살펴볼 것입니다. -특히 우리는 가장 많이 사용되는 공격 방법 중 하나인 FGSM (Fast Gradient Sign Attack)을 이용해 MNIST 분류기를 속여 볼 것입니다. - +특히 우리는 가장 많이 사용되는 공격 방법 중 하나인 FGSM (Fast Gradient Sign Attack)을 이용해 MNIST 분류기를 속여 볼 것입니다. + """ ###################################################################### # 위협 모델 # ------------ -# +# # 상황에 따라 다양한 범주의 적대적 공격이 있는데 각각 목표가 다르고 공격자가 알고 있는 정보 # 대한 가정도 다릅니다. 그러나 보통 가장 중요한 목표는 입력 데이터에 최소한의 작은 변화를 # 추가하여 이것이 의도적으로 잘못 분류되게 하는 것입니다. 공격자가 가지고 있는 정보에 대한 @@ -37,10 +37,10 @@ # # 이 경우 FGSM 공격은 *오분류* 를 목표로 하는 화이트 박스 공격입니다. # 이런 배경 정보를 갖고 공격에 대해 자세히 알아 보겠습니다. -# +# # 빠른 변화도 부호 공격 # ------------------------- -# +# # 공격 방법에 있어 초기 방식이면서 가장 유명한 방식은 *빠른 변화도 부호 공격 (FGSM)* 이라고 하며 # `적대적 예제에 대한 설명과 활용 `__ 에서 # 이안 갓펠로우가 기고하였습니다. @@ -95,16 +95,16 @@ # # 이 학습서에는 입력이 3 개이며 다음과 같이 정의됩니다: # -# - **epsilons** - 실행에 사용할 엡실론의 리스트입니다. 엡실론 0의 값은 원래 테스트 셋의 모델 성능을 +# - ``epsilons`` - 실행에 사용할 엡실론의 리스트입니다. 엡실론 0의 값은 원래 테스트 셋의 모델 성능을 # 나타내므로 목록에 유지하는 것이 중요합니다. 또한 직관적으로 엡실론이 클수록 작은 변화가 더 눈에 띄지만 # 모델 정확도를 저하 시키는 측면에서 더 효과가 있습니다. 여기서 데이터의 범위는 0-1 이기 때문에 # 엡실론의 값은 1을 초과할 수 없습니다. # -# - **pretrained_model** - `pytorch/examples/mnist `__ +# - ``pretrained_model`` - `pytorch/examples/mnist `__ # 를 통해 미리 학습된 MNIST 모델의 경로. # 튜토리얼을 간편하게 하려면 `여기 `__ 에서 미리 학습된 모델을 다운로드하세요. # -# - **use_cuda** - CUDA 를 사용할지 말지 정하는 이진 플래그. +# - ``use_cuda`` - CUDA 를 사용할지 말지 정하는 이진 플래그. # 본 튜토리얼에서는 CPU 시간이 오래 걸리지 않으므로 CUDA를 지원하는 GPU 의 여부는 중요하지 않습니다. # @@ -301,7 +301,7 @@ def test( model, device, test_loader, epsilon ): # :math:`\epsilon=0.2` 에서의 정확도는 :math:`\epsilon=0.15` 보다 약 25% 정도 낮습니다. # 또한, :math:`\epsilon=0.25` 와 :math:`\epsilon=0.3` 사이의 모델 정확도는 랜덤으로 # 10개중 1개를 선택했을 때의 정확도와 유사한 수준입니다. -# +# plt.figure(figsize=(5,5)) plt.plot(epsilons, accuracies, "*-") @@ -325,7 +325,7 @@ def test( model, device, test_loader, epsilon ): # 각 이미지의 위의 글자는 "원래 분류 결과 -> 적대적 분류 결과"를 나타냅니다. # :math:`\epsilon=0.15` 에서 작은 변화가 눈에 띄기 시작하고 :math:`\epsilon=0.3` 에서는 확실해 보입니다. # 그러나 모든 경우에 대해서 노이즈가 추가되었더라도 사람은 올바르게 분류를 수행할 수 있습니다. -# +# # 각 엡실론에서 적대적 샘플의 몇 가지 예를 도식화합니다 cnt = 0 diff --git a/beginner_source/flava_finetuning_tutorial.py b/beginner_source/flava_finetuning_tutorial.py index 92bc50317..12e20f475 100644 --- a/beginner_source/flava_finetuning_tutorial.py +++ b/beginner_source/flava_finetuning_tutorial.py @@ -24,7 +24,7 @@ ###################################################################### # Installation # ----------------- -# We will use TextVQA dataset and bert tokenizer from HuggingFace for this +# We will use TextVQA dataset and ``bert tokenizer`` from Hugging Face for this # tutorial. So you need to install datasets and transformers in addition to TorchMultimodal. # # .. note:: @@ -40,21 +40,21 @@ # ###################################################################### -# Steps +# Steps # ----- -# -# 1. Download the HuggingFace dataset to a directory on your computer by running the following command: -# +# +# 1. Download the Hugging Face dataset to a directory on your computer by running the following command: +# # .. code-block:: -# +# # wget http://dl.fbaipublicfiles.com/pythia/data/vocab.tar.gz # tar xf vocab.tar.gz -# +# # .. note:: # If you are running this tutorial in Google Colab, run these commands # in a new cell and prepend these commands with an exclamation mark (!) # -# +# # 2. For this tutorial, we treat VQA as a classification task where # the inputs are images and question (text) and the output is an answer class. # So we need to download the vocab file with answer classes and create the answer to @@ -62,7 +62,7 @@ # # We also load the `textvqa # dataset `__ containing 34602 training samples -# (images,questions and answers) from HuggingFace +# (images,questions and answers) from Hugging Face # # We see there are 3997 answer classes including a class representing # unknown answers. @@ -98,8 +98,8 @@ # 3. Next, we write the transform function to convert the image and text into # Tensors consumable by our model - For images, we use the transforms from # torchvision to convert to Tensor and resize to uniform sizes - For text, -# we tokenize (and pad) them using the BertTokenizer from HuggingFace - -# For answers (i.e. labels), we take the most frequently occuring answer +# we tokenize (and pad) them using the ``BertTokenizer`` from Hugging Face - +# For answers (i.e. labels), we take the most frequently occurring answer # as the label to train with: # @@ -133,8 +133,8 @@ def transform(tokenizer, input): ###################################################################### -# 4. Finally, we import the flava_model_for_classification from -# torchmultimodal. It loads the pretrained flava checkpoint by default and +# 4. Finally, we import the ``flava_model_for_classification`` from +# ``torchmultimodal``. It loads the pretrained FLAVA checkpoint by default and # includes a classification head. # # The model forward function passes the image through the visual encoder @@ -172,7 +172,7 @@ def transform(tokenizer, input): loss.backward() optimizer.step() print(f"Loss at step {idx} = {loss}") - if idx > MAX_STEPS-1: + if idx >= MAX_STEPS-1: break diff --git a/beginner_source/former_torchies/parallelism_tutorial.py b/beginner_source/former_torchies/parallelism_tutorial.py index 9e1c5f29b..ac7c8ec83 100644 --- a/beginner_source/former_torchies/parallelism_tutorial.py +++ b/beginner_source/former_torchies/parallelism_tutorial.py @@ -130,7 +130,7 @@ def forward(self, x): # - `포럼에서 PyTorch에 대해 얘기하기`_ # - `Slack에서 다른 사용자와 대화하기`_ # -# .. _`PyTorch로 딥러닝하기 : 60분만에 끝장내기`: https://github.com/pytorch/tutorials/blob/master/Deep%20Learning%20with%20PyTorch.ipynb +# .. _`PyTorch로 딥러닝하기 : 60분만에 끝장내기`: https://github.com/pytorch/tutorials/blob/main/Deep%20Learning%20with%20PyTorch.ipynb # .. _imagenet으로 최첨단(state-of-the-art) ResNet 신경망 학습시키기: https://github.com/pytorch/examples/tree/master/imagenet # .. _적대적 생성 신경망으로 얼굴 생성기 학습시키기: https://github.com/pytorch/examples/tree/master/dcgan # .. _순환 LSTM 네트워크를 사용해 단어 단위 언어 모델 학습시키기: https://github.com/pytorch/examples/tree/master/word_language_model diff --git a/beginner_source/hyperparameter_tuning_tutorial.py b/beginner_source/hyperparameter_tuning_tutorial.py index 7f45e0d58..3d4b57601 100644 --- a/beginner_source/hyperparameter_tuning_tutorial.py +++ b/beginner_source/hyperparameter_tuning_tutorial.py @@ -1,28 +1,38 @@ # -*- coding: utf-8 -*- """ Ray Tune을 이용한 하이퍼파라미터 튜닝 -=================================== +====================================== + **번역**: `심형준 `_ -하이퍼파라미터 튜닝은 보통의 모델과 매우 정확한 모델간의 차이를 만들어 낼 수 있습니다. + +하이퍼파라미터 튜닝은 보통의 모델과 매우 정확한 모델간의 차이를 만들어 낼 수 있습니다. 종종 다른 학습률(Learnig rate)을 선택하거나 layer size를 변경하는 것과 같은 간단한 작업만으로도 모델 성능에 큰 영향을 미치기도 합니다. + 다행히, 최적의 매개변수 조합을 찾는데 도움이 되는 도구가 있습니다. -`Ray Tune `_ 은 분산 하이퍼파라미터 튜닝을 위한 업계 표준 도구입니다. +`Ray Tune `_ 은 분산 하이퍼파라미터 튜닝을 위한 업계 표준 도구입니다. Ray Tune은 최신 하이퍼파라미터 검색 알고리즘을 포함하고 TensorBoard 및 기타 분석 라이브러리와 통합되며 기본적으로 -`Ray' 의 분산 기계 학습 엔진 -`_ 을 통해 교육을 지원합니다. +`Ray 의 분산 기계 학습 엔진 `_ 을 통해 학습을 지원합니다. + 이 튜토리얼은 Ray Tune을 파이토치 학습 workflow에 통합하는 방법을 알려줍니다. CIFAR10 이미지 분류기를 훈련하기 위해 `파이토치 문서에서 이 튜토리얼을 `_ 확장할 것입니다. + 아래와 같이 약간의 수정만 추가하면 됩니다. + 1. 함수에서 데이터 로딩 및 학습 부분을 감싸두고, 2. 일부 네트워크 파라미터를 구성 가능하게 하고, 3. 체크포인트를 추가하고 (선택 사항), 4. 모델 튜닝을 위한 검색 공간을 정의합니다. + | -이 튜토리얼을 실행하기 위해 아래의 패키지가 설치되어 있는지 확인하십시오. + +이 튜토리얼을 실행하기 위해 아래의 패키지가 설치되어 있는지 확인하세요: + - ``ray[tune]``: 배포된 하이퍼파라미터 튜닝 라이브러리 -- ``torchvision``: 데이터 트랜스포머의 경우 -설정 / Imports ---------------- +- ``torchvision``: 데이터 변형을 위해 필요 + +설정 / 불러오기 +----------------- + import들로 시작합니다. """ from functools import partial @@ -40,12 +50,12 @@ from ray.tune.schedulers import ASHAScheduler ###################################################################### -# 대부분의 import들은 파이토치 모델을 빌드하는데 필요합니다. +# 대부분의 import들은 파이토치 모델을 빌드하는데 필요합니다. # 마지막 세 개의 import들만 Ray Tune을 사용하기 위한 것입니다. # # Data loaders -# ------------ -# data loader를 자체 함수로 감싸두고 전역 데이터 디렉토리로 전달합니다. +# ------------- +# data loader를 자체 함수로 감싸두고 전역 데이터 디렉토리로 전달합니다. # 이런 식으로 서로 다른 실험들 간에 데이터 디렉토리를 공유할 수 있습니다. @@ -66,8 +76,8 @@ def load_data(data_dir="./data"): ###################################################################### # 구성 가능한 신경망 # --------------------------- -# 구성 가능한 파라미터만 튜닝이 가능합니다. -# 이 예시를 통해 fully connected layer 크기를 지정할 수 있습니다. +# 구성 가능한 파라미터만 튜닝이 가능합니다. +# 이 예시를 통해 fully connected layer 크기를 지정할 수 있습니다: class Net(nn.Module): @@ -92,12 +102,12 @@ def forward(self, x): ###################################################################### # 학습 함수 # ------------------ -# 흥미롭게 하기 위해 `파이토치 문서에서 `_ -# 예제에 일부를 변경하여 소개합니다. +# 흥미를 더해보고자 `파이토치 문서의 예제 `_ +# 일부를 변경하여 소개합니다. # -# 훈련 스크립트를 ``train_cifar(config, checkpoint_dir=None, data_dir=None)`` 함수로 감싸둡니다. +# 학습 스크립트를 ``train_cifar(config, checkpoint_dir=None, data_dir=None)`` 함수로 감싸둡니다. # 짐작할 수 있듯이, ``config`` 매개변수는 훈련할 하이퍼파라미터를 받습니다. ``checkpoint_dir`` 매개변수는 체크포인트를 -# 복원하는 데 사용됩니다. ``data_dir`` 은 데이터를 읽고 저장하는 디렉토리를 지정하므로, +# 복원하는 데 사용됩니다. ``data_dir`` 은 데이터를 읽고 저장하는 디렉토리를 지정하므로, # 여러 실행들이 동일한 데이터 소스를 공유할 수 있습니다. # # .. code-block:: python @@ -116,12 +126,12 @@ def forward(self, x): # # optimizer = optim.SGD(net.parameters(), lr=config["lr"], momentum=0.9) # -# 또한 학습 데이터를 학습 및 검증 세트로 나눕니다. 따라서 데이터의 80%는 모델 학습에 사용하고, +# 또한 학습 데이터를 학습 및 검증 세트로 나눕니다. 따라서 데이터의 80%는 모델 학습에 사용하고, # 나머지 20%에 대해 유효성 검사 및 손실을 계산합니다. 학습 및 테스트 세트를 반복하는 배치 크기도 구성할 수 있습니다. # # DataParallel을 이용한 GPU(다중)지원 추가 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -# 이미지 분류는 GPU를 사용할 때 이점이 많습니다. 운좋게도 Ray Tune에서 파이토치의 추상화를 계속 사용할 수 있습니다. +# 이미지 분류는 GPU를 사용할 때 이점이 많습니다. 운좋게도 Ray Tune에서 파이토치의 추상화를 계속 사용할 수 있습니다. # 따라서 여러 GPU에서 데이터 병렬 훈련을 지원하기 위해 모델을 ``nn.DataParallel`` 으로 감쌀 수 있습니다. # # .. code-block:: python @@ -133,7 +143,7 @@ def forward(self, x): # net = nn.DataParallel(net) # net.to(device) # -# ``device`` 변수를 사용하여 사용 가능한 GPU가 없을 때도 학습이 가능한지 확인합니다. +# ``device`` 변수를 사용하여 사용 가능한 GPU가 없을 때도 학습이 가능한지 확인합니다. # 파이토치는 다음과 같이 데이터를 GPU메모리에 명시적으로 보내도록 요구합니다. # # .. code-block:: python @@ -142,8 +152,8 @@ def forward(self, x): # inputs, labels = data # inputs, labels = inputs.to(device), labels.to(device) # -# 이 코드는 이제 CPU들, 단일 GPU 및 다중 GPU에 대한 학습을 지원합니다. -# 특히 Ray는 `부분GPU `_ 도 지원하므로 +# 이 코드는 이제 CPU들, 단일 GPU 및 다중 GPU에 대한 학습을 지원합니다. +# 특히 Ray는 `fractional-GPU `_ 도 지원하므로 # 모델이 GPU 메모리에 적합한 상황에서는 테스트 간에 GPU를 공유할 수 있습니다. 이는 나중에 다룰 것입니다. # # Ray Tune과 소통하기 @@ -159,17 +169,17 @@ def forward(self, x): # # tune.report(loss=(val_loss / val_steps), accuracy=correct / total) # -# 여기서 먼저 체크포인트를 저장한 다음 일부 메트릭을 Ray Tune에 다시 보냅니다. 특히, validation loss와 accuracy를 -# Ray Tune으로 다시 보냅니다. 그 후 Ray Tune은 이러한 메트릭을 사용하여 최상의 결과를 유도하는 하이퍼파라미터 구성을 +# 여기서 먼저 체크포인트를 저장한 다음 일부 메트릭을 Ray Tune에 다시 보냅니다. 특히, validation loss와 accuracy를 +# Ray Tune으로 다시 보냅니다. 그 후 Ray Tune은 이러한 메트릭을 사용하여 최상의 결과를 유도하는 하이퍼파라미터 구성을 # 결정할 수 있습니다. 이러한 메트릭들은 또한 리소스 낭비를 방지하기 위해 성능이 좋지 않은 실험을 조기에 중지하는 데 사용할 수 있습니다. # -# 체크포인트 저장은 선택사항이지만 `Population Based Training `_ +# 체크포인트 저장은 선택사항이지만 `Population Based Training `_ # 과 같은 고급 스케줄러를 사용하려면 필요합니다. 또한 체크포인트를 저장하면 나중에 학습된 모델을 로드하고 평가 세트(test set)에서 검증할 수 있습니다. # # Full training function -# ~~~~~~~~~~~~~~~~~~~~~~ +# ~~~~~~~~~~~~~~~~~~~~~~~~ # -# 전체 코드 예제는 다음과 같습니다. +# 전체 예제 코드는 다음과 같습니다. def train_cifar(config, checkpoint_dir=None, data_dir=None): @@ -264,7 +274,7 @@ def train_cifar(config, checkpoint_dir=None, data_dir=None): # # Test set 정확도(accuracy) # ----------------- -# 일반적으로 머신러닝 모델의 성능은 모델 학습에 사용되지 않은 데이터를 사용해 테스트합니다. +# 일반적으로 머신러닝 모델의 성능은 모델 학습에 사용되지 않은 데이터를 사용해 테스트합니다. # Test set 또한 함수로 감싸둘 수 있습니다. @@ -303,11 +313,11 @@ def test_accuracy(net, device="cpu"): # "batch_size": tune.choice([2, 4, 8, 16]) # } # -# ``tune.sample_from()`` 함수를 사용하면 고유한 샘플 방법을 정의하여 하이퍼파라미터를 얻을 수 있습니다. -# 이 예제에서 ``l1`` 과 ``l2`` 파라미터는 4와 256 사이의 2의 거듭제곱이어야 하므로 4, 8, 16, 32, 64, 128, 256입니다. +# ``tune.sample_from()`` 함수를 사용하면 고유한 샘플 방법을 정의하여 하이퍼파라미터를 얻을 수 있습니다. +# 이 예제에서 ``l1`` 과 ``l2`` 파라미터는 4와 256 사이의 2의 거듭제곱이어야 하므로 4, 8, 16, 32, 64, 128, 256입니다. # ``lr`` (학습률)은 0.0001과 0.1 사이에서 균일하게 샘플링 되어야 합니다. 마지막으로, 배치 크기는 2, 4, 8, 16중에서 선택할 수 있습니다. # -# 각 실험에서, Ray Tune은 이제 이러한 검색 공간에서 매개변수 조합을 무작위로 샘플링합니다. +# 각 실험에서, Ray Tune은 이제 이러한 검색 공간에서 매개변수 조합을 무작위로 샘플링합니다. # 그런 다음 여러 모델을 병렬로 훈련하고 이 중에서 가장 성능이 좋은 모델을 찾습니다. 또한 성능이 좋지 않은 실험을 조기에 종료하는 ``ASHAScheduler`` 를 사용합니다. # # 상수 ``data_dir`` 파라미터를 설정하기 위해 ``functools.partial`` 로 ``train_cifar`` 함수를 감싸둡니다. 또한 각 실험에 사용할 수 있는 자원들(resources)을 Ray Tune에 알릴 수 있습니다. @@ -325,7 +335,7 @@ def test_accuracy(net, device="cpu"): # progress_reporter=reporter, # checkpoint_at_end=True) # -# 파이토치 ``DataLoader`` 인스턴스의 ``num_workers`` 을 늘리기 위해 CPU 수를 지정하고 사용할 수 있습니다. +# 파이토치 ``DataLoader`` 인스턴스의 ``num_workers`` 을 늘리기 위해 CPU 수를 지정하고 사용할 수 있습니다. # 각 실험에서 선택한 수의 GPU들은 파이토치에 표시됩니다. 실험들은 요청되지 않은 GPU에 액세스할 수 없으므로 같은 자원들을 사용하는 중복된 실험에 대해 신경쓰지 않아도 됩니다. # # 부분 GPUs를 지정할 수도 있으므로, ``gpus_per_trial=0.5`` 와 같은 것 또한 가능합니다. 이후 각 실험은 GPU를 공유합니다. 사용자는 모델이 여전히 GPU메모리에 적합한지만 확인하면 됩니다. @@ -351,7 +361,7 @@ def main(num_samples=10, max_num_epochs=10, gpus_per_trial=2): grace_period=1, reduction_factor=2) reporter = CLIReporter( - # parameter_columns=["l1", "l2", "lr", "batch_size"], + # ``parameter_columns=["l1", "l2", "lr", "batch_size"]``, metric_columns=["loss", "accuracy", "training_iteration"]) result = tune.run( partial(train_cifar, data_dir=data_dir), @@ -386,6 +396,12 @@ def main(num_samples=10, max_num_epochs=10, gpus_per_trial=2): if __name__ == "__main__": + # sphinx_gallery_start_ignore + # Fixes ``AttributeError: '_LoggingTee' object has no attribute 'fileno'``. + # This is only needed to run with sphinx-build. + import sys + sys.stdout.fileno = lambda: False + # sphinx_gallery_end_ignore # You can change the number of GPUs per trial here: main(num_samples=10, max_num_epochs=10, gpus_per_trial=0) diff --git a/beginner_source/introyt/autogradyt_tutorial.py b/beginner_source/introyt/autogradyt_tutorial.py index 19c25c5e8..a2ed238e5 100644 --- a/beginner_source/introyt/autogradyt_tutorial.py +++ b/beginner_source/introyt/autogradyt_tutorial.py @@ -153,7 +153,7 @@ ####################################################################### # This ``grad_fn`` gives us a hint that when we execute the # backpropagation step and compute gradients, we’ll need to compute the -# derivative of :math:`sin(x)` for all this tensor’s inputs. +# derivative of :math:`\sin(x)` for all this tensor’s inputs. # # Let’s perform some more computations: # @@ -222,8 +222,8 @@ # out = d.sum() # # Adding a constant, as we did to compute ``d``, does not change the -# derivative. That leaves :math:`c = 2 * b = 2 * sin(a)`, the derivative -# of which should be :math:`2 * cos(a)`. Looking at the graph above, +# derivative. That leaves :math:`c = 2 * b = 2 * \sin(a)`, the derivative +# of which should be :math:`2 * \cos(a)`. Looking at the graph above, # that’s just what we see. # # Be aware that only *leaf nodes* of the computation have their gradients @@ -334,7 +334,7 @@ def forward(self, x): print(model.layer2.weight.grad[0][0:10]) -optimizer.zero_grad() +optimizer.zero_grad(set_to_none=False) print(model.layer2.weight.grad[0][0:10]) diff --git a/beginner_source/introyt/captumyt.py b/beginner_source/introyt/captumyt.py index 5e1bb9201..2ff8e9e70 100644 --- a/beginner_source/introyt/captumyt.py +++ b/beginner_source/introyt/captumyt.py @@ -106,13 +106,13 @@ To install Captum in an Anaconda or pip virtual environment, use the appropriate command for your environment below: -With ``conda``: +With ``conda``:: -``conda install pytorch torchvision captum -c pytorch`` + conda install pytorch torchvision captum -c pytorch -With ``pip``: +With ``pip``:: -``pip install torch torchvision captum`` + pip install torch torchvision captum Restart this notebook in the environment you set up, and you’re ready to go! @@ -155,7 +155,7 @@ # now. # -model = models.resnet101(pretrained=True) +model = models.resnet101(weights='IMAGENET1K_V1') model = model.eval() diff --git a/beginner_source/introyt/introyt1_tutorial.py b/beginner_source/introyt/introyt1_tutorial.py index 77a72f1e1..9aa32238c 100644 --- a/beginner_source/introyt/introyt1_tutorial.py +++ b/beginner_source/introyt/introyt1_tutorial.py @@ -173,11 +173,11 @@ class LeNet(nn.Module): def __init__(self): super(LeNet, self).__init__() - # 입력 이미지 채널, 6개의 output 채널, 3x3 정방 합성곱 커널을 사용합니다. - self.conv1 = nn.Conv2d(1, 6, 3) - self.conv2 = nn.Conv2d(6, 16, 3) + # 입력 이미지 채널, 6개의 output 채널, 5x5 정방 합성곱 커널을 사용합니다. + self.conv1 = nn.Conv2d(1, 6, 5) + self.conv2 = nn.Conv2d(6, 16, 5) # 아핀 변환: y = Wx + b - self.fc1 = nn.Linear(16 * 6 * 6, 120) # 6*6 이미지 차원 + self.fc1 = nn.Linear(16 * 5 * 5, 120) # 5x5 이미지 차원 self.fc2 = nn.Linear(120, 84) self.fc3 = nn.Linear(84, 10) diff --git a/beginner_source/introyt/modelsyt_tutorial.py b/beginner_source/introyt/modelsyt_tutorial.py index 884fcbdb1..8126ce841 100644 --- a/beginner_source/introyt/modelsyt_tutorial.py +++ b/beginner_source/introyt/modelsyt_tutorial.py @@ -46,15 +46,15 @@ class is a subclass of ``torch.Tensor``, with the special behavior that import torch class TinyModel(torch.nn.Module): - + def __init__(self): super(TinyModel, self).__init__() - + self.linear1 = torch.nn.Linear(100, 200) self.activation = torch.nn.ReLU() self.linear2 = torch.nn.Linear(200, 10) self.softmax = torch.nn.Softmax() - + def forward(self, x): x = self.linear1(x) x = self.activation(x) @@ -85,19 +85,19 @@ def forward(self, x): # model, and a ``forward()`` method where the computation gets done. Note # that we can print the model, or any of its submodules, to learn about # its structure. -# +# # Common Layer Types # ------------------ -# +# # Linear Layers # ~~~~~~~~~~~~~ -# +# # The most basic type of neural network layer is a *linear* or *fully # connected* layer. This is a layer where every input influences every # output of the layer to a degree specified by the layer’s weights. If a # model has *m* inputs and *n* outputs, the weights will be an *m* x *n* # matrix. For example: -# +# lin = torch.nn.Linear(3, 2) x = torch.rand(1, 3) @@ -117,22 +117,22 @@ def forward(self, x): # If you do the matrix multiplication of ``x`` by the linear layer’s # weights, and add the biases, you’ll find that you get the output vector # ``y``. -# +# # One other important feature to note: When we checked the weights of our # layer with ``lin.weight``, it reported itself as a ``Parameter`` (which # is a subclass of ``Tensor``), and let us know that it’s tracking # gradients with autograd. This is a default behavior for ``Parameter`` # that differs from ``Tensor``. -# +# # Linear layers are used widely in deep learning models. One of the most # common places you’ll see them is in classifier models, which will # usually have one or more linear layers at the end, where the last layer # will have *n* outputs, where *n* is the number of classes the classifier # addresses. -# +# # Convolutional Layers # ~~~~~~~~~~~~~~~~~~~~ -# +# # *Convolutional* layers are built to handle data with a high degree of # spatial correlation. They are very commonly used in computer vision, # where they detect close groupings of features which the compose into @@ -140,9 +140,9 @@ def forward(self, x): # in NLP applications, where a word’s immediate context (that is, the # other words nearby in the sequence) can affect the meaning of a # sentence. -# +# # We saw convolutional layers in action in LeNet5 in an earlier video: -# +# import torch.functional as F @@ -182,7 +182,7 @@ def num_flat_features(self, x): ########################################################################## # Let’s break down what’s happening in the convolutional layers of this # model. Starting with ``conv1``: -# +# # - LeNet5 is meant to take in a 1x32x32 black & white image. **The first # argument to a convolutional layer’s constructor is the number of # input channels.** Here, it is 1. If we were building this model to @@ -198,14 +198,14 @@ def num_flat_features(self, x): # size.** Here, the “5” means we’ve chosen a 5x5 kernel. (If you want a # kernel with height different from width, you can specify a tuple for # this argument - e.g., ``(3, 5)`` to get a 3x5 convolution kernel.) -# +# # The output of a convolutional layer is an *activation map* - a spatial # representation of the presence of features in the input tensor. # ``conv1`` will give us an output tensor of 6x28x28; 6 is the number of # features, and 28 is the height and width of our map. (The 28 comes from # the fact that when scanning a 5-pixel window over a 32-pixel row, there # are only 28 valid positions.) -# +# # We then pass the output of the convolution through a ReLU activation # function (more on activation functions later), then through a max # pooling layer. The max pooling layer takes features near each other in @@ -214,14 +214,14 @@ def num_flat_features(self, x): # cell, and assigning that cell the maximum value of the 4 cells that went # into it. This gives us a lower-resolution version of the activation map, # with dimensions 6x14x14. -# +# # Our next convolutional layer, ``conv2``, expects 6 input channels # (corresponding to the 6 features sought by the first layer), has 16 # output channels, and a 3x3 kernel. It puts out a 16x12x12 activation # map, which is again reduced by a max pooling layer to 16x6x6. Prior to # passing this output to the linear layers, it is reshaped to a 16 \* 6 \* # 6 = 576-element vector for consumption by the next layer. -# +# # There are convolutional layers for addressing 1D, 2D, and 3D tensors. # There are also many more optional arguments for a conv layer # constructor, including stride length(e.g., only scanning every second or @@ -229,22 +229,22 @@ def num_flat_features(self, x): # edges of the input), and more. See the # `documentation `__ # for more information. -# +# # Recurrent Layers # ~~~~~~~~~~~~~~~~ -# +# # *Recurrent neural networks* (or *RNNs)* are used for sequential data - # anything from time-series measurements from a scientific instrument to # natural language sentences to DNA nucleotides. An RNN does this by # maintaining a *hidden state* that acts as a sort of memory for what it # has seen in the sequence so far. -# +# # The internal structure of an RNN layer - or its variants, the LSTM (long # short-term memory) and GRU (gated recurrent unit) - is moderately # complex and beyond the scope of this video, but we’ll show you what one # looks like in action with an LSTM-based part-of-speech tagger (a type of # classifier that tells you if a word is a noun, verb, etc.): -# +# class LSTMTagger(torch.nn.Module): @@ -271,7 +271,7 @@ def forward(self, sentence): ######################################################################## # The constructor has four arguments: -# +# # - ``vocab_size`` is the number of words in the input vocabulary. Each # word is a one-hot vector (or unit vector) in a # ``vocab_size``-dimensional space. @@ -281,7 +281,7 @@ def forward(self, sentence): # space, where words with similar meanings are close together in the # space. # - ``hidden_dim`` is the size of the LSTM’s memory. -# +# # The input will be a sentence with the words represented as indices of # one-hot vectors. The embedding layer will then map these down to an # ``embedding_dim``-dimensional space. The LSTM takes this sequence of @@ -290,15 +290,15 @@ def forward(self, sentence): # ``log_softmax()`` to the output of the final layer converts the output # into a normalized set of estimated probabilities that a given word maps # to a given tag. -# +# # If you’d like to see this network in action, check out the `Sequence # Models and LSTM -# Networks `__ +# Networks `__ # tutorial on pytorch.org. -# +# # Transformers # ~~~~~~~~~~~~ -# +# # *Transformers* are multi-purpose networks that have taken over the state # of the art in NLP with models like BERT. A discussion of transformer # architecture is beyond the scope of this video, but PyTorch has a @@ -312,22 +312,22 @@ def forward(self, sentence): # ``TransformerDecoderLayer``). For details, check out the # `documentation `__ # on transformer classes, and the relevant -# `tutorial `__ +# `tutorial `__ # on pytorch.org. -# +# # Other Layers and Functions # -------------------------- -# +# # Data Manipulation Layers # ~~~~~~~~~~~~~~~~~~~~~~~~ -# +# # There are other layer types that perform important functions in models, # but don’t participate in the learning process themselves. -# +# # **Max pooling** (and its twin, min pooling) reduce a tensor by combining # cells, and assigning the maximum value of the input cells to the output # cell (we saw this). For example: -# +# my_tensor = torch.rand(1, 6, 6) print(my_tensor) @@ -340,12 +340,12 @@ def forward(self, sentence): # If you look closely at the values above, you’ll see that each of the # values in the maxpooled output is the maximum value of each quadrant of # the 6x6 input. -# +# # **Normalization layers** re-center and normalize the output of one layer # before feeding it to another. Centering the and scaling the intermediate # tensors has a number of beneficial effects, such as letting you use # higher learning rates without exploding/vanishing gradients. -# +# my_tensor = torch.rand(1, 4, 4) * 20 + 5 print(my_tensor) @@ -366,22 +366,22 @@ def forward(self, sentence): # in the neighborhood of 15. After running it through the normalization # layer, you can see that the values are smaller, and grouped around zero # - in fact, the mean should be very small (> 1e-8). -# +# # This is beneficial because many activation functions (discussed below) # have their strongest gradients near 0, but sometimes suffer from # vanishing or exploding gradients for inputs that drive them far away # from zero. Keeping the data centered around the area of steepest # gradient will tend to mean faster, better learning and higher feasible # learning rates. -# +# # **Dropout layers** are a tool for encouraging *sparse representations* # in your model - that is, pushing it to do inference with less data. -# +# # Dropout layers work by randomly setting parts of the input tensor # *during training* - dropout layers are always turned off for inference. # This forces the model to learn against this masked or reduced dataset. # For example: -# +# my_tensor = torch.rand(1, 4, 4) @@ -394,10 +394,10 @@ def forward(self, sentence): # Above, you can see the effect of dropout on a sample tensor. You can use # the optional ``p`` argument to set the probability of an individual # weight dropping out; if you don’t it defaults to 0.5. -# +# # Activation Functions # ~~~~~~~~~~~~~~~~~~~~ -# +# # Activation functions make deep learning possible. A neural network is # really a program - with many parameters - that *simulates a mathematical # function*. If all we did was multiple tensors by layer weights @@ -406,17 +406,17 @@ def forward(self, sentence): # reduce could be reduced to a single matrix multiplication. Inserting # *non-linear* activation functions between layers is what allows a deep # learning model to simulate any function, rather than just linear ones. -# +# # ``torch.nn.Module`` has objects encapsulating all of the major # activation functions including ReLU and its many variants, Tanh, # Hardtanh, sigmoid, and more. It also includes other functions, such as # Softmax, that are most useful at the output stage of a model. -# +# # Loss Functions # ~~~~~~~~~~~~~~ -# +# # Loss functions tell us how far a model’s prediction is from the correct # answer. PyTorch contains a variety of loss functions, including common # MSE (mean squared error = L2 norm), Cross Entropy Loss and Negative # Likelihood Loss (useful for classifiers), and others. -# +# diff --git a/beginner_source/introyt/tensorboardyt_tutorial.py b/beginner_source/introyt/tensorboardyt_tutorial.py index 3b07fe0b8..8e3263204 100644 --- a/beginner_source/introyt/tensorboardyt_tutorial.py +++ b/beginner_source/introyt/tensorboardyt_tutorial.py @@ -24,14 +24,14 @@ To run this tutorial, you’ll need to install PyTorch, TorchVision, Matplotlib, and TensorBoard. -With ``conda``: +With ``conda``:: -``conda install pytorch torchvision -c pytorch`` -``conda install matplotlib tensorboard`` + conda install pytorch torchvision -c pytorch + conda install matplotlib tensorboard -With ``pip``: +With ``pip``:: -``pip install torch torchvision matplotlib tensorboard`` + pip install torch torchvision matplotlib tensorboard Once the dependencies are installed, restart this notebook in the Python environment where you installed them. @@ -39,11 +39,11 @@ Introduction ------------ - + In this notebook, we’ll be training a variant of LeNet-5 against the Fashion-MNIST dataset. Fashion-MNIST is a set of image tiles depicting various garments, with ten class labels indicating the type of garment -depicted. +depicted. """ @@ -68,9 +68,9 @@ ###################################################################### # Showing Images in TensorBoard # ----------------------------- -# +# # Let’s start by adding sample images from our dataset to TensorBoard: -# +# # Gather datasets and prepare them for consumption transform = transforms.Compose( @@ -127,7 +127,7 @@ def matplotlib_imshow(img, one_channel=False): # minibatch of our input data. Below, we use the ``add_image()`` call on # ``SummaryWriter`` to log the image for consumption by TensorBoard, and # we also call ``flush()`` to make sure it’s written to disk right away. -# +# # Default log_dir argument is "runs" - but it's good to be specific # torch.utils.tensorboard.SummaryWriter is imported above @@ -146,17 +146,17 @@ def matplotlib_imshow(img, one_channel=False): # If you start TensorBoard at the command line and open it in a new # browser tab (usually at `localhost:6006 `__), you should # see the image grid under the IMAGES tab. -# +# # Graphing Scalars to Visualize Training # -------------------------------------- -# +# # TensorBoard is useful for tracking the progress and efficacy of your # training. Below, we’ll run a training loop, track some metrics, and save # the data for TensorBoard’s consumption. -# +# # Let’s define a model to categorize our image tiles, and an optimizer and # loss function for training: -# +# class Net(nn.Module): def __init__(self): @@ -176,7 +176,7 @@ def forward(self, x): x = F.relu(self.fc2(x)) x = self.fc3(x) return x - + net = Net() criterion = nn.CrossEntropyLoss() @@ -186,7 +186,7 @@ def forward(self, x): ########################################################################## # Now let’s train a single epoch, and evaluate the training vs. validation # set losses every 1000 batches: -# +# print(len(validation_loader)) for epoch in range(1): # loop over the dataset multiple times @@ -206,7 +206,7 @@ def forward(self, x): print('Batch {}'.format(i + 1)) # Check against the validation set running_vloss = 0.0 - + net.train(False) # Don't need to track gradents for validation for j, vdata in enumerate(validation_loader, 0): vinputs, vlabels = vdata @@ -214,10 +214,10 @@ def forward(self, x): vloss = criterion(voutputs, vlabels) running_vloss += vloss.item() net.train(True) # Turn gradients back on for training - + avg_loss = running_loss / 1000 avg_vloss = running_vloss / len(validation_loader) - + # Log the running loss averaged per batch writer.add_scalars('Training vs. Validation Loss', { 'Training' : avg_loss, 'Validation' : avg_vloss }, @@ -231,14 +231,14 @@ def forward(self, x): ######################################################################### # Switch to your open TensorBoard and have a look at the SCALARS tab. -# +# # Visualizing Your Model # ---------------------- -# +# # TensorBoard can also be used to examine the data flow within your model. # To do this, call the ``add_graph()`` method with a model and sample # input. When you open -# +# # Again, grab a single mini-batch of images dataiter = iter(training_loader) @@ -254,10 +254,10 @@ def forward(self, x): # When you switch over to TensorBoard, you should see a GRAPHS tab. # Double-click the “NET” node to see the layers and data flow within your # model. -# +# # Visualizing Your Dataset with Embeddings # ---------------------------------------- -# +# # The 28-by-28 image tiles we’re using can be modeled as 784-dimensional # vectors (28 \* 28 = 784). It can be instructive to project this to a # lower-dimensional representation. The ``add_embedding()`` method will @@ -265,9 +265,9 @@ def forward(self, x): # and display them as an interactive 3D chart. The ``add_embedding()`` # method does this automatically by projecting to the three dimensions # with highest variance. -# +# # Below, we’ll take a sample of our data, and generate such an embedding: -# +# # Select a random subset of data and corresponding labels def select_n_random(data, labels, n=100): @@ -297,19 +297,19 @@ def select_n_random(data, labels, n=100): # zoom the model. Examine it at large and small scales, and see whether # you can spot patterns in the projected data and the clustering of # labels. -# +# # For better visibility, it’s recommended to: -# +# # - Select “label” from the “Color by” drop-down on the left. # - Toggle the Night Mode icon along the top to place the # light-colored images on a dark background. -# +# # Other Resources # --------------- -# +# # For more information, have a look at: -# +# # - PyTorch documentation on `torch.utils.tensorboard.SummaryWriter `__ -# - Tensorboard tutorial content in the `PyTorch.org Tutorials `__ +# - Tensorboard tutorial content in the `PyTorch.org Tutorials `__ # - For more information about TensorBoard, see the `TensorBoard # documentation `__ diff --git a/beginner_source/introyt/tensors_deeper_tutorial.py b/beginner_source/introyt/tensors_deeper_tutorial.py index 1f6d72488..8b2c1630a 100644 --- a/beginner_source/introyt/tensors_deeper_tutorial.py +++ b/beginner_source/introyt/tensors_deeper_tutorial.py @@ -544,11 +544,11 @@ print(c) # contents of c have changed assert c is d # test c & d are same object, not just containing equal values -assert id(c), old_id # make sure that our new c is the same object as the old one +assert id(c) == old_id # make sure that our new c is the same object as the old one torch.rand(2, 2, out=c) # works for creation too! print(c) # c has changed again -assert id(c), old_id # still the same object! +assert id(c) == old_id # still the same object! ########################################################################## diff --git a/beginner_source/introyt/trainingyt.py b/beginner_source/introyt/trainingyt.py index 5c374ca7a..84750da77 100644 --- a/beginner_source/introyt/trainingyt.py +++ b/beginner_source/introyt/trainingyt.py @@ -25,13 +25,13 @@ - Building models with the neural network layers and functions of the torch.nn module - The mechanics of automated gradient computation, which is central to - gradient-based model training + gradient-based model training - Using TensorBoard to visualize training progress and other activities In this video, we’ll be adding some new tools to your inventory: - We’ll get familiar with the dataset and dataloader abstractions, and how - they ease the process of feeding data to your model during a training loop + they ease the process of feeding data to your model during a training loop - We’ll discuss specific loss functions and when to use them - We’ll look at PyTorch optimizers, which implement algorithms to adjust model weights based on the outcome of a loss function @@ -42,26 +42,26 @@ Dataset and DataLoader ---------------------- - + The ``Dataset`` and ``DataLoader`` classes encapsulate the process of pulling your data from storage and exposing it to your training loop in batches. The ``Dataset`` is responsible for accessing and processing single instances of data. - + The ``DataLoader`` pulls instances of data from the ``Dataset`` (either automatically or with a sampler that you define), collects them in batches, and returns them for consumption by your training loop. The ``DataLoader`` works with all kinds of datasets, regardless of the type of data they contain. - + For this tutorial, we’ll be using the Fashion-MNIST dataset provided by TorchVision. We use ``torchvision.transforms.Normalize()`` to zero-center and normalize the distribution of the image tile content, and download both training and validation data splits. -""" +""" import torch import torchvision @@ -81,8 +81,8 @@ validation_set = torchvision.datasets.FashionMNIST('./data', train=False, transform=transform, download=True) # Create data loaders for our datasets; shuffle for training, not for validation -training_loader = torch.utils.data.DataLoader(training_set, batch_size=4, shuffle=True, num_workers=2) -validation_loader = torch.utils.data.DataLoader(validation_set, batch_size=4, shuffle=False, num_workers=2) +training_loader = torch.utils.data.DataLoader(training_set, batch_size=4, shuffle=True) +validation_loader = torch.utils.data.DataLoader(validation_set, batch_size=4, shuffle=False) # Class labels classes = ('T-shirt/top', 'Trouser', 'Pullover', 'Dress', 'Coat', @@ -95,7 +95,7 @@ ###################################################################### # As always, let’s visualize the data as a sanity check: -# +# import matplotlib.pyplot as plt import numpy as np @@ -123,10 +123,10 @@ def matplotlib_imshow(img, one_channel=False): ######################################################################### # The Model # --------- -# +# # The model we’ll use in this example is a variant of LeNet-5 - it should # be familiar if you’ve watched the previous videos in this series. -# +# import torch.nn as nn import torch.nn.functional as F @@ -150,7 +150,7 @@ def forward(self, x): x = F.relu(self.fc2(x)) x = self.fc3(x) return x - + model = GarmentClassifier() @@ -158,11 +158,11 @@ def forward(self, x): ########################################################################## # Loss Function # ------------- -# +# # For this example, we’ll be using a cross-entropy loss. For demonstration # purposes, we’ll create batches of dummy output and label values, run # them through the loss function, and examine the result. -# +# loss_fn = torch.nn.CrossEntropyLoss() @@ -171,7 +171,7 @@ def forward(self, x): dummy_outputs = torch.rand(4, 10) # Represents the correct class among the 10 being tested dummy_labels = torch.tensor([1, 5, 3, 7]) - + print(dummy_outputs) print(dummy_labels) @@ -182,21 +182,21 @@ def forward(self, x): ################################################################################# # Optimizer # --------- -# +# # For this example, we’ll be using simple `stochastic gradient # descent `__ with momentum. -# +# # It can be instructive to try some variations on this optimization # scheme: -# +# # - Learning rate determines the size of the steps the optimizer # takes. What does a different learning rate do to the your training # results, in terms of accuracy and convergence time? # - Momentum nudges the optimizer in the direction of strongest gradient over -# multiple steps. What does changing this value do to your results? +# multiple steps. What does changing this value do to your results? # - Try some different optimization algorithms, such as averaged SGD, Adagrad, or # Adam. How do your results differ? -# +# # Optimizers specified in the torch.optim package optimizer = torch.optim.SGD(model.parameters(), lr=0.001, momentum=0.9) @@ -205,13 +205,13 @@ def forward(self, x): ####################################################################################### # The Training Loop # ----------------- -# +# # Below, we have a function that performs one training epoch. It # enumerates data from the DataLoader, and on each pass of the loop does # the following: -# +# # - Gets a batch of training data from the DataLoader -# - Zeros the optimizer’s gradients +# - Zeros the optimizer’s gradients # - Performs an inference - that is, gets predictions from the model for an input batch # - Calculates the loss for that set of predictions vs. the labels on the dataset # - Calculates the backward gradients over the learning weights @@ -221,32 +221,32 @@ def forward(self, x): # - It reports on the loss for every 1000 batches. # - Finally, it reports the average per-batch loss for the last # 1000 batches, for comparison with a validation run -# +# def train_one_epoch(epoch_index, tb_writer): running_loss = 0. last_loss = 0. - + # Here, we use enumerate(training_loader) instead of # iter(training_loader) so that we can track the batch # index and do some intra-epoch reporting for i, data in enumerate(training_loader): # Every data instance is an input + label pair inputs, labels = data - + # Zero your gradients for every batch! optimizer.zero_grad() - + # Make predictions for this batch outputs = model(inputs) - + # Compute the loss and its gradients loss = loss_fn(outputs, labels) loss.backward() - + # Adjust learning weights optimizer.step() - + # Gather data and report running_loss += loss.item() if i % 1000 == 999: @@ -255,24 +255,24 @@ def train_one_epoch(epoch_index, tb_writer): tb_x = epoch_index * len(training_loader) + i + 1 tb_writer.add_scalar('Loss/train', last_loss, tb_x) running_loss = 0. - + return last_loss ################################################################################## # Per-Epoch Activity # ~~~~~~~~~~~~~~~~~~ -# -# There are a couple of things we’ll want to do once per epoch: +# +# There are a couple of things we’ll want to do once per epoch: # # - Perform validation by checking our relative loss on a set of data that was not -# used for training, and report this +# used for training, and report this # - Save a copy of the model -# +# # Here, we’ll do our reporting in TensorBoard. This will require going to # the command line to start TensorBoard, and opening it in another browser # tab. -# +# # Initializing in a separate cell so we can easily add more epochs to the same run timestamp = datetime.now().strftime('%Y%m%d_%H%M%S') @@ -285,58 +285,58 @@ def train_one_epoch(epoch_index, tb_writer): for epoch in range(EPOCHS): print('EPOCH {}:'.format(epoch_number + 1)) - + # Make sure gradient tracking is on, and do a pass over the data model.train(True) avg_loss = train_one_epoch(epoch_number, writer) - + # We don't need gradients on to do reporting model.train(False) - + running_vloss = 0.0 for i, vdata in enumerate(validation_loader): vinputs, vlabels = vdata voutputs = model(vinputs) vloss = loss_fn(voutputs, vlabels) running_vloss += vloss - + avg_vloss = running_vloss / (i + 1) print('LOSS train {} valid {}'.format(avg_loss, avg_vloss)) - + # Log the running loss averaged per batch # for both training and validation writer.add_scalars('Training vs. Validation Loss', { 'Training' : avg_loss, 'Validation' : avg_vloss }, epoch_number + 1) writer.flush() - + # Track best performance, and save the model's state if avg_vloss < best_vloss: best_vloss = avg_vloss model_path = 'model_{}_{}'.format(timestamp, epoch_number) torch.save(model.state_dict(), model_path) - + epoch_number += 1 ######################################################################### # To load a saved version of the model: -# -# :: -# -# saved_model = GarmentClassifier() -# saved_model.load_state_dict(torch.load(PATH)) -# +# +# .. code:: python +# +# saved_model = GarmentClassifier() +# saved_model.load_state_dict(torch.load(PATH)) +# # Once you’ve loaded the model, it’s ready for whatever you need it for - # more training, inference, or analysis. -# +# # Note that if your model has constructor parameters that affect model # structure, you’ll need to provide them and configure the model # identically to the state in which it was saved. -# +# # Other Resources # --------------- -# +# # - Docs on the `data # utilities `__, including # Dataset and DataLoader, at pytorch.org @@ -355,10 +355,10 @@ def train_one_epoch(epoch_index, tb_writer): # includes optimizers and related tools, such as learning rate # scheduling # - A detailed `tutorial on saving and loading -# models `__ +# models `__ # - The `Tutorials section of -# pytorch.org `__ contains tutorials on +# pytorch.org `__ contains tutorials on # a broad variety of training tasks, including classification in # different domains, generative adversarial networks, reinforcement -# learning, and more -# +# learning, and more +# diff --git a/beginner_source/nn_tutorial.py b/beginner_source/nn_tutorial.py index c629b4d0e..e9bf54214 100644 --- a/beginner_source/nn_tutorial.py +++ b/beginner_source/nn_tutorial.py @@ -2,16 +2,14 @@ """ `torch.nn` 이 *실제로* 무엇인가요? ===================================== -저자: Jeremy Howard, `fast.ai `_. -도움: Rachel Thomas, Francisco Ingham. - -번역: `남상호 `_ +**저자**: Jeremy Howard, `fast.ai `_. Rachel Thomas, Francisco Ingham에 감사합니다. +**번역**: `남상호 `_ """ ############################################################################### -# 이 튜토리얼을 스크립트가 아닌 노트북으로 실행하기를 권장합니다. 노트북 (.ipynb) 파일을 다운 받으시려면, +# 이 튜토리얼을 스크립트가 아닌 노트북으로 실행하기를 권장합니다. 노트북 (``.ipynb``) 파일을 다운 받으시려면, # 페이지 상단에 있는 링크를 클릭해주세요. # # PyTorch 는 여러분이 신경망(neural network)를 생성하고 학습시키는 것을 도와주기 위해서 @@ -30,7 +28,7 @@ # 이것이 어떻게 코드를 더 간결하고 유연하게 만드는지 보여줄 것입니다. # # **이 튜토리얼은 여러분이 이미 PyTorch를 설치하였고, 그리고 텐서 연산의 기초에 대해 익숙하다고 가정합니다.** -# (만약 여러분이 Numpy 배열(array) 연산에 익숙하다면, 여기에서 사용되는 PyTorch 텐서 연산도 +# (만약 여러분이 NumPy 배열(array) 연산에 익숙하다면, 여기에서 사용되는 PyTorch 텐서 연산도 # 거의 동일하다는 것을 알게 될 것입니다). # # MNIST 데이터 준비 @@ -61,7 +59,7 @@ (PATH / FILENAME).open("wb").write(content) ############################################################################### -# 이 데이터셋은 numpy 배열 포맷이고, 데이터를 직렬화하기 위한 +# 이 데이터셋은 NumPy 배열 포맷이고, 데이터를 직렬화하기 위한 # python 전용 포맷 pickle 을 이용하여 저장되어 있습니다. import pickle @@ -81,7 +79,7 @@ print(x_train.shape) ############################################################################### -# PyTorch는 numpy 배열 보다는 ``torch.tensor`` 를 사용하므로, 우리는 데이터를 변환해야 합니다. +# PyTorch는 NumPy 배열 보다는 ``torch.tensor`` 를 사용하므로, 우리는 데이터를 변환해야 합니다. import torch @@ -94,8 +92,8 @@ print(y_train.min(), y_train.max()) ############################################################################### -# torch.nn 없이 밑바닥부터 신경망 만들기 -# --------------------------------------------- +# ()``torch.nn`` 없이) 밑바닥부터 신경망 만들기 +# ----------------------------------------------- # # PyTorch 텐서 연산만으로 첫 모델을 만들어봅시다. # 여러분이 신경망의 기초에 대해서 이미 익숙하다고 가정합니다. @@ -113,7 +111,7 @@ # (PyTorch에서 ``_`` 다음에 오는 메소드 이름은 연산이 인플레이스(in-place)로 수행되는 것을 의미합니다.) # # .. note:: `Xavier initialisation `_ -# 기법을 이용하여 가중치를 초기화 합니다. (1/sqrt(n)을 곱해주는 것을 통해서 초기화). +# 기법을 이용하여 가중치를 초기화 합니다. (``1/sqrt(n)`` 을 곱해서 초기화). import math @@ -242,8 +240,8 @@ def accuracy(out, yb): print(loss_func(model(xb), yb), accuracy(model(xb), yb)) ############################################################################### -# torch.nn.functional 사용하기 -# ------------------------------ +# ``torch.nn.functional`` 사용하기 +# ----------------------------------- # # 이제 우리는 코드를 리팩토링(refactoring) 하겠습니다, 그럼으로써 이전과 동일하지만, # PyTorch의 ``nn`` 클래스의 장점을 활용하여 더 간결하고 유연하게 만들 것입니다. @@ -260,7 +258,7 @@ def accuracy(out, yb): # 앞으로 보시겠지만 일반적으로 라이브러리의 다른 부분을 사용하여 더 잘 처리 할 수 있습니다.) # # 만약 여러분들이 음의 로그 우도 손실과 로그 소프트맥스 (log softmax) 활성화 함수를 사용하는 경우, -# Pytorch는 이 둘을 결합하는 단일 함수인 ``F.cross_entropy`` 를 제공합니다. +# PyTorch는 이 둘을 결합하는 단일 함수인 ``F.cross_entropy`` 를 제공합니다. # 따라서 모델에서 활성화 함수를 제거할 수도 있습니다. import torch.nn.functional as F @@ -277,8 +275,8 @@ def model(xb): print(loss_func(model(xb), yb), accuracy(model(xb), yb)) ############################################################################### -# nn.Module 을 이용하여 리팩토링 하기 -# -------------------------------------- +# ``nn.Module`` 을 이용하여 리팩토링 하기 +# ----------------------------------------- # 다음으로, 더 명확하고 간결한 훈련 루프를 위해 ``nn.Module`` 및 ``nn.Parameter`` 를 사용합니다. # 우리는 ``nn.Module`` (자체가 클래스이고 상태를 추척할 수 있는) 하위 클래스(subclass)를 만듭니다. # 이 경우에는, 포워드(forward) 단계에 대한 가중치, 절편, 그리고 메소드(method) 등을 유지하는 @@ -311,7 +309,7 @@ def forward(self, xb): ############################################################################### # 이제 우리는 이전과 동일한 방식으로 손실을 계산할 수 있습니다. # 여기서 ``nn.Module`` 오브젝트들은 마치 함수처럼 사용됩니다 (즉, 이들은 *호출가능* 합니다), -# 그러나 배후에서 Pytorch 는 우리의 ``forward`` 메소드를 자동으로 호출합니다. +# 그러나 배후에서 PyTorch 는 우리의 ``forward`` 메소드를 자동으로 호출합니다. print(loss_func(model(xb), yb)) @@ -321,11 +319,11 @@ def forward(self, xb): # # :: # -# with torch.no_grad(): -# weights -= weights.grad * lr -# bias -= bias.grad * lr -# weights.grad.zero_() -# bias.grad.zero_() +# with torch.no_grad(): +# weights -= weights.grad * lr +# bias -= bias.grad * lr +# weights.grad.zero_() +# bias.grad.zero_() # # # 이제 우리는 model.parameters() 및 model.zero_grad() (모두 @@ -334,9 +332,9 @@ def forward(self, xb): # # :: # -# with torch.no_grad(): -# for p in model.parameters(): p -= p.grad * lr -# model.zero_grad() +# with torch.no_grad(): +# for p in model.parameters(): p -= p.grad * lr +# model.zero_grad() # # # 이제 이것을 나중에 다시 실행할 수 있도록 ``fit`` 함수로 작은 훈련 루프를 감쌀 것입니다. @@ -365,15 +363,15 @@ def fit(): print(loss_func(model(xb), yb)) ############################################################################### -# nn.Linear 를 이용하여 리팩토링 하기 -# ------------------------------------ +# ``nn.Linear`` 를 사용하여 리팩토링 하기 +# ----------------------------------------- # # 계속해서 코드를 리팩토링 합니다. ``self.weights`` 및 ``self.bias`` 를 수동으로 정의 및 # 초기화하고, ``xb @ self.weights + self.bias`` 를 계산하는 대신에, -# 위의 모든 것을 해줄 Pytorch 클래스인 +# 위의 모든 것을 해줄 PyTorch 클래스인 # `nn.Linear `_ 를 선형 # 레이어로 사용합니다. -# Pytorch 에는 다양한 유형의 코드를 크게 단순화 할 수 있는 미리 정의된 레이어가 있고 이는 또한 +# PyTorch 에는 다양한 유형의 코드를 크게 단순화 할 수 있는 미리 정의된 레이어가 있고 이는 또한 # 종종 기존 코드보다 속도를 빠르게 합니다. class Mnist_Logistic(nn.Module): @@ -398,10 +396,10 @@ def forward(self, xb): print(loss_func(model(xb), yb)) ############################################################################### -# optim 을 이용하여 리팩토링 하기 +# ``torch.optim`` 을 이용하여 리팩토링 하기 # --------------------------------- # -# Pytorch에는 다양한 최적화(optimization) 알고리즘을 가진 패키지인 ``torch.optim`` 도 있습니다. +# PyTorch에는 다양한 최적화(optimization) 알고리즘을 가진 패키지인 ``torch.optim`` 도 있습니다. # 각 매개변수를 수동으로 업데이트 하는 대신, 옵티마이저(optimizer)의 ``step`` 메소드를 사용하여 # 업데이트를 진행할 수 있습니다. # @@ -409,16 +407,16 @@ def forward(self, xb): # # :: # -# with torch.no_grad(): -# for p in model.parameters(): p -= p.grad * lr -# model.zero_grad() +# with torch.no_grad(): +# for p in model.parameters(): p -= p.grad * lr +# model.zero_grad() # # 대신에 이렇게 말이죠: # # :: # -# opt.step() -# opt.zero_grad() +# opt.step() +# opt.zero_grad() # # (``optim.zero_grad()`` 는 기울기를 0으로 재설정 해줍니다. 다음 미니 배치에 대한 # 기울기를 계산하기 전에 호출해야 합니다.) @@ -476,7 +474,7 @@ def get_model(): train_ds = TensorDataset(x_train, y_train) ############################################################################### -# 이전에는 x 및 y 값의 미니 배치를 별도로 반복해야했습니다: +# 이전에는 ``x`` 및 ``y`` 값의 미니 배치를 별도로 반복해야 했습니다: # # :: # @@ -484,7 +482,7 @@ def get_model(): # yb = y_train[start_i:end_i] # # -# 이제 이 두 단계를 함께 수행 할 수 있습니다: +# 이제 이 두 단계를 함께 수행할 수 있습니다: # # :: # @@ -506,14 +504,14 @@ def get_model(): print(loss_func(model(xb), yb)) ############################################################################### -# DataLoader 를 이용하여 리팩토링하기 +# ``DataLoader`` 를 사용하여 리팩토링하기 # ----------------------------------- # -# Pytorch 의 ``DataLoader`` 는 배치 관리를 담당합니다. +# PyTorch 의 ``DataLoader`` 는 배치 관리를 담당합니다. # 여러분들은 모든 ``Dataset`` 으로부터 ``DataLoader`` 를 생성할 수 있습니다. # ``DataLoader`` 는 배치들에 대해서 반복하기 쉽게 만들어줍니다. # ``train_ds[i*bs : i*bs+bs]`` 를 사용하는 대신, -# DataLoader 는 매 미니배치를 자동적으로 제공합니다. +# ``DataLoader`` 는 매 미니배치를 자동적으로 제공합니다. from torch.utils.data import DataLoader @@ -521,20 +519,20 @@ def get_model(): train_dl = DataLoader(train_ds, batch_size=bs) ############################################################################### -# 이전에는 루프가 다음과 같이 배치 (xb, yb)를 반복했습니다: +# 이전에는 루프가 다음과 같이 배치 ``(xb, yb)`` 를 반복했습니다: # # :: # -# for i in range((n-1)//bs + 1): -# xb,yb = train_ds[i*bs : i*bs+bs] -# pred = model(xb) +# for i in range((n-1)//bs + 1): +# xb,yb = train_ds[i*bs : i*bs+bs] +# pred = model(xb) # # 이제 (xb, yb)가 DataLoader 에서 자동으로 로드되므로 루프가 훨씬 깨끗해졌습니다: # # :: # -# for xb,yb in train_dl: -# pred = model(xb) +# for xb,yb in train_dl: +# pred = model(xb) model, opt = get_model() @@ -550,7 +548,7 @@ def get_model(): print(loss_func(model(xb), yb)) ############################################################################### -# Pytorch의 nn.Module, nn.Parameter, Dataset 및 DataLoader 덕분에 이제 훈련 루프가 +# PyTorch의 nn.Module, nn.Parameter, Dataset 및 DataLoader 덕분에 이제 훈련 루프가 # 훨씬 더 작아지고 이해하기 쉬워졌습니다. # 이제 실제로 효과적인 모델을 만드는 데 필요한 기본 기능을 추가해 보겠습니다. # @@ -673,11 +671,11 @@ def get_data(train_ds, valid_ds, bs): # 이전 섹션의 어떤 함수도 모델의 형식에 대해 가정하지 않기 때문에, # 별도의 수정없이 CNN을 학습하는 데 사용할 수 있습니다. # -# Pytorch 의 사전정의된 +# Pytorch의 사전정의된 # `Conv2d `_ 클래스를 # 컨볼루션 레이어로 사용합니다. 3개의 컨볼루션 레이어로 CNN을 정의합니다. # 각 컨볼루션 뒤에는 ReLU가 있습니다. 마지막으로 평균 풀링(average pooling)을 수행합니다. -# (``view`` 는 PyTorch의 numpy ``reshape`` 버전입니다.) +# (``view`` 는 PyTorch의 NumPy ``reshape`` 버전입니다.) class Mnist_CNN(nn.Module): def __init__(self): @@ -707,8 +705,8 @@ def forward(self, xb): fit(epochs, model, loss_func, opt, train_dl, valid_dl) ############################################################################### -# nn.Sequential -# ------------------------ +# ``nn.Sequential`` 사용하기 +# ---------------------------- # # ``torch.nn`` 에는 코드를 간단히 사용할 수 있는 또 다른 편리한 클래스인 # `Sequential `_ @@ -753,7 +751,7 @@ def preprocess(x): fit(epochs, model, loss_func, opt, train_dl, valid_dl) ############################################################################### -# DataLoader 감싸기 +# ``DataLoader`` 감싸기 # ----------------------------- # # 우리의 CNN은 상당히 간결하지만, MNIST에서만 작동합니다, 왜냐하면: @@ -813,7 +811,7 @@ def __iter__(self): # # 만약 여러분들이 운이 좋아서 CUDA 지원 GPU (대부분의 클라우드 제공 업체에서 # 시간당 약 $0.50 에 이용할 수 있습니다) 를 사용할 수 있다면, 코드 실행 속도를 높일 수 있습니다. -# 먼저 GPU가 Pytorch에서 작동하는지 확인합니다: +# 먼저 GPU가 PyTorch에서 작동하는지 확인합니다: print(torch.cuda.is_available()) @@ -864,7 +862,7 @@ def preprocess(x, y): # 그리고 ``DataLoader`` 의 각 예제를 통해 설명하겠다고 이야기했었습니다. # 이제 위의 내용들을 요약해보겠습니다: # -# - **torch.nn** +# - ``torch.nn``: # # + ``Module``: 함수처럼 동작하지만, 또한 상태(state) (예를 들어, 신경망의 레이어 가중치)를 # 포함할 수 있는 호출 가능한 오브젝트를 생성합니다. @@ -877,6 +875,6 @@ def preprocess(x, y): # 상태를 저장하지않는(non-stateful) 버전의 레이어를 포함합니다. # - ``torch.optim``: 역전파 단계에서 ``Parameter`` 의 가중치를 업데이트하는, # ``SGD`` 와 같은 옵티마이저를 포함합니다. -# - ``Dataset``: ``TensorDataset`` 과 같이 Pytorch와 함께 제공되는 클래스를 포함하여 ``__len__`` 및 +# - ``Dataset``: ``TensorDataset`` 과 같이 PyTorch와 함께 제공되는 클래스를 포함하여 ``__len__`` 및 # ``__getitem__`` 이 있는 객체의 추상 인터페이스 # - ``DataLoader``: 모든 종류의 ``Dataset`` 을 기반으로 데이터의 배치들을 출력하는 반복자(iterator)를 생성합니다. diff --git a/beginner_source/saving_loading_models.py b/beginner_source/saving_loading_models.py index d89f84efd..9633ad899 100644 --- a/beginner_source/saving_loading_models.py +++ b/beginner_source/saving_loading_models.py @@ -148,8 +148,8 @@ # PyTorch 버전 1.6에서는 ``torch.save`` 가 새로운 Zip파일-기반의 파일 # 포맷을 사용하도록 변경되었습니다. ``torch.load`` 는 예전 방식의 파일들을 # 읽어올 수 있도록 하고 있습니다. 어떤 이유에서든 ``torch.save`` 가 예전 -# 방식을 사용하도록 하고 싶다면, ``_use_new_zipfile_serialization=False`` 을 -# kwarg로 전달하세요. +# 방식을 사용하도록 하고 싶다면, ``kwarg`` 매개변수로 +# ``_use_new_zipfile_serialization=False`` 을 전달하세요. # # 추론을 위해 모델을 저장할 때는 그 모델의 학습된 매개변수만 저장하면 됩니다. # ``torch.save()`` 를 사용하여 모델의 *state_dict* 를 저장하는 것이 나중에 모델을 @@ -471,7 +471,7 @@ # # 모델에서 사용하는 input Tensor들은 input = input.to(device) 을 호출해야 합니다. # # CPU에서 학습한 모델을 GPU에서 불러올 때는 ``torch.load()`` 함수의 -# ``map_location`` 인자에 *cuda:device_id* 을 설정합니다. 이렇게 하면 모델이 해당 +# ``map_location`` 인자에 ``cuda:device_id`` 을 설정합니다. 이렇게 하면 모델이 해당 # GPU 장치에 불러와집니다. 다음으로 ``model.to(torch.device('cuda'))`` 을 호출하여 # 모델의 매개변수 Tensor들을 CUDA Tensor들로 변환해야 합니다. 마지막으로 모든 # 모델 입력에 ``.to(torch.device('cuda'))`` 을 사용하여 CUDA 최적화된 모델을 위한 diff --git a/beginner_source/t5_tutorial.py b/beginner_source/t5_tutorial.py new file mode 100644 index 000000000..8f77cd278 --- /dev/null +++ b/beginner_source/t5_tutorial.py @@ -0,0 +1,456 @@ +""" +T5-Base Model for Summarization, Sentiment Classification, and Translation +========================================================================== + +**Authors**: `Pendo Abbo `__, `Joe Cummings `__ + +""" + +###################################################################### +# Overview +# -------- +# +# This tutorial demonstrates how to use a pretrained T5 Model for summarization, sentiment classification, and +# translation tasks. We will demonstrate how to use the torchtext library to: +# +# 1. Build a text preprocessing pipeline for a T5 model +# 2. Instantiate a pretrained T5 model with base configuration +# 3. Read in the CNNDM, IMDB, and Multi30k datasets and preprocess their texts in preparation for the model +# 4. Perform text summarization, sentiment classification, and translation +# +# .. note:: +# This tutorial requires PyTorch 2.0.0 or later. +# +####################################################################### +# Data Transformation +# ------------------- +# +# The T5 model does not work with raw text. Instead, it requires the text to be transformed into numerical form +# in order to perform training and inference. The following transformations are required for the T5 model: +# +# 1. Tokenize text +# 2. Convert tokens into (integer) IDs +# 3. Truncate the sequences to a specified maximum length +# 4. Add end-of-sequence (EOS) and padding token IDs +# +# T5 uses a ``SentencePiece`` model for text tokenization. Below, we use a pretrained ``SentencePiece`` model to build +# the text preprocessing pipeline using torchtext's T5Transform. Note that the transform supports both +# batched and non-batched text input (for example, one can either pass a single sentence or a list of sentences), however the T5 model expects the input to be batched. +# + +from torchtext.models import T5Transform + +padding_idx = 0 +eos_idx = 1 +max_seq_len = 512 +t5_sp_model_path = "https://download.pytorch.org/models/text/t5_tokenizer_base.model" + +transform = T5Transform( + sp_model_path=t5_sp_model_path, + max_seq_len=max_seq_len, + eos_idx=eos_idx, + padding_idx=padding_idx, +) + +####################################################################### +# Alternatively, we can also use the transform shipped with the pretrained models that does all of the above out-of-the-box +# +# .. code-block:: +# +# from torchtext.models import T5_BASE_GENERATION +# transform = T5_BASE_GENERATION.transform() +# + + +###################################################################### +# Model Preparation +# ----------------- +# +# torchtext provides SOTA pretrained models that can be used directly for NLP tasks or fine-tuned on downstream tasks. Below +# we use the pretrained T5 model with standard base configuration to perform text summarization, sentiment classification, and +# translation. For additional details on available pretrained models, see `the torchtext documentation `__ +# +# +from torchtext.models import T5_BASE_GENERATION + + +t5_base = T5_BASE_GENERATION +transform = t5_base.transform() +model = t5_base.get_model() +model.eval() + + +####################################################################### +# Using ``GenerationUtils`` +# ------------------------- +# +# We can use torchtext's ``GenerationUtils`` to produce an output sequence based on the input sequence provided. This calls on the +# model's encoder and decoder, and iteratively expands the decoded sequences until the end-of-sequence token is generated +# for all sequences in the batch. The ``generate`` method shown below uses greedy search to generate the sequences. Beam search and +# other decoding strategies are also supported. +# +# +from torchtext.prototype.generate import GenerationUtils + +sequence_generator = GenerationUtils(model) + + +####################################################################### +# Datasets +# -------- +# torchtext provides several standard NLP datasets. For a complete list, refer to the documentation +# at https://pytorch.org/text/stable/datasets.html. These datasets are built using composable torchdata +# datapipes and hence support standard flow-control and mapping/transformation using user defined +# functions and transforms. +# +# Below we demonstrate how to preprocess the CNNDM dataset to include the prefix necessary for the +# model to identify the task it is performing. The CNNDM dataset has a train, validation, and test +# split. Below we demo on the test split. +# +# The T5 model uses the prefix "summarize" for text summarization. For more information on task +# prefixes, please visit Appendix D of the `T5 Paper `__ +# +# .. note:: +# Using datapipes is still currently subject to a few caveats. If you wish +# to extend this example to include shuffling, multi-processing, or +# distributed learning, please see :ref:`this note ` +# for further instructions. + +from functools import partial + +from torch.utils.data import DataLoader +from torchtext.datasets import CNNDM + +cnndm_batch_size = 5 +cnndm_datapipe = CNNDM(split="test") +task = "summarize" + + +def apply_prefix(task, x): + return f"{task}: " + x[0], x[1] + + +cnndm_datapipe = cnndm_datapipe.map(partial(apply_prefix, task)) +cnndm_datapipe = cnndm_datapipe.batch(cnndm_batch_size) +cnndm_datapipe = cnndm_datapipe.rows2columnar(["article", "abstract"]) +cnndm_dataloader = DataLoader(cnndm_datapipe, shuffle=True, batch_size=None) + +####################################################################### +# Alternately, we can also use batched API, for example, apply the prefix on the whole batch: +# +# .. code-block:: +# +# def batch_prefix(task, x): +# return { +# "article": [f'{task}: ' + y for y in x["article"]], +# "abstract": x["abstract"] +# } +# +# cnndm_batch_size = 5 +# cnndm_datapipe = CNNDM(split="test") +# task = 'summarize' +# +# cnndm_datapipe = cnndm_datapipe.batch(cnndm_batch_size).rows2columnar(["article", "abstract"]) +# cnndm_datapipe = cnndm_datapipe.map(partial(batch_prefix, task)) +# cnndm_dataloader = DataLoader(cnndm_datapipe, batch_size=None) +# + +####################################################################### +# We can also load the IMDB dataset, which will be used to demonstrate sentiment classification using the T5 model. +# This dataset has a train and test split. Below we demo on the test split. +# +# The T5 model was trained on the SST2 dataset (also available in torchtext) for sentiment classification using the +# prefix ``sst2 sentence``. Therefore, we will use this prefix to perform sentiment classification on the IMDB dataset. +# + +from torchtext.datasets import IMDB + +imdb_batch_size = 3 +imdb_datapipe = IMDB(split="test") +task = "sst2 sentence" +labels = {"1": "negative", "2": "positive"} + + +def process_labels(labels, x): + return x[1], labels[str(x[0])] + + +imdb_datapipe = imdb_datapipe.map(partial(process_labels, labels)) +imdb_datapipe = imdb_datapipe.map(partial(apply_prefix, task)) +imdb_datapipe = imdb_datapipe.batch(imdb_batch_size) +imdb_datapipe = imdb_datapipe.rows2columnar(["text", "label"]) +imdb_dataloader = DataLoader(imdb_datapipe, batch_size=None) + +####################################################################### +# Finally, we can also load the Multi30k dataset to demonstrate English to German translation using the T5 model. +# This dataset has a train, validation, and test split. Below we demo on the test split. +# +# The T5 model uses the prefix "translate English to German" for this task. + +from torchtext.datasets import Multi30k + +multi_batch_size = 5 +language_pair = ("en", "de") +multi_datapipe = Multi30k(split="test", language_pair=language_pair) +task = "translate English to German" + +multi_datapipe = multi_datapipe.map(partial(apply_prefix, task)) +multi_datapipe = multi_datapipe.batch(multi_batch_size) +multi_datapipe = multi_datapipe.rows2columnar(["english", "german"]) +multi_dataloader = DataLoader(multi_datapipe, batch_size=None) + +####################################################################### +# Generate Summaries +# ------------------ +# +# We can put all of the components together to generate summaries on the first batch of articles in the CNNDM test set +# using a beam size of 1. +# + +batch = next(iter(cnndm_dataloader)) +input_text = batch["article"] +target = batch["abstract"] +beam_size = 1 + +model_input = transform(input_text) +model_output = sequence_generator.generate(model_input, eos_idx=eos_idx, num_beams=beam_size) +output_text = transform.decode(model_output.tolist()) + +for i in range(cnndm_batch_size): + print(f"Example {i+1}:\n") + print(f"prediction: {output_text[i]}\n") + print(f"target: {target[i]}\n\n") + + +####################################################################### +# Summarization Output (Might vary since we shuffle the dataloader) +# -------------------- +# +# .. code-block:: +# +# Example 1: +# +# prediction: the 24-year-old has been tattooed for over a decade . he has landed in australia +# to start work on a new campaign . he says he is 'taking it in your stride' to be honest . +# +# target: London-based model Stephen James Hendry famed for his full body tattoo . The supermodel +# is in Sydney for a new modelling campaign . Australian fans understood to have already located +# him at his hotel . The 24-year-old heartthrob is recently single . +# +# +# Example 2: +# +# prediction: a stray pooch has used up at least three of her own after being hit by a +# car and buried in a field . the dog managed to stagger to a nearby farm, dirt-covered +# and emaciated, where she was found . she suffered a dislocated jaw, leg injuries and a +# caved-in sinus cavity -- and still requires surgery to help her breathe . +# +# target: Theia, a bully breed mix, was apparently hit by a car, whacked with a hammer +# and buried in a field . "She's a true miracle dog and she deserves a good life," says +# Sara Mellado, who is looking for a home for Theia . +# +# +# Example 3: +# +# prediction: mohammad Javad Zarif arrived in Iran on a sunny friday morning . he has gone +# a long way to bring Iran in from the cold and allow it to rejoin the international +# community . but there are some facts about him that are less well-known . +# +# target: Mohammad Javad Zarif has spent more time with John Kerry than any other +# foreign minister . He once participated in a takeover of the Iranian Consulate in San +# Francisco . The Iranian foreign minister tweets in English . +# +# +# Example 4: +# +# prediction: five americans were monitored for three weeks after being exposed to Ebola in +# west africa . one of the five had a heart-related issue and has been discharged but hasn't +# left the area . they are clinicians for Partners in Health, a Boston-based aid group . +# +# target: 17 Americans were exposed to the Ebola virus while in Sierra Leone in March . +# Another person was diagnosed with the disease and taken to hospital in Maryland . +# National Institutes of Health says the patient is in fair condition after weeks of +# treatment . +# +# +# Example 5: +# +# prediction: the student was identified during an investigation by campus police and +# the office of student affairs . he admitted to placing the noose on the tree early +# Wednesday morning . the incident is one of several recent racist events to affect +# college students . +# +# target: Student is no longer on Duke University campus and will face disciplinary +# review . School officials identified student during investigation and the person +# admitted to hanging the noose, Duke says . The noose, made of rope, was discovered on +# campus about 2 a.m. +# + + +####################################################################### +# Generate Sentiment Classifications +# ---------------------------------- +# +# Similarly, we can use the model to generate sentiment classifications on the first batch of reviews from the IMDB test set +# using a beam size of 1. +# + +batch = next(iter(imdb_dataloader)) +input_text = batch["text"] +target = batch["label"] +beam_size = 1 + +model_input = transform(input_text) +model_output = sequence_generator.generate(model_input, eos_idx=eos_idx, num_beams=beam_size) +output_text = transform.decode(model_output.tolist()) + +for i in range(imdb_batch_size): + print(f"Example {i+1}:\n") + print(f"input_text: {input_text[i]}\n") + print(f"prediction: {output_text[i]}\n") + print(f"target: {target[i]}\n\n") + + +####################################################################### +# Sentiment Output +# ---------------- +# +# :: +# +# Example 1: +# +# input_text: sst2 sentence: I love sci-fi and am willing to put up with a lot. Sci-fi +# movies/TV are usually underfunded, under-appreciated and misunderstood. I tried to like +# this, I really did, but it is to good TV sci-fi as Babylon 5 is to Star Trek (the original). +# Silly prosthetics, cheap cardboard sets, stilted dialogues, CG that doesn't match the +# background, and painfully one-dimensional characters cannot be overcome with a 'sci-fi' +# setting. (I'm sure there are those of you out there who think Babylon 5 is good sci-fi TV. +# It's not. It's clichéd and uninspiring.) While US viewers might like emotion and character +# development, sci-fi is a genre that does not take itself seriously (cf. Star Trek). It may +# treat important issues, yet not as a serious philosophy. It's really difficult to care about +# the characters here as they are not simply foolish, just missing a spark of life. Their +# actions and reactions are wooden and predictable, often painful to watch. The makers of Earth +# KNOW it's rubbish as they have to always say "Gene Roddenberry's Earth..." otherwise people +# would not continue watching. Roddenberry's ashes must be turning in their orbit as this dull, +# cheap, poorly edited (watching it without advert breaks really brings this home) trudging +# Trabant of a show lumbers into space. Spoiler. So, kill off a main character. And then bring +# him back as another actor. Jeeez. Dallas all over again. +# +# prediction: negative +# +# target: negative +# +# +# Example 2: +# +# input_text: sst2 sentence: Worth the entertainment value of a rental, especially if you like +# action movies. This one features the usual car chases, fights with the great Van Damme kick +# style, shooting battles with the 40 shell load shotgun, and even terrorist style bombs. All +# of this is entertaining and competently handled but there is nothing that really blows you +# away if you've seen your share before.

The plot is made interesting by the +# inclusion of a rabbit, which is clever but hardly profound. Many of the characters are +# heavily stereotyped -- the angry veterans, the terrified illegal aliens, the crooked cops, +# the indifferent feds, the bitchy tough lady station head, the crooked politician, the fat +# federale who looks like he was typecast as the Mexican in a Hollywood movie from the 1940s. +# All passably acted but again nothing special.

I thought the main villains were +# pretty well done and fairly well acted. By the end of the movie you certainly knew who the +# good guys were and weren't. There was an emotional lift as the really bad ones got their just +# deserts. Very simplistic, but then you weren't expecting Hamlet, right? The only thing I found +# really annoying was the constant cuts to VDs daughter during the last fight scene.

+# Not bad. Not good. Passable 4. +# +# prediction: positive +# +# target: negative +# +# +# Example 3: +# +# input_text: sst2 sentence: its a totally average film with a few semi-alright action sequences +# that make the plot seem a little better and remind the viewer of the classic van dam films. +# parts of the plot don't make sense and seem to be added in to use up time. the end plot is that +# of a very basic type that doesn't leave the viewer guessing and any twists are obvious from the +# beginning. the end scene with the flask backs don't make sense as they are added in and seem to +# have little relevance to the history of van dam's character. not really worth watching again, +# bit disappointed in the end production, even though it is apparent it was shot on a low budget +# certain shots and sections in the film are of poor directed quality. +# +# prediction: negative +# +# target: negative +# + + +####################################################################### +# Generate Translations +# --------------------- +# +# Finally, we can also use the model to generate English to German translations on the first batch of examples from the Multi30k +# test set. +# + +batch = next(iter(multi_dataloader)) +input_text = batch["english"] +target = batch["german"] + +model_input = transform(input_text) +model_output = sequence_generator.generate(model_input, eos_idx=eos_idx, num_beams=beam_size) +output_text = transform.decode(model_output.tolist()) + +for i in range(multi_batch_size): + print(f"Example {i+1}:\n") + print(f"input_text: {input_text[i]}\n") + print(f"prediction: {output_text[i]}\n") + print(f"target: {target[i]}\n\n") + + +####################################################################### +# Translation Output +# ------------------ +# +# :: +# +# Example 1: +# +# input_text: translate English to German: A man in an orange hat starring at something. +# +# prediction: Ein Mann in einem orangen Hut, der an etwas schaut. +# +# target: Ein Mann mit einem orangefarbenen Hut, der etwas anstarrt. +# +# +# Example 2: +# +# input_text: translate English to German: A Boston Terrier is running on lush green grass in front of a white fence. +# +# prediction: Ein Boston Terrier läuft auf üppigem grünem Gras vor einem weißen Zaun. +# +# target: Ein Boston Terrier läuft über saftig-grünes Gras vor einem weißen Zaun. +# +# +# Example 3: +# +# input_text: translate English to German: A girl in karate uniform breaking a stick with a front kick. +# +# prediction: Ein Mädchen in Karate-Uniform bricht einen Stöck mit einem Frontkick. +# +# target: Ein Mädchen in einem Karateanzug bricht ein Brett mit einem Tritt. +# +# +# Example 4: +# +# input_text: translate English to German: Five people wearing winter jackets and helmets stand in the snow, with snowmobiles in the background. +# +# prediction: Fünf Menschen mit Winterjacken und Helmen stehen im Schnee, mit Schneemobilen im Hintergrund. +# +# target: Fünf Leute in Winterjacken und mit Helmen stehen im Schnee mit Schneemobilen im Hintergrund. +# +# +# Example 5: +# +# input_text: translate English to German: People are fixing the roof of a house. +# +# prediction: Die Leute fixieren das Dach eines Hauses. +# +# target: Leute Reparieren das Dach eines Hauses. +# diff --git a/beginner_source/template_tutorial.py b/beginner_source/template_tutorial.py new file mode 100644 index 000000000..75e8a551f --- /dev/null +++ b/beginner_source/template_tutorial.py @@ -0,0 +1,92 @@ +# -*- coding: utf-8 -*- + +""" +Template Tutorial +================= + +**Author:** `FirstName LastName `_ + +.. grid:: 2 + + .. grid-item-card:: :octicon:`mortar-board;1em;` What you will learn + + * Item 1 + * Item 2 + * Item 3 + + .. grid-item-card:: :octicon:`list-unordered;1em;` Prerequisites + + * PyTorch v2.0.0 + * GPU ??? + * Other items 3 + +If you have a video, add it here like this: + +.. raw:: html + +
+ +
+ +To test your tutorial locally, you can do one of the following: + +* You can control specific files that generate the results by using + ``GALLERY_PATTERN`` environment variable. The GALLERY_PATTERN variable + respects regular expressions. + For example to run only ``neural_style_transfer_tutorial.py``, + use the following command: + + .. code-block:: sh + + GALLERY_PATTERN="neural_style_transfer_tutorial.py" make html + + or + + .. code-block:: sh + + GALLERY_PATTERN="neural_style_transfer_tutorial.py" sphinx-build . _build + +* Make a copy of this repository and add only your + tutorial to the `beginner_source` directory removing all other tutorials. + Then run ``make html``. + +Verify that all outputs were generated correctly in the created HTML. +""" + +######################################################################### +# Overview +# -------- +# +# Describe Why is this topic important? Add Links to relevant research papers. +# +# This tutorial walks you through the process of.... +# +# Steps +# ----- +# +# Example code (the output below is generated automatically): +# +import torch +x = torch.rand(5, 3) +print(x) + +###################################################################### +# (Optional) Additional Exercises +# ------------------------------- +# +# Add additional practice exercises for users to test their knowledge. +# Example: `NLP from Scratch `__. +# + +###################################################################### +# Conclusion +# ---------- +# +# Summarize the steps and concepts covered. Highlight key takeaways. +# +# Further Reading +# --------------- +# +# * Link1 +# * Link2 + diff --git a/beginner_source/text_sentiment_ngrams_tutorial.py b/beginner_source/text_sentiment_ngrams_tutorial.py index 98bc04ebc..b6ec7b11e 100644 --- a/beginner_source/text_sentiment_ngrams_tutorial.py +++ b/beginner_source/text_sentiment_ngrams_tutorial.py @@ -166,7 +166,7 @@ class TextClassificationModel(nn.Module): def __init__(self, vocab_size, embed_dim, num_class): super(TextClassificationModel, self).__init__() - self.embedding = nn.EmbeddingBag(vocab_size, embed_dim, sparse=True) + self.embedding = nn.EmbeddingBag(vocab_size, embed_dim, sparse=False) self.fc = nn.Linear(embed_dim, num_class) self.init_weights() @@ -183,7 +183,7 @@ def forward(self, text, offsets): ###################################################################### # 인스턴스 생성하기 -# ----------------- +# ------------------- # # ``AG_NEWS`` 데이터셋에는 4종류의 레이블이 존재하므로 클래스의 개수도 4개입니다. # @@ -208,7 +208,7 @@ def forward(self, text, offsets): ###################################################################### # 모델을 학습하고 결과를 평가하는 함수 정의하기 -# --------------------------------------------- +# ----------------------------------------------- # @@ -251,7 +251,7 @@ def evaluate(dataloader): ###################################################################### # 데이터셋을 분할하고 모델 수행하기 -# --------------------------------- +# ----------------------------------- # # 원본 ``AG_NEWS`` 에는 검증용 데이터가 포함되어 있지 않기 때문에, 우리는 학습 # 데이터를 학습 및 검증 데이터로 분할하려 합니다. 이때 데이터를 분할하는 @@ -331,7 +331,7 @@ def evaluate(dataloader): ###################################################################### # 임의의 뉴스로 평가하기 -# ---------------------- +# ------------------------ # # 현재까지 최고의 모델로 골프 뉴스를 테스트해보겠습니다. # diff --git a/beginner_source/transfer_learning_tutorial.py b/beginner_source/transfer_learning_tutorial.py index 8dca57e8c..13691ae24 100644 --- a/beginner_source/transfer_learning_tutorial.py +++ b/beginner_source/transfer_learning_tutorial.py @@ -2,6 +2,7 @@ """ 컴퓨터 비전(Vision)을 위한 전이학습(Transfer Learning) ======================================================= + **Author**: `Sasank Chilamkurthy `_ **번역**: `박정환 `_ @@ -103,7 +104,7 @@ # 데이터 증가를 이해하기 위해 일부 학습용 이미지를 시각화해보겠습니다. def imshow(inp, title=None): - """Imshow for Tensor.""" + """tensor를 입력받아 일반적인 이미지로 보여줍니다.""" inp = inp.numpy().transpose((1, 2, 0)) mean = np.array([0.485, 0.456, 0.406]) std = np.array([0.229, 0.224, 0.225]) @@ -126,7 +127,7 @@ def imshow(inp, title=None): ###################################################################### # 모델 학습하기 -# -------------- +# --------------- # # 이제 모델을 학습하기 위한 일반 함수를 작성해보겠습니다. 여기서는 다음 내용들을 # 설명합니다: @@ -240,15 +241,15 @@ def visualize_model(model, num_images=6): ###################################################################### # 합성곱 신경망 미세조정(finetuning) -# ---------------------------------- +# ------------------------------------ # # 미리 학습한 모델을 불러온 후 마지막의 완전히 연결된 계층을 초기화합니다. # -model_ft = models.resnet18(pretrained=True) +model_ft = models.resnet18(weights='IMAGENET1K_V1') num_ftrs = model_ft.fc.in_features # 여기서 각 출력 샘플의 크기는 2로 설정합니다. -# 또는, nn.Linear(num_ftrs, len (class_names))로 일반화할 수 있습니다. +# 또는, ``nn.Linear(num_ftrs, len (class_names))`` 로 일반화할 수 있습니다. model_ft.fc = nn.Linear(num_ftrs, 2) model_ft = model_ft.to(device) @@ -290,7 +291,7 @@ def visualize_model(model, num_images=6): # 에서 확인할 수 있습니다. # -model_conv = torchvision.models.resnet18(pretrained=True) +model_conv = torchvision.models.resnet18(weights='IMAGENET1K_V1') for param in model_conv.parameters(): param.requires_grad = False diff --git a/beginner_source/transformer_tutorial.py b/beginner_source/transformer_tutorial.py index d39dfe7c1..5fe3bc27e 100644 --- a/beginner_source/transformer_tutorial.py +++ b/beginner_source/transformer_tutorial.py @@ -1,6 +1,6 @@ """ -nn.Transformer 와 TorchText 로 시퀀스-투-시퀀스(Sequence-to-Sequence) 모델링하기 -================================================================================= +``nn.Transformer`` 와 torchtext로 시퀀스-투-시퀀스(Sequence-to-Sequence) 모델링하기 +===================================================================================== 이 튜토리얼에서는 `nn.Transformer `__ 모듈을 @@ -42,6 +42,8 @@ # import math +import os +from tempfile import TemporaryDirectory from typing import Tuple import torch @@ -73,12 +75,12 @@ def init_weights(self) -> None: def forward(self, src: Tensor, src_mask: Tensor) -> Tensor: """ - Args: - src: Tensor, shape [seq_len, batch_size] - src_mask: Tensor, shape [seq_len, seq_len] + Arguments: + src: Tensor, shape ``[seq_len, batch_size]`` + src_mask: Tensor, shape ``[seq_len, seq_len]`` Returns: - output Tensor of shape [seq_len, batch_size, ntoken] + output Tensor of shape ``[seq_len, batch_size, ntoken]`` """ src = self.encoder(src) * math.sqrt(self.d_model) src = self.pos_encoder(src) @@ -88,7 +90,7 @@ def forward(self, src: Tensor, src_mask: Tensor) -> Tensor: def generate_square_subsequent_mask(sz: int) -> Tensor: - """Generates an upper-triangular matrix of -inf, with zeros on diag.""" + """Generates an upper-triangular matrix of ``-inf``, with zeros on ``diag``.""" return torch.triu(torch.ones(sz, sz) * float('-inf'), diagonal=1) @@ -113,8 +115,8 @@ def __init__(self, d_model: int, dropout: float = 0.1, max_len: int = 5000): def forward(self, x: Tensor) -> Tensor: """ - Args: - x: Tensor, shape [seq_len, batch_size, embedding_dim] + Arguments: + x: Tensor, shape ``[seq_len, batch_size, embedding_dim]`` """ x = x + self.pe[:x.size(0)] return self.dropout(x) @@ -172,7 +174,7 @@ def data_process(raw_text_iter: dataset.IterableDataset) -> Tensor: data = [torch.tensor(vocab(tokenizer(item)), dtype=torch.long) for item in raw_text_iter] return torch.cat(tuple(filter(lambda t: t.numel() > 0, data))) -# train_iter was "consumed" by the process of building the vocab, +# ``train_iter`` was "consumed" by the process of building the vocab, # so we have to create it again train_iter, val_iter, test_iter = WikiText2() train_data = data_process(train_iter) @@ -182,15 +184,15 @@ def data_process(raw_text_iter: dataset.IterableDataset) -> Tensor: device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') def batchify(data: Tensor, bsz: int) -> Tensor: - """Divides the data into bsz separate sequences, removing extra elements + """Divides the data into ``bsz`` separate sequences, removing extra elements that wouldn't cleanly fit. - Args: - data: Tensor, shape [N] + Arguments: + data: Tensor, shape ``[N]`` bsz: int, batch size Returns: - Tensor of shape [N // bsz, bsz] + Tensor of shape ``[N // bsz, bsz]`` """ seq_len = data.size(0) // bsz data = data[:seq_len * bsz] @@ -225,13 +227,13 @@ def batchify(data: Tensor, bsz: int) -> Tensor: bptt = 35 def get_batch(source: Tensor, i: int) -> Tuple[Tensor, Tensor]: """ - Args: - source: Tensor, shape [full_seq_len, batch_size] + Arguments: + source: Tensor, shape ``[full_seq_len, batch_size]`` i: int Returns: - tuple (data, target), where data has shape [seq_len, batch_size] and - target has shape [seq_len * batch_size] + tuple ``(data, target)``, where data has shape ``[seq_len, batch_size]`` and + target has shape ``[seq_len * batch_size]`` """ seq_len = min(bptt, len(source) - 1 - i) data = source[i:i+seq_len] @@ -247,21 +249,21 @@ def get_batch(source: Tensor, i: int) -> Tuple[Tensor, Tensor]: ###################################################################### # 모델의 하이퍼파라미터(hyperparameter)는 아래와 같이 정의됩니다. -# 단어 사이즈는 단어 오브젝트의 길이와 일치 합니다. +# 어휘집( ``vocab`` )의 크기는 단어 오브젝트의 길이와 일치 합니다. # ntokens = len(vocab) # 단어 사전(어휘집)의 크기 emsize = 200 # 임베딩 차원 -d_hid = 200 # nn.TransformerEncoder 에서 피드포워드 네트워크(feedforward network) 모델의 차원 -nlayers = 2 # nn.TransformerEncoder 내부의 nn.TransformerEncoderLayer 개수 -nhead = 2 # nn.MultiheadAttention의 헤드 개수 +d_hid = 200 # ``nn.TransformerEncoder`` 에서 피드포워드 네트워크(feedforward network) 모델의 차원 +nlayers = 2 # ``nn.TransformerEncoder`` 내부의 nn.TransformerEncoderLayer 개수 +nhead = 2 # ``nn.MultiheadAttention`` 의 헤드 개수 dropout = 0.2 # 드랍아웃(dropout) 확률 model = TransformerModel(ntokens, emsize, nhead, d_hid, nlayers, dropout).to(device) ###################################################################### # 모델 실행하기 -# ------------- +# --------------- # @@ -335,24 +337,27 @@ def evaluate(model: nn.Module, eval_data: Tensor) -> float: best_val_loss = float('inf') epochs = 3 -best_model = None -for epoch in range(1, epochs + 1): - epoch_start_time = time.time() - train(model) - val_loss = evaluate(model, val_data) - val_ppl = math.exp(val_loss) - elapsed = time.time() - epoch_start_time - print('-' * 89) - print(f'| end of epoch {epoch:3d} | time: {elapsed:5.2f}s | ' - f'valid loss {val_loss:5.2f} | valid ppl {val_ppl:8.2f}') - print('-' * 89) +with TemporaryDirectory() as tempdir: + best_model_params_path = os.path.join(tempdir, "best_model_params.pt") - if val_loss < best_val_loss: - best_val_loss = val_loss - best_model = copy.deepcopy(model) + for epoch in range(1, epochs + 1): + epoch_start_time = time.time() + train(model) + val_loss = evaluate(model, val_data) + val_ppl = math.exp(val_loss) + elapsed = time.time() - epoch_start_time + print('-' * 89) + print(f'| end of epoch {epoch:3d} | time: {elapsed:5.2f}s | ' + f'valid loss {val_loss:5.2f} | valid ppl {val_ppl:8.2f}') + print('-' * 89) - scheduler.step() + if val_loss < best_val_loss: + best_val_loss = val_loss + torch.save(model.state_dict(), best_model_params_path) + + scheduler.step() + model.load_state_dict(torch.load(best_model_params_path)) # load best model states ###################################################################### @@ -360,9 +365,9 @@ def evaluate(model: nn.Module, eval_data: Tensor) -> float: # ------------------------------------------------- # -test_loss = evaluate(best_model, test_data) +test_loss = evaluate(model, test_data) test_ppl = math.exp(test_loss) print('=' * 89) print(f'| End of training | test loss {test_loss:5.2f} | ' f'test ppl {test_ppl:8.2f}') -print('=' * 89) +print('=' * 89) \ No newline at end of file diff --git a/beginner_source/translation_transformer.py b/beginner_source/translation_transformer.py index 68a75c70d..768228523 100644 --- a/beginner_source/translation_transformer.py +++ b/beginner_source/translation_transformer.py @@ -1,5 +1,5 @@ """ -nn.Transformer와 torchtext로 언어 번역하기 +``nn.Transformer`` 와 torchtext로 언어 번역하기 ====================================================== 이 튜토리얼에서는, @@ -41,13 +41,17 @@ token_transform = {} vocab_transform = {} - +################################################################################### # 출발어(source)와 목적어(target)의 토크나이저(tokenizer)를 생성합니다. # 아래 필요 사항(dependency)을 모두 설치해주세요. -# pip install -U torchdata -# pip install -U spacy -# python -m spacy download en_core_web_sm -# python -m spacy download de_core_news_sm +# +# .. code-block:: python +# +# pip install -U torchdata +# pip install -U spacy +# python -m spacy download en_core_web_sm +# python -m spacy download de_core_news_sm + token_transform[SRC_LANGUAGE] = get_tokenizer('spacy', language='de_core_news_sm') token_transform[TGT_LANGUAGE] = get_tokenizer('spacy', language='en_core_web_sm') @@ -73,9 +77,9 @@ def yield_tokens(data_iter: Iterable, language: str) -> List[str]: specials=special_symbols, special_first=True) -# UNK_IDX를 기본 인덱스로 설정합니다. 이 인덱스는 토큰을 찾지 못하는 경우에 반환됩니다. +# ``UNK_IDX`` 를 기본 인덱스로 설정합니다. 이 인덱스는 토큰을 찾지 못하는 경우에 반환됩니다. # 만약 기본 인덱스를 설정하지 않으면 어휘집(Vocabulary)에서 토큰을 찾지 못하는 경우 -# RuntimeError가 발생합니다. +# ``RuntimeError`` 가 발생합니다. for ln in [SRC_LANGUAGE, TGT_LANGUAGE]: vocab_transform[ln].set_default_index(UNK_IDX) @@ -183,7 +187,7 @@ def decode(self, tgt: Tensor, memory: Tensor, tgt_mask: Tensor): ###################################################################### -# 학습하는 동안, 모델이 예측하는 동안 정답(이후 출현하는 단어)을 보지 못하도록 하는 +# 학습하는 동안, 모델이 예측할 때 정답(이후 출현하는 단어)을 보지 못하도록 하는 # 후속 단어 마스크(subsequent word mask)가 필요합니다. 또한, 출발어와 도착어의 패딩(padding) 토큰들 # 또한 숨겨야 합니다. 아래에 두 가지 모두를 처리할 함수를 정의해보겠습니다. # diff --git a/beginner_source/vt_tutorial.py b/beginner_source/vt_tutorial.py index 483a6d6c1..7ded2f824 100644 --- a/beginner_source/vt_tutorial.py +++ b/beginner_source/vt_tutorial.py @@ -44,16 +44,20 @@ # DeiT를 활용한 이미지 분류 # ------------------------------- # -# DeiT를 사용하여 이미지를 분류하는 방법에 대한 자세한 정보는 DeiT 저장소에 README를 참고하시길 바랍니다. +# DeiT를 사용하여 이미지를 분류하는 방법에 대한 자세한 정보는 DeiT 저장소의 ``README.md`` 를 참고하시길 바랍니다. # 빠른 테스트를 위해서, 먼저 필요한 패키지들을 # 설치합니다: # -# pip install torch torchvision timm pandas requests +# .. code-block:: python +# +# pip install torch torchvision timm pandas requests ####################################################### # Google Colab에서는 아래와 같이 실행합니다: - -# !pip install timm pandas requests +# +# .. code-block:: python +# +# !pip install timm pandas requests ############################# # 그런 다음 아래 스크립트를 실행합니다: @@ -89,7 +93,7 @@ ###################################################################### # ImageNet 목록에 따라 `라벨(labels) 파일 `_ -# 클래스 인덱스의 출력은 269여야 하며, 이는 ‘timber wolf, grey wolf, gray wolf, Canis lupus’에 매핑됩니다. +# 클래스 인덱스의 출력은 269여야 하며, 이는 ``timber wolf, grey wolf, gray wolf, Canis lupus`` 에 매핑됩니다. # # 이제 DeiT 모델을 사용하여 이미지들을 분류할 수 있음을 확인했습니다. # iOS 및 Android 앱에서 실행할 수 있도록 모델을 수정하는 방법을 살펴보겠습니다. @@ -112,7 +116,7 @@ ###################################################################### -# 약 346MB 크기의 스크립팅된 모델 파일 fbdeit_scripted.pt가 생성됩니다. +# 약 346MB 크기의 스크립팅된 모델 파일 ``fbdeit_scripted.pt`` 가 생성됩니다. # # @@ -131,8 +135,10 @@ # 아래의 코드를 실행시켜 봅시다. # +# 서버 추론을 위해 'x86'을, 모바일 추론을 위해 ``qnnpack`` 을 사용합니다. +# (이전의 'fbgemm' 또한 여전히 사용 가능하지만, 'x86'을 기본으로 사용하는 것을 권장합니다.) # 서버 추론을 위해 'fbgemm'을, 모바일 추론을 위해 'qnnpack'을 사용해 봅시다. -backend = "fbgemm" # 이 주피터 노트북에서는 양자화된 모델의 더 느린 추론 속도를 일으키는 qnnpack으로 대체되었습니다. +backend = "x86" # 이 주피터 노트북에서는 양자화된 모델의 더 느린 추론 속도를 일으키는 ``qnnpack`` 으로 대체되었습니다. model.qconfig = torch.quantization.get_default_qconfig(backend) torch.backends.quantized.engine = backend @@ -142,7 +148,7 @@ ###################################################################### -# fbdeit_quantized_scripted.pt 모델의 스크립팅과 양자화가 적용된 버전이 만들어졌습니다. +# ``fbdeit_quantized_scripted.pt`` 모델의 스크립팅과 양자화가 적용된 버전이 만들어졌습니다. # 모델의 크기는 단지 89MB 입니다. # 양자화가 적용되지 않은 모델의 크기인 346MB보다 74%나 감소했습니다! # @@ -170,7 +176,7 @@ ###################################################################### -# 생성된 fbdeit_optimized_scripted_quantized.pt 파일은 +# 생성된 ``fbdeit_optimized_scripted_quantized.pt`` 파일은 # 양자화되고 스크립트되지만 최적화되지 않은 모델과 크기가 거의 같습니다. # 추론 결과는 동일하게 유지됩니다. # diff --git a/conf.py b/conf.py index e66bd0bea..193a08a3a 100644 --- a/conf.py +++ b/conf.py @@ -30,7 +30,7 @@ import os import sys sys.path.insert(0, os.path.abspath('.')) -sys.path.insert(0, os.path.abspath('./.build')) +sys.path.insert(0, os.path.abspath('./.build')) # pytorch/tutorials의 .jenkins/ 의 일부 파일들을 .build/ 에 복사하여 사용 import pytorch_sphinx_theme import torch import glob @@ -38,7 +38,7 @@ from custom_directives import IncludeDirective, GalleryItemDirective, CustomGalleryItemDirective, CustomCalloutItemDirective, CustomCardItemDirective import distutils.file_util import re -from validate_tutorials_built import NOT_RUN +from get_sphinx_filenames import SPHINX_SHOULD_RUN import plotly.io as pio pio.renderers.default = 'sphinx_gallery' @@ -80,6 +80,8 @@ intersphinx_mapping = { "torch": ("https://pytorch.org/docs/stable/", None), + "tensordict": ("https://pytorch-labs.github.io/tensordict/", None), + "torchrl": ("https://pytorch.org/rl/", None), "torchaudio": ("https://pytorch.org/audio/stable/", None), "torchtext": ("https://pytorch.org/text/stable/", None), "torchvision": ("https://pytorch.org/vision/stable/", None), @@ -107,10 +109,12 @@ 'examples_dirs': ['beginner_source', 'intermediate_source', 'advanced_source', 'recipes_source', 'prototype_source'], 'gallery_dirs': ['beginner', 'intermediate', 'advanced', 'recipes', 'prototype'], - 'filename_pattern': '.py', - 'ignore_pattern': re.compile(f"({'|'.join(NOT_RUN)}).py$"), + 'filename_pattern': re.compile(SPHINX_SHOULD_RUN), 'promote_jupyter_magic': True, - 'backreferences_dir': None + 'backreferences_dir': None, + 'first_notebook_cell': ("# Google Colab에서 노트북을 실행하실 때에는 \n" + "# https://tutorials.pytorch.kr/beginner/colab 를 참고하세요.\n" + "%matplotlib inline") } if os.getenv('GALLERY_PATTERN'): @@ -153,7 +157,7 @@ # General information about the project. project = 'PyTorch Tutorials' -copyright = '2022, PyTorch & 파이토치 한국 사용자 모임(PyTorch Korea User Group)' +copyright = '2018-2023, PyTorch & 파이토치 한국 사용자 모임(PyTorch Korea User Group)' author = 'PyTorch contributors' # The version info for the project you're documenting, acts as replacement for @@ -230,6 +234,7 @@ 'collapse_navigation': False, 'display_version': True, 'logo_only': False, + 'navigation_with_keys': True, } diff --git a/distributed/home.rst b/distributed/home.rst index aac2a1df4..09008a50e 100644 --- a/distributed/home.rst +++ b/distributed/home.rst @@ -27,7 +27,7 @@ Learn DDP .. grid-item-card:: :octicon:`file-code;1em` DDP Intro Video Tutorials - :link: https://pytorch.org/tutorials/beginner/ddp_series_intro.html?utm_source=distr_landing&utm_medium=ddp_series_intro + :link: https://tutorials.pytorch.kr/beginner/ddp_series_intro.html?utm_source=distr_landing&utm_medium=ddp_series_intro :link-type: url A step-by-step video series on how to get started with @@ -37,7 +37,7 @@ Learn DDP .. grid-item-card:: :octicon:`file-code;1em` Getting Started with Distributed Data Parallel - :link: https://pytorch.org/tutorials/intermediate/ddp_tutorial.html?utm_source=distr_landing&utm_medium=intermediate_ddp_tutorial + :link: https://tutorials.pytorch.kr/intermediate/ddp_tutorial.html?utm_source=distr_landing&utm_medium=intermediate_ddp_tutorial :link-type: url This tutorial provides a short and gentle intro to the PyTorch @@ -48,10 +48,10 @@ Learn DDP .. grid-item-card:: :octicon:`file-code;1em` Distributed Training with Uneven Inputs Using the Join Context Manager - :link: https://pytorch.org/tutorials/advanced/generic_join.html?utm_source=distr_landing&utm_medium=generic_join + :link: https://tutorials.pytorch.kr/advanced/generic_join.html?utm_source=distr_landing&utm_medium=generic_join :link-type: url - This tutorial describes the Join context manager and + This tutorial describes the Join context manager and demonstrates it's use with DistributedData Parallel. +++ :octicon:`code;1em` Code @@ -65,7 +65,7 @@ Learn FSDP .. grid-item-card:: :octicon:`file-code;1em` Getting Started with FSDP - :link: https://pytorch.org/tutorials/intermediate/FSDP_tutorial.html?utm_source=distr_landing&utm_medium=FSDP_getting_started + :link: https://tutorials.pytorch.kr/intermediate/FSDP_tutorial.html?utm_source=distr_landing&utm_medium=FSDP_getting_started :link-type: url This tutorial demonstrates how you can perform distributed training @@ -75,7 +75,7 @@ Learn FSDP .. grid-item-card:: :octicon:`file-code;1em` FSDP Advanced - :link: https://pytorch.org/tutorials/intermediate/FSDP_adavnced_tutorial.html?utm_source=distr_landing&utm_medium=FSDP_advanced + :link: https://tutorials.pytorch.kr/intermediate/FSDP_adavnced_tutorial.html?utm_source=distr_landing&utm_medium=FSDP_advanced :link-type: url In this tutorial, you will learn how to fine-tune a HuggingFace (HF) T5 @@ -92,7 +92,7 @@ Learn RPC .. grid-item-card:: :octicon:`file-code;1em` Getting Started with Distributed RPC Framework - :link: https://pytorch.org/tutorials/intermediate/rpc_tutorial.html?utm_source=distr_landing&utm_medium=rpc_getting_started + :link: https://tutorials.pytorch.kr/intermediate/rpc_tutorial.html?utm_source=distr_landing&utm_medium=rpc_getting_started :link-type: url This tutorial demonstrates how to get started with RPC-based distributed @@ -102,7 +102,7 @@ Learn RPC .. grid-item-card:: :octicon:`file-code;1em` Implementing a Parameter Server Using Distributed RPC Framework - :link: https://pytorch.org/tutorials/intermediate/rpc_param_server_tutorial.html?utm_source=distr_landing&utm_medium=rpc_param_server_tutorial + :link: https://tutorials.pytorch.kr/intermediate/rpc_param_server_tutorial.html?utm_source=distr_landing&utm_medium=rpc_param_server_tutorial :link-type: url This tutorial walks you through a simple example of implementing a @@ -112,7 +112,7 @@ Learn RPC .. grid-item-card:: :octicon:`file-code;1em` Implementing Batch RPC Processing Using Asynchronous Executions - :link: https://pytorch.org/tutorials/intermediate/rpc_async_execution.html?utm_source=distr_landing&utm_medium=rpc_async_execution + :link: https://tutorials.pytorch.kr/intermediate/rpc_async_execution.html?utm_source=distr_landing&utm_medium=rpc_async_execution :link-type: url In this tutorial you will build batch-processing RPC applications @@ -124,7 +124,7 @@ Learn RPC .. grid-item-card:: :octicon:`file-code;1em` Combining Distributed DataParallel with Distributed RPC Framework - :link: https://pytorch.org/tutorials/advanced/rpc_ddp_tutorial.html?utm_source=distr_landing&utm_medium=rpc_plus_ddp + :link: https://tutorials.pytorch.kr/advanced/rpc_ddp_tutorial.html?utm_source=distr_landing&utm_medium=rpc_plus_ddp :link-type: url In this tutorial you will learn how to combine distributed data @@ -141,7 +141,7 @@ Custom Extensions .. grid-item-card:: :octicon:`file-code;1em` Customize Process Group Backends Using Cpp Extensions - :link: https://pytorch.org/tutorials/intermediate/process_group_cpp_extension_tutorial.html?utm_source=distr_landing&utm_medium=custom_extensions_cpp + :link: https://tutorials.pytorch.kr/intermediate/process_group_cpp_extension_tutorial.html?utm_source=distr_landing&utm_medium=custom_extensions_cpp :link-type: url In this tutorial you will learn to implement a custom `ProcessGroup` diff --git a/docs/_downloads/09dab7b70298bcb798ab79840558b800/maskedtensor_sparsity.ipynb b/docs/_downloads/09dab7b70298bcb798ab79840558b800/maskedtensor_sparsity.ipynb index f213f30f8..a739336da 100644 --- a/docs/_downloads/09dab7b70298bcb798ab79840558b800/maskedtensor_sparsity.ipynb +++ b/docs/_downloads/09dab7b70298bcb798ab79840558b800/maskedtensor_sparsity.ipynb @@ -22,7 +22,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "Before working on this tutorial, please make sure to review our\n`MaskedTensor Overview tutorial `.\n\n## Introduction\n\nSparsity has been an area of rapid growth and importance within PyTorch; if any sparsity terms are confusing below,\nplease refer to the [sparsity tutorial](https://pytorch.org/docs/stable/sparse.html)_ for additional details.\n\nSparse storage formats have been proven to be powerful in a variety of ways. As a primer, the first use case\nmost practitioners think about is when the majority of elements are equal to zero (a high degree of sparsity),\nbut even in cases of lower sparsity, certain formats (e.g. BSR) can take advantage of substructures within a matrix.\n\n

Note

At the moment, MaskedTensor supports COO and CSR tensors with plans to support additional formats\n (such as BSR and CSC) in the future. If you have any requests for additional formats,\n please file a feature request [here](https://github.com/pytorch/pytorch/issues)_!

\n\n## Principles\n\nWhen creating a :class:`MaskedTensor` with sparse tensors, there are a few principles that must be observed:\n\n1. ``data`` and ``mask`` must have the same storage format, whether that's :attr:`torch.strided`, :attr:`torch.sparse_coo`, or :attr:`torch.sparse_csr`\n2. ``data`` and ``mask`` must have the same size, indicated by :func:`size()`\n\n\n## Sparse COO tensors\n\nIn accordance with Principle #1, a sparse COO MaskedTensor is created by passing in two sparse COO tensors,\nwhich can be initialized by any of its constructors, for example :func:`torch.sparse_coo_tensor`.\n\nAs a recap of [sparse COO tensors](https://pytorch.org/docs/stable/sparse.html#sparse-coo-tensors)_, the COO format\nstands for \"coordinate format\", where the specified elements are stored as tuples of their indices and the\ncorresponding values. That is, the following are provided:\n\n* ``indices``: array of size ``(ndim, nse)`` and dtype ``torch.int64``\n* ``values``: array of size `(nse,)` with any integer or floating point dtype\n\nwhere ``ndim`` is the dimensionality of the tensor and ``nse`` is the number of specified elements.\n\nFor both sparse COO and CSR tensors, you can construct a :class:`MaskedTensor` by doing either:\n\n1. ``masked_tensor(sparse_tensor_data, sparse_tensor_mask)``\n2. ``dense_masked_tensor.to_sparse_coo()`` or ``dense_masked_tensor.to_sparse_csr()``\n\nThe second method is easier to illustrate so we've shown that below, but for more on the first and the nuances behind\nthe approach, please read the `Sparse COO Appendix `.\n\n\n" + "Before working on this tutorial, please make sure to review our\n`MaskedTensor Overview tutorial `.\n\n## Introduction\n\nSparsity has been an area of rapid growth and importance within PyTorch; if any sparsity terms are confusing below,\nplease refer to the [sparsity tutorial](https://pytorch.org/docs/stable/sparse.html)_ for additional details.\n\nSparse storage formats have been proven to be powerful in a variety of ways. As a primer, the first use case\nmost practitioners think about is when the majority of elements are equal to zero (a high degree of sparsity),\nbut even in cases of lower sparsity, certain formats (e.g. BSR) can take advantage of substructures within a matrix.\n\n

Note

At the moment, MaskedTensor supports COO and CSR tensors with plans to support additional formats\n (such as BSR and CSC) in the future. If you have any requests for additional formats,\n please file a feature request [here](https://github.com/pytorch/pytorch/issues)_!

\n\n## Principles\n\nWhen creating a :class:`MaskedTensor` with sparse tensors, there are a few principles that must be observed:\n\n1. ``data`` and ``mask`` must have the same storage format, whether that's :attr:`torch.strided`, :attr:`torch.sparse_coo`, or :attr:`torch.sparse_csr`\n2. ``data`` and ``mask`` must have the same size, indicated by :func:`size()`\n\n\n## Sparse COO tensors\n\nIn accordance with Principle #1, a sparse COO MaskedTensor is created by passing in two sparse COO tensors,\nwhich can be initialized by any of its constructors, for example :func:`torch.sparse_coo_tensor`.\n\nAs a recap of [sparse COO tensors](https://pytorch.org/docs/stable/sparse.html#sparse-coo-tensors)_, the COO format\nstands for \"coordinate format\", where the specified elements are stored as tuples of their indices and the\ncorresponding values. That is, the following are provided:\n\n* ``indices``: array of size ``(ndim, nse)`` and dtype ``torch.int64``\n* ``values``: array of size `(nse,)` with any integer or floating point dtype\n\nwhere ``ndim`` is the dimensionality of the tensor and ``nse`` is the number of specified elements.\n\nFor both sparse COO and CSR tensors, you can construct a :class:`MaskedTensor` by doing either:\n\n1. ``masked_tensor(sparse_tensor_data, sparse_tensor_mask)``\n2. ``dense_masked_tensor.to_sparse_coo()`` or ``dense_masked_tensor.to_sparse_csr()``\n\nThe second method is easier to illustrate so we've shown that below, but for more on the first and the nuances behind\nthe approach, please read the `Sparse COO Appendix `.\n\n\n" ] }, { @@ -76,7 +76,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "### Binary\n[Binary operators](https://pytorch.org/docs/master/masked.html#unary-operators)_ are also supported, but the\ninput masks from the two masked tensors must match. For more information on why this decision was made, please\nfind our [MaskedTensor: Advanced Semantics tutorial](https://pytorch.org/tutorials/prototype/maskedtensor_advanced_semantics.html)_.\n\nPlease find an example below:\n\n\n" + "### Binary\n[Binary operators](https://pytorch.org/docs/master/masked.html#unary-operators)_ are also supported, but the\ninput masks from the two masked tensors must match. For more information on why this decision was made, please\nfind our [MaskedTensor: Advanced Semantics tutorial](https://tutorials.pytorch.kr/prototype/maskedtensor_advanced_semantics.html)_.\n\nPlease find an example below:\n\n\n" ] }, { @@ -296,7 +296,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## Conclusion\nIn this tutorial, we have introduced how to use :class:`MaskedTensor` with sparse COO and CSR formats and\ndiscussed some of the subtleties under the hood in case users decide to access the underlying data structures\ndirectly. Sparse storage formats and masked semantics indeed have strong synergies, so much so that they are\nsometimes used as proxies for each other (as we will see in the next tutorial). In the future, we certainly plan\nto invest and continue developing in this direction.\n\n## Further Reading\n\nTo continue learning more, you can find our\n[Efficiently writing \"sparse\" semantics for Adagrad with MaskedTensor tutorial](https://pytorch.org/tutorials/prototype/maskedtensor_adagrad.html)_\nto see an example of how MaskedTensor can simplify existing workflows with native masking semantics.\n\n\n" + "## Conclusion\nIn this tutorial, we have introduced how to use :class:`MaskedTensor` with sparse COO and CSR formats and\ndiscussed some of the subtleties under the hood in case users decide to access the underlying data structures\ndirectly. Sparse storage formats and masked semantics indeed have strong synergies, so much so that they are\nsometimes used as proxies for each other (as we will see in the next tutorial). In the future, we certainly plan\nto invest and continue developing in this direction.\n\n## Further Reading\n\nTo continue learning more, you can find our\n[Efficiently writing \"sparse\" semantics for Adagrad with MaskedTensor tutorial](https://tutorials.pytorch.kr/prototype/maskedtensor_adagrad.html)_\nto see an example of how MaskedTensor can simplify existing workflows with native masking semantics.\n\n\n" ] } ], diff --git a/docs/_downloads/25b0b65f5731eb063aab5b9034092772/maskedtensor_adagrad.py b/docs/_downloads/25b0b65f5731eb063aab5b9034092772/maskedtensor_adagrad.py index 445da1e0e..97c7483f3 100644 --- a/docs/_downloads/25b0b65f5731eb063aab5b9034092772/maskedtensor_adagrad.py +++ b/docs/_downloads/25b0b65f5731eb063aab5b9034092772/maskedtensor_adagrad.py @@ -7,8 +7,8 @@ ###################################################################### # Before working through this tutorial, please review the MaskedTensor -# `Overview `__ and -# `Sparsity `__ tutorials. +# `Overview `__ and +# `Sparsity `__ tutorials. # # Introduction and Motivation # --------------------------- @@ -212,7 +212,7 @@ def _make_sparse(grad, grad_indices, values): # --------------- # # To continue learning more, you can find our final review (for now) on -# `MaskedTensor Advanced Semantics `__ +# `MaskedTensor Advanced Semantics `__ # to see some of the differences in design decisions between :class:`MaskedTensor` and NumPy's MaskedArray, as well # as reduction semantics. # diff --git a/docs/_downloads/38991cbc7763ed7e0f1b711da737b391/tuning_guide.ipynb b/docs/_downloads/38991cbc7763ed7e0f1b711da737b391/tuning_guide.ipynb index 15641278c..143452576 100644 --- a/docs/_downloads/38991cbc7763ed7e0f1b711da737b391/tuning_guide.ipynb +++ b/docs/_downloads/38991cbc7763ed7e0f1b711da737b391/tuning_guide.ipynb @@ -111,7 +111,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "Enable channels_last memory format for computer vision models\n~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\nPyTorch 1.5 introduced support for ``channels_last`` memory format for\nconvolutional networks. This format is meant to be used in conjunction with\n`AMP `_ to further accelerate\nconvolutional neural networks with\n`Tensor Cores `_.\n\nSupport for ``channels_last`` is experimental, but it's expected to work for\nstandard computer vision models (e.g. ResNet-50, SSD). To convert models to\n``channels_last`` format follow\n`Channels Last Memory Format Tutorial `_.\nThe tutorial includes a section on\n`converting existing models `_.\n\n" + "Enable channels_last memory format for computer vision models\n~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\nPyTorch 1.5 introduced support for ``channels_last`` memory format for\nconvolutional networks. This format is meant to be used in conjunction with\n`AMP `_ to further accelerate\nconvolutional neural networks with\n`Tensor Cores `_.\n\nSupport for ``channels_last`` is experimental, but it's expected to work for\nstandard computer vision models (e.g. ResNet-50, SSD). To convert models to\n``channels_last`` format follow\n`Channels Last Memory Format Tutorial `_.\nThe tutorial includes a section on\n`converting existing models `_.\n\n" ] }, { @@ -368,7 +368,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "Use mixed precision and AMP\n~~~~~~~~~~~~~~~~~~~~~~~~~~~\nMixed precision leverages\n`Tensor Cores `_\nand offers up to 3x overall speedup on Volta and newer GPU architectures. To\nuse Tensor Cores AMP should be enabled and matrix/tensor dimensions should\nsatisfy requirements for calling kernels that use Tensor Cores.\n\nTo use Tensor Cores:\n\n* set sizes to multiples of 8 (to map onto dimensions of Tensor Cores)\n\n * see\n `Deep Learning Performance Documentation\n `_\n for more details and guidelines specific to layer type\n * if layer size is derived from other parameters rather than fixed, it can\n still be explicitly padded e.g. vocabulary size in NLP models\n\n* enable AMP\n\n * Introduction to Mixed Precision Training and AMP:\n `video `_,\n `slides `_\n * native PyTorch AMP is available starting from PyTorch 1.6:\n `documentation `_,\n `examples `_,\n `tutorial `_\n\n\n\n" + "Use mixed precision and AMP\n~~~~~~~~~~~~~~~~~~~~~~~~~~~\nMixed precision leverages\n`Tensor Cores `_\nand offers up to 3x overall speedup on Volta and newer GPU architectures. To\nuse Tensor Cores AMP should be enabled and matrix/tensor dimensions should\nsatisfy requirements for calling kernels that use Tensor Cores.\n\nTo use Tensor Cores:\n\n* set sizes to multiples of 8 (to map onto dimensions of Tensor Cores)\n\n * see\n `Deep Learning Performance Documentation\n `_\n for more details and guidelines specific to layer type\n * if layer size is derived from other parameters rather than fixed, it can\n still be explicitly padded e.g. vocabulary size in NLP models\n\n* enable AMP\n\n * Introduction to Mixed Precision Training and AMP:\n `video `_,\n `slides `_\n * native PyTorch AMP is available starting from PyTorch 1.6:\n `documentation `_,\n `examples `_,\n `tutorial `_\n\n\n\n" ] }, { diff --git a/docs/_downloads/4b012086d592d0c5d81c7f66a6259343/maskedtensor_advanced_semantics.ipynb b/docs/_downloads/4b012086d592d0c5d81c7f66a6259343/maskedtensor_advanced_semantics.ipynb index c2b3a23fa..bf5782669 100644 --- a/docs/_downloads/4b012086d592d0c5d81c7f66a6259343/maskedtensor_advanced_semantics.ipynb +++ b/docs/_downloads/4b012086d592d0c5d81c7f66a6259343/maskedtensor_advanced_semantics.ipynb @@ -22,7 +22,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "Before working on this tutorial, please make sure to review our\n`MaskedTensor Overview tutorial `.\n\nThe purpose of this tutorial is to help users understand how some of the advanced semantics work\nand how they came to be. We will focus on two particular ones:\n\n*. Differences between MaskedTensor and [NumPy's MaskedArray](https://numpy.org/doc/stable/reference/maskedarray.html)_ \n*. Reduction semantics\n\n## Preparation\n\n\n" + "Before working on this tutorial, please make sure to review our\n`MaskedTensor Overview tutorial `.\n\nThe purpose of this tutorial is to help users understand how some of the advanced semantics work\nand how they came to be. We will focus on two particular ones:\n\n*. Differences between MaskedTensor and [NumPy's MaskedArray](https://numpy.org/doc/stable/reference/maskedarray.html)_ \n*. Reduction semantics\n\n## Preparation\n\n\n" ] }, { @@ -94,7 +94,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "Note that the mask is `mt0.get_mask() & mt1.get_mask()` since :class:`MaskedTensor`'s mask is the inverse of NumPy's.\n\n\n## Reduction Semantics\n\nRecall in [MaskedTensor's Overview tutorial](https://pytorch.org/tutorials/prototype/maskedtensor_overview.html)_\nwe discussed \"Implementing missing torch.nan* ops\". Those are examples of reductions -- operators that remove one\n(or more) dimensions from a Tensor and then aggregate the result. In this section, we will use reduction semantics\nto motivate our strict requirements around matching masks from above.\n\nFundamentally, :class:`MaskedTensor`s perform the same reduction operation while ignoring the masked out\n(unspecified) values. By way of example:\n\n\n" + "Note that the mask is `mt0.get_mask() & mt1.get_mask()` since :class:`MaskedTensor`'s mask is the inverse of NumPy's.\n\n\n## Reduction Semantics\n\nRecall in [MaskedTensor's Overview tutorial](https://tutorials.pytorch.kr/prototype/maskedtensor_overview.html)_\nwe discussed \"Implementing missing torch.nan* ops\". Those are examples of reductions -- operators that remove one\n(or more) dimensions from a Tensor and then aggregate the result. In this section, we will use reduction semantics\nto motivate our strict requirements around matching masks from above.\n\nFundamentally, :class:`MaskedTensor`s perform the same reduction operation while ignoring the masked out\n(unspecified) values. By way of example:\n\n\n" ] }, { diff --git a/docs/_downloads/5b467300752c6d9599d51103ddd06f4b/maskedtensor_overview.py b/docs/_downloads/5b467300752c6d9599d51103ddd06f4b/maskedtensor_overview.py index 288286936..e65e86d41 100644 --- a/docs/_downloads/5b467300752c6d9599d51103ddd06f4b/maskedtensor_overview.py +++ b/docs/_downloads/5b467300752c6d9599d51103ddd06f4b/maskedtensor_overview.py @@ -328,6 +328,6 @@ # =============== # # To continue learning more, you can find our -# `MaskedTensor Sparsity tutorial `__ +# `MaskedTensor Sparsity tutorial `__ # to see how MaskedTensor enables sparsity and the different storage formats we currently support. # diff --git a/docs/_downloads/7f8a3da4497ba6bec12ef2e8d4d82051/maskedtensor_overview.ipynb b/docs/_downloads/7f8a3da4497ba6bec12ef2e8d4d82051/maskedtensor_overview.ipynb index b6850cd6d..53f1a216a 100644 --- a/docs/_downloads/7f8a3da4497ba6bec12ef2e8d4d82051/maskedtensor_overview.ipynb +++ b/docs/_downloads/7f8a3da4497ba6bec12ef2e8d4d82051/maskedtensor_overview.ipynb @@ -343,7 +343,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "This is a similar problem to safe softmax where `0/0 = nan` when what we really want is an undefined value.\n\n## Conclusion\n\nIn this tutorial, we've introduced what MaskedTensors are, demonstrated how to use them, and motivated their\nvalue through a series of examples and issues that they've helped resolve.\n\n## Further Reading\n\nTo continue learning more, you can find our\n[MaskedTensor Sparsity tutorial](https://pytorch.org/tutorials/prototype/maskedtensor_sparsity.html)_\nto see how MaskedTensor enables sparsity and the different storage formats we currently support.\n\n\n" + "This is a similar problem to safe softmax where `0/0 = nan` when what we really want is an undefined value.\n\n## Conclusion\n\nIn this tutorial, we've introduced what MaskedTensors are, demonstrated how to use them, and motivated their\nvalue through a series of examples and issues that they've helped resolve.\n\n## Further Reading\n\nTo continue learning more, you can find our\n[MaskedTensor Sparsity tutorial](https://tutorials.pytorch.kr/prototype/maskedtensor_sparsity.html)_\nto see how MaskedTensor enables sparsity and the different storage formats we currently support.\n\n\n" ] } ], diff --git a/docs/_downloads/88355d650eb3d5ee6afedaebb57fb9b3/modelsyt_tutorial.py b/docs/_downloads/88355d650eb3d5ee6afedaebb57fb9b3/modelsyt_tutorial.py index 884fcbdb1..8126ce841 100644 --- a/docs/_downloads/88355d650eb3d5ee6afedaebb57fb9b3/modelsyt_tutorial.py +++ b/docs/_downloads/88355d650eb3d5ee6afedaebb57fb9b3/modelsyt_tutorial.py @@ -46,15 +46,15 @@ class is a subclass of ``torch.Tensor``, with the special behavior that import torch class TinyModel(torch.nn.Module): - + def __init__(self): super(TinyModel, self).__init__() - + self.linear1 = torch.nn.Linear(100, 200) self.activation = torch.nn.ReLU() self.linear2 = torch.nn.Linear(200, 10) self.softmax = torch.nn.Softmax() - + def forward(self, x): x = self.linear1(x) x = self.activation(x) @@ -85,19 +85,19 @@ def forward(self, x): # model, and a ``forward()`` method where the computation gets done. Note # that we can print the model, or any of its submodules, to learn about # its structure. -# +# # Common Layer Types # ------------------ -# +# # Linear Layers # ~~~~~~~~~~~~~ -# +# # The most basic type of neural network layer is a *linear* or *fully # connected* layer. This is a layer where every input influences every # output of the layer to a degree specified by the layer’s weights. If a # model has *m* inputs and *n* outputs, the weights will be an *m* x *n* # matrix. For example: -# +# lin = torch.nn.Linear(3, 2) x = torch.rand(1, 3) @@ -117,22 +117,22 @@ def forward(self, x): # If you do the matrix multiplication of ``x`` by the linear layer’s # weights, and add the biases, you’ll find that you get the output vector # ``y``. -# +# # One other important feature to note: When we checked the weights of our # layer with ``lin.weight``, it reported itself as a ``Parameter`` (which # is a subclass of ``Tensor``), and let us know that it’s tracking # gradients with autograd. This is a default behavior for ``Parameter`` # that differs from ``Tensor``. -# +# # Linear layers are used widely in deep learning models. One of the most # common places you’ll see them is in classifier models, which will # usually have one or more linear layers at the end, where the last layer # will have *n* outputs, where *n* is the number of classes the classifier # addresses. -# +# # Convolutional Layers # ~~~~~~~~~~~~~~~~~~~~ -# +# # *Convolutional* layers are built to handle data with a high degree of # spatial correlation. They are very commonly used in computer vision, # where they detect close groupings of features which the compose into @@ -140,9 +140,9 @@ def forward(self, x): # in NLP applications, where a word’s immediate context (that is, the # other words nearby in the sequence) can affect the meaning of a # sentence. -# +# # We saw convolutional layers in action in LeNet5 in an earlier video: -# +# import torch.functional as F @@ -182,7 +182,7 @@ def num_flat_features(self, x): ########################################################################## # Let’s break down what’s happening in the convolutional layers of this # model. Starting with ``conv1``: -# +# # - LeNet5 is meant to take in a 1x32x32 black & white image. **The first # argument to a convolutional layer’s constructor is the number of # input channels.** Here, it is 1. If we were building this model to @@ -198,14 +198,14 @@ def num_flat_features(self, x): # size.** Here, the “5” means we’ve chosen a 5x5 kernel. (If you want a # kernel with height different from width, you can specify a tuple for # this argument - e.g., ``(3, 5)`` to get a 3x5 convolution kernel.) -# +# # The output of a convolutional layer is an *activation map* - a spatial # representation of the presence of features in the input tensor. # ``conv1`` will give us an output tensor of 6x28x28; 6 is the number of # features, and 28 is the height and width of our map. (The 28 comes from # the fact that when scanning a 5-pixel window over a 32-pixel row, there # are only 28 valid positions.) -# +# # We then pass the output of the convolution through a ReLU activation # function (more on activation functions later), then through a max # pooling layer. The max pooling layer takes features near each other in @@ -214,14 +214,14 @@ def num_flat_features(self, x): # cell, and assigning that cell the maximum value of the 4 cells that went # into it. This gives us a lower-resolution version of the activation map, # with dimensions 6x14x14. -# +# # Our next convolutional layer, ``conv2``, expects 6 input channels # (corresponding to the 6 features sought by the first layer), has 16 # output channels, and a 3x3 kernel. It puts out a 16x12x12 activation # map, which is again reduced by a max pooling layer to 16x6x6. Prior to # passing this output to the linear layers, it is reshaped to a 16 \* 6 \* # 6 = 576-element vector for consumption by the next layer. -# +# # There are convolutional layers for addressing 1D, 2D, and 3D tensors. # There are also many more optional arguments for a conv layer # constructor, including stride length(e.g., only scanning every second or @@ -229,22 +229,22 @@ def num_flat_features(self, x): # edges of the input), and more. See the # `documentation `__ # for more information. -# +# # Recurrent Layers # ~~~~~~~~~~~~~~~~ -# +# # *Recurrent neural networks* (or *RNNs)* are used for sequential data - # anything from time-series measurements from a scientific instrument to # natural language sentences to DNA nucleotides. An RNN does this by # maintaining a *hidden state* that acts as a sort of memory for what it # has seen in the sequence so far. -# +# # The internal structure of an RNN layer - or its variants, the LSTM (long # short-term memory) and GRU (gated recurrent unit) - is moderately # complex and beyond the scope of this video, but we’ll show you what one # looks like in action with an LSTM-based part-of-speech tagger (a type of # classifier that tells you if a word is a noun, verb, etc.): -# +# class LSTMTagger(torch.nn.Module): @@ -271,7 +271,7 @@ def forward(self, sentence): ######################################################################## # The constructor has four arguments: -# +# # - ``vocab_size`` is the number of words in the input vocabulary. Each # word is a one-hot vector (or unit vector) in a # ``vocab_size``-dimensional space. @@ -281,7 +281,7 @@ def forward(self, sentence): # space, where words with similar meanings are close together in the # space. # - ``hidden_dim`` is the size of the LSTM’s memory. -# +# # The input will be a sentence with the words represented as indices of # one-hot vectors. The embedding layer will then map these down to an # ``embedding_dim``-dimensional space. The LSTM takes this sequence of @@ -290,15 +290,15 @@ def forward(self, sentence): # ``log_softmax()`` to the output of the final layer converts the output # into a normalized set of estimated probabilities that a given word maps # to a given tag. -# +# # If you’d like to see this network in action, check out the `Sequence # Models and LSTM -# Networks `__ +# Networks `__ # tutorial on pytorch.org. -# +# # Transformers # ~~~~~~~~~~~~ -# +# # *Transformers* are multi-purpose networks that have taken over the state # of the art in NLP with models like BERT. A discussion of transformer # architecture is beyond the scope of this video, but PyTorch has a @@ -312,22 +312,22 @@ def forward(self, sentence): # ``TransformerDecoderLayer``). For details, check out the # `documentation `__ # on transformer classes, and the relevant -# `tutorial `__ +# `tutorial `__ # on pytorch.org. -# +# # Other Layers and Functions # -------------------------- -# +# # Data Manipulation Layers # ~~~~~~~~~~~~~~~~~~~~~~~~ -# +# # There are other layer types that perform important functions in models, # but don’t participate in the learning process themselves. -# +# # **Max pooling** (and its twin, min pooling) reduce a tensor by combining # cells, and assigning the maximum value of the input cells to the output # cell (we saw this). For example: -# +# my_tensor = torch.rand(1, 6, 6) print(my_tensor) @@ -340,12 +340,12 @@ def forward(self, sentence): # If you look closely at the values above, you’ll see that each of the # values in the maxpooled output is the maximum value of each quadrant of # the 6x6 input. -# +# # **Normalization layers** re-center and normalize the output of one layer # before feeding it to another. Centering the and scaling the intermediate # tensors has a number of beneficial effects, such as letting you use # higher learning rates without exploding/vanishing gradients. -# +# my_tensor = torch.rand(1, 4, 4) * 20 + 5 print(my_tensor) @@ -366,22 +366,22 @@ def forward(self, sentence): # in the neighborhood of 15. After running it through the normalization # layer, you can see that the values are smaller, and grouped around zero # - in fact, the mean should be very small (> 1e-8). -# +# # This is beneficial because many activation functions (discussed below) # have their strongest gradients near 0, but sometimes suffer from # vanishing or exploding gradients for inputs that drive them far away # from zero. Keeping the data centered around the area of steepest # gradient will tend to mean faster, better learning and higher feasible # learning rates. -# +# # **Dropout layers** are a tool for encouraging *sparse representations* # in your model - that is, pushing it to do inference with less data. -# +# # Dropout layers work by randomly setting parts of the input tensor # *during training* - dropout layers are always turned off for inference. # This forces the model to learn against this masked or reduced dataset. # For example: -# +# my_tensor = torch.rand(1, 4, 4) @@ -394,10 +394,10 @@ def forward(self, sentence): # Above, you can see the effect of dropout on a sample tensor. You can use # the optional ``p`` argument to set the probability of an individual # weight dropping out; if you don’t it defaults to 0.5. -# +# # Activation Functions # ~~~~~~~~~~~~~~~~~~~~ -# +# # Activation functions make deep learning possible. A neural network is # really a program - with many parameters - that *simulates a mathematical # function*. If all we did was multiple tensors by layer weights @@ -406,17 +406,17 @@ def forward(self, sentence): # reduce could be reduced to a single matrix multiplication. Inserting # *non-linear* activation functions between layers is what allows a deep # learning model to simulate any function, rather than just linear ones. -# +# # ``torch.nn.Module`` has objects encapsulating all of the major # activation functions including ReLU and its many variants, Tanh, # Hardtanh, sigmoid, and more. It also includes other functions, such as # Softmax, that are most useful at the output stage of a model. -# +# # Loss Functions # ~~~~~~~~~~~~~~ -# +# # Loss functions tell us how far a model’s prediction is from the correct # answer. PyTorch contains a variety of loss functions, including common # MSE (mean squared error = L2 norm), Cross Entropy Loss and Negative # Likelihood Loss (useful for classifiers), and others. -# +# diff --git a/docs/_downloads/8c575aa36ad9a61584ec0ddf11cbe84d/fx_profiling_tutorial.py b/docs/_downloads/8c575aa36ad9a61584ec0ddf11cbe84d/fx_profiling_tutorial.py index 06726e4dd..8b4faf72f 100644 --- a/docs/_downloads/8c575aa36ad9a61584ec0ddf11cbe84d/fx_profiling_tutorial.py +++ b/docs/_downloads/8c575aa36ad9a61584ec0ddf11cbe84d/fx_profiling_tutorial.py @@ -218,7 +218,7 @@ def summary(self, should_sort : bool = False) -> str: # https://github.com/pytorch/pytorch/issues/51393 # * BatchNorm2d also takes up significant time. We can continue this # line of thinking and optimize this in the Conv-BN Fusion with FX -# `tutorial `_. +# `tutorial `_. # # # Conclusion diff --git a/docs/_downloads/8c82db84c10318a94cbe213adb618139/tuning_guide.py b/docs/_downloads/8c82db84c10318a94cbe213adb618139/tuning_guide.py index 86d0d4cf4..b969fea32 100644 --- a/docs/_downloads/8c82db84c10318a94cbe213adb618139/tuning_guide.py +++ b/docs/_downloads/8c82db84c10318a94cbe213adb618139/tuning_guide.py @@ -137,9 +137,9 @@ def fused_gelu(x): # Support for ``channels_last`` is experimental, but it's expected to work for # standard computer vision models (e.g. ResNet-50, SSD). To convert models to # ``channels_last`` format follow -# `Channels Last Memory Format Tutorial `_. +# `Channels Last Memory Format Tutorial `_. # The tutorial includes a section on -# `converting existing models `_. +# `converting existing models `_. ############################################################################### # Checkpoint intermediate buffers @@ -363,7 +363,7 @@ def fused_gelu(x): # * native PyTorch AMP is available starting from PyTorch 1.6: # `documentation `_, # `examples `_, -# `tutorial `_ +# `tutorial `_ # # diff --git a/docs/_downloads/939d5d91f629a3f2d79e69156ffbcaad/maskedtensor_sparsity.py b/docs/_downloads/939d5d91f629a3f2d79e69156ffbcaad/maskedtensor_sparsity.py index 74024f8e2..0ef0b8f5b 100644 --- a/docs/_downloads/939d5d91f629a3f2d79e69156ffbcaad/maskedtensor_sparsity.py +++ b/docs/_downloads/939d5d91f629a3f2d79e69156ffbcaad/maskedtensor_sparsity.py @@ -7,7 +7,7 @@ ###################################################################### # Before working on this tutorial, please make sure to review our -# `MaskedTensor Overview tutorial `. +# `MaskedTensor Overview tutorial `. # # Introduction # ------------ @@ -117,7 +117,7 @@ # ^^^^^^ # `Binary operators `__ are also supported, but the # input masks from the two masked tensors must match. For more information on why this decision was made, please -# find our `MaskedTensor: Advanced Semantics tutorial `__. +# find our `MaskedTensor: Advanced Semantics tutorial `__. # # Please find an example below: # @@ -310,6 +310,6 @@ # --------------- # # To continue learning more, you can find our -# `Efficiently writing "sparse" semantics for Adagrad with MaskedTensor tutorial `__ +# `Efficiently writing "sparse" semantics for Adagrad with MaskedTensor tutorial `__ # to see an example of how MaskedTensor can simplify existing workflows with native masking semantics. # diff --git a/docs/_downloads/945dab6b984b8789385e32187d4a8964/fx_profiling_tutorial.ipynb b/docs/_downloads/945dab6b984b8789385e32187d4a8964/fx_profiling_tutorial.ipynb index 5c4727071..495ee721c 100644 --- a/docs/_downloads/945dab6b984b8789385e32187d4a8964/fx_profiling_tutorial.ipynb +++ b/docs/_downloads/945dab6b984b8789385e32187d4a8964/fx_profiling_tutorial.ipynb @@ -158,7 +158,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "There are two things we should call out here:\n\n* MaxPool2d takes up the most time. This is a known issue:\n https://github.com/pytorch/pytorch/issues/51393\n* BatchNorm2d also takes up significant time. We can continue this\n line of thinking and optimize this in the Conv-BN Fusion with FX\n [tutorial](https://pytorch.org/tutorials/intermediate/fx_conv_bn_fuser.html). \n\n\n## Conclusion\nAs we can see, using FX we can easily capture PyTorch programs (even\nones we don't have the source code for!) in a machine-interpretable\nformat and use that for analysis, such as the performance analysis\nwe've done here. FX opens up an exiciting world of possibilities for\nworking with PyTorch programs.\n\nFinally, since FX is still in beta, we would be happy to hear any\nfeedback you have about using it. Please feel free to use the\nPyTorch Forums (https://discuss.pytorch.org/) and the issue tracker\n(https://github.com/pytorch/pytorch/issues) to provide any feedback\nyou might have.\n\n" + "There are two things we should call out here:\n\n* MaxPool2d takes up the most time. This is a known issue:\n https://github.com/pytorch/pytorch/issues/51393\n* BatchNorm2d also takes up significant time. We can continue this\n line of thinking and optimize this in the Conv-BN Fusion with FX\n [tutorial](https://tutorials.pytorch.kr/intermediate/fx_conv_bn_fuser.html). \n\n\n## Conclusion\nAs we can see, using FX we can easily capture PyTorch programs (even\nones we don't have the source code for!) in a machine-interpretable\nformat and use that for analysis, such as the performance analysis\nwe've done here. FX opens up an exiciting world of possibilities for\nworking with PyTorch programs.\n\nFinally, since FX is still in beta, we would be happy to hear any\nfeedback you have about using it. Please feel free to use the\nPyTorch Forums (https://discuss.pytorch.org/) and the issue tracker\n(https://github.com/pytorch/pytorch/issues) to provide any feedback\nyou might have.\n\n" ] } ], diff --git a/docs/_downloads/9c494adfa705afca5e6375d6cb339a14/maskedtensor_adagrad.ipynb b/docs/_downloads/9c494adfa705afca5e6375d6cb339a14/maskedtensor_adagrad.ipynb index 537d92c70..7f58469f1 100644 --- a/docs/_downloads/9c494adfa705afca5e6375d6cb339a14/maskedtensor_adagrad.ipynb +++ b/docs/_downloads/9c494adfa705afca5e6375d6cb339a14/maskedtensor_adagrad.ipynb @@ -22,7 +22,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "Before working through this tutorial, please review the MaskedTensor\n[Overview](https://pytorch.org/tutorials/prototype/maskedtensor_overview.html)_ and\n[Sparsity](https://pytorch.org/tutorials/prototype/maskedtensor_sparsity.html)_ tutorials.\n\n## Introduction and Motivation\n[Issue 1369](https://github.com/pytorch/pytorch/issues/1369)_ discussed the additional lines of code\nthat were introduced while writing \"sparse\" semantics for Adagrad, but really,\nthe code uses sparsity as a proxy for masked semantics rather than the intended use case of sparsity:\na compression and optimization technique.\nPreviously, we worked around the lack of formal masked semantics by introducing one-off semantics and operators\nwhile forcing users to be aware of storage details such as indices and values.\n\nNow that we have masked semantics, we are better equipped to point out when sparsity is used as a semantic extension.\nWe'll also compare and contrast this with equivalent code written using MaskedTensor.\nIn the end the code snippets are repeated without additional comments to show the difference in brevity.\n\n## Preparation\n\n\n" + "Before working through this tutorial, please review the MaskedTensor\n[Overview](https://tutorials.pytorch.kr/prototype/maskedtensor_overview.html)_ and\n[Sparsity](https://tutorials.pytorch.kr/prototype/maskedtensor_sparsity.html)_ tutorials.\n\n## Introduction and Motivation\n[Issue 1369](https://github.com/pytorch/pytorch/issues/1369)_ discussed the additional lines of code\nthat were introduced while writing \"sparse\" semantics for Adagrad, but really,\nthe code uses sparsity as a proxy for masked semantics rather than the intended use case of sparsity:\na compression and optimization technique.\nPreviously, we worked around the lack of formal masked semantics by introducing one-off semantics and operators\nwhile forcing users to be aware of storage details such as indices and values.\n\nNow that we have masked semantics, we are better equipped to point out when sparsity is used as a semantic extension.\nWe'll also compare and contrast this with equivalent code written using MaskedTensor.\nIn the end the code snippets are repeated without additional comments to show the difference in brevity.\n\n## Preparation\n\n\n" ] }, { @@ -123,7 +123,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## Conclusion\n\nIn this tutorial, we've discussed how native masked semantics can enable a cleaner developer experience for\nAdagrad's existing implementation in PyTorch, which used sparsity as a proxy for writing masked semantics.\nBut more importantly, allowing masked semantics to be a first class citizen through MaskedTensor\nremoves the reliance on sparsity or unreliable hacks to mimic masking, thereby allowing for proper independence\nand development, while enabling sparse semantics, such as this one.\n\n## Further Reading\n\nTo continue learning more, you can find our final review (for now) on\n[MaskedTensor Advanced Semantics](https://pytorch.org/tutorials/prototype/maskedtensor_advanced_semantics.html)_\nto see some of the differences in design decisions between :class:`MaskedTensor` and NumPy's MaskedArray, as well\nas reduction semantics.\n\n\n" + "## Conclusion\n\nIn this tutorial, we've discussed how native masked semantics can enable a cleaner developer experience for\nAdagrad's existing implementation in PyTorch, which used sparsity as a proxy for writing masked semantics.\nBut more importantly, allowing masked semantics to be a first class citizen through MaskedTensor\nremoves the reliance on sparsity or unreliable hacks to mimic masking, thereby allowing for proper independence\nand development, while enabling sparse semantics, such as this one.\n\n## Further Reading\n\nTo continue learning more, you can find our final review (for now) on\n[MaskedTensor Advanced Semantics](https://tutorials.pytorch.kr/prototype/maskedtensor_advanced_semantics.html)_\nto see some of the differences in design decisions between :class:`MaskedTensor` and NumPy's MaskedArray, as well\nas reduction semantics.\n\n\n" ] } ], diff --git a/docs/_downloads/ba6d64f1f8bd0d6b3c21839705dc840a/tensorboardyt_tutorial.py b/docs/_downloads/ba6d64f1f8bd0d6b3c21839705dc840a/tensorboardyt_tutorial.py index 3b07fe0b8..7903e898f 100644 --- a/docs/_downloads/ba6d64f1f8bd0d6b3c21839705dc840a/tensorboardyt_tutorial.py +++ b/docs/_downloads/ba6d64f1f8bd0d6b3c21839705dc840a/tensorboardyt_tutorial.py @@ -39,11 +39,11 @@ Introduction ------------ - + In this notebook, we’ll be training a variant of LeNet-5 against the Fashion-MNIST dataset. Fashion-MNIST is a set of image tiles depicting various garments, with ten class labels indicating the type of garment -depicted. +depicted. """ @@ -68,9 +68,9 @@ ###################################################################### # Showing Images in TensorBoard # ----------------------------- -# +# # Let’s start by adding sample images from our dataset to TensorBoard: -# +# # Gather datasets and prepare them for consumption transform = transforms.Compose( @@ -127,7 +127,7 @@ def matplotlib_imshow(img, one_channel=False): # minibatch of our input data. Below, we use the ``add_image()`` call on # ``SummaryWriter`` to log the image for consumption by TensorBoard, and # we also call ``flush()`` to make sure it’s written to disk right away. -# +# # Default log_dir argument is "runs" - but it's good to be specific # torch.utils.tensorboard.SummaryWriter is imported above @@ -146,17 +146,17 @@ def matplotlib_imshow(img, one_channel=False): # If you start TensorBoard at the command line and open it in a new # browser tab (usually at `localhost:6006 `__), you should # see the image grid under the IMAGES tab. -# +# # Graphing Scalars to Visualize Training # -------------------------------------- -# +# # TensorBoard is useful for tracking the progress and efficacy of your # training. Below, we’ll run a training loop, track some metrics, and save # the data for TensorBoard’s consumption. -# +# # Let’s define a model to categorize our image tiles, and an optimizer and # loss function for training: -# +# class Net(nn.Module): def __init__(self): @@ -176,7 +176,7 @@ def forward(self, x): x = F.relu(self.fc2(x)) x = self.fc3(x) return x - + net = Net() criterion = nn.CrossEntropyLoss() @@ -186,7 +186,7 @@ def forward(self, x): ########################################################################## # Now let’s train a single epoch, and evaluate the training vs. validation # set losses every 1000 batches: -# +# print(len(validation_loader)) for epoch in range(1): # loop over the dataset multiple times @@ -206,7 +206,7 @@ def forward(self, x): print('Batch {}'.format(i + 1)) # Check against the validation set running_vloss = 0.0 - + net.train(False) # Don't need to track gradents for validation for j, vdata in enumerate(validation_loader, 0): vinputs, vlabels = vdata @@ -214,10 +214,10 @@ def forward(self, x): vloss = criterion(voutputs, vlabels) running_vloss += vloss.item() net.train(True) # Turn gradients back on for training - + avg_loss = running_loss / 1000 avg_vloss = running_vloss / len(validation_loader) - + # Log the running loss averaged per batch writer.add_scalars('Training vs. Validation Loss', { 'Training' : avg_loss, 'Validation' : avg_vloss }, @@ -231,14 +231,14 @@ def forward(self, x): ######################################################################### # Switch to your open TensorBoard and have a look at the SCALARS tab. -# +# # Visualizing Your Model # ---------------------- -# +# # TensorBoard can also be used to examine the data flow within your model. # To do this, call the ``add_graph()`` method with a model and sample # input. When you open -# +# # Again, grab a single mini-batch of images dataiter = iter(training_loader) @@ -254,10 +254,10 @@ def forward(self, x): # When you switch over to TensorBoard, you should see a GRAPHS tab. # Double-click the “NET” node to see the layers and data flow within your # model. -# +# # Visualizing Your Dataset with Embeddings # ---------------------------------------- -# +# # The 28-by-28 image tiles we’re using can be modeled as 784-dimensional # vectors (28 \* 28 = 784). It can be instructive to project this to a # lower-dimensional representation. The ``add_embedding()`` method will @@ -265,9 +265,9 @@ def forward(self, x): # and display them as an interactive 3D chart. The ``add_embedding()`` # method does this automatically by projecting to the three dimensions # with highest variance. -# +# # Below, we’ll take a sample of our data, and generate such an embedding: -# +# # Select a random subset of data and corresponding labels def select_n_random(data, labels, n=100): @@ -297,19 +297,19 @@ def select_n_random(data, labels, n=100): # zoom the model. Examine it at large and small scales, and see whether # you can spot patterns in the projected data and the clustering of # labels. -# +# # For better visibility, it’s recommended to: -# +# # - Select “label” from the “Color by” drop-down on the left. # - Toggle the Night Mode icon along the top to place the # light-colored images on a dark background. -# +# # Other Resources # --------------- -# +# # For more information, have a look at: -# +# # - PyTorch documentation on `torch.utils.tensorboard.SummaryWriter `__ -# - Tensorboard tutorial content in the `PyTorch.org Tutorials `__ +# - Tensorboard tutorial content in the `PyTorch.org Tutorials `__ # - For more information about TensorBoard, see the `TensorBoard # documentation `__ diff --git a/docs/_downloads/d58dcaa8f603e286a2501a4e2a87d1b7/maskedtensor_advanced_semantics.py b/docs/_downloads/d58dcaa8f603e286a2501a4e2a87d1b7/maskedtensor_advanced_semantics.py index 7a0233042..ccc3d90af 100644 --- a/docs/_downloads/d58dcaa8f603e286a2501a4e2a87d1b7/maskedtensor_advanced_semantics.py +++ b/docs/_downloads/d58dcaa8f603e286a2501a4e2a87d1b7/maskedtensor_advanced_semantics.py @@ -6,14 +6,14 @@ """ ###################################################################### -# +# # Before working on this tutorial, please make sure to review our -# `MaskedTensor Overview tutorial `. +# `MaskedTensor Overview tutorial `. # # The purpose of this tutorial is to help users understand how some of the advanced semantics work # and how they came to be. We will focus on two particular ones: # -# *. Differences between MaskedTensor and `NumPy's MaskedArray `__ +# *. Differences between MaskedTensor and `NumPy's MaskedArray `__ # *. Reduction semantics # # Preparation @@ -89,7 +89,7 @@ # Reduction Semantics # ------------------- # -# Recall in `MaskedTensor's Overview tutorial `__ +# Recall in `MaskedTensor's Overview tutorial `__ # we discussed "Implementing missing torch.nan* ops". Those are examples of reductions -- operators that remove one # (or more) dimensions from a Tensor and then aggregate the result. In this section, we will use reduction semantics # to motivate our strict requirements around matching masks from above. @@ -167,4 +167,4 @@ # the associative property amongst binary operations), which in turn can necessitate the user # to be more intentional with their code at times, but we believe this to be the better move. # If you have any thoughts on this, please `let us know `__! -# +# diff --git a/docs/_downloads/e2e556f6b4693c2cef716dd7f40caaf6/tensorboardyt_tutorial.ipynb b/docs/_downloads/e2e556f6b4693c2cef716dd7f40caaf6/tensorboardyt_tutorial.ipynb index bd9f38633..2259bff64 100644 --- a/docs/_downloads/e2e556f6b4693c2cef716dd7f40caaf6/tensorboardyt_tutorial.ipynb +++ b/docs/_downloads/e2e556f6b4693c2cef716dd7f40caaf6/tensorboardyt_tutorial.ipynb @@ -141,7 +141,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "Now if you switch to TensorBoard and select the PROJECTOR tab, you\nshould see a 3D representation of the projection. You can rotate and\nzoom the model. Examine it at large and small scales, and see whether\nyou can spot patterns in the projected data and the clustering of\nlabels.\n\nFor better visibility, it\u2019s recommended to:\n\n- Select \u201clabel\u201d from the \u201cColor by\u201d drop-down on the left.\n- Toggle the Night Mode icon along the top to place the\n light-colored images on a dark background.\n\n## Other Resources\n\nFor more information, have a look at:\n\n- PyTorch documentation on [torch.utils.tensorboard.SummaryWriter](https://pytorch.org/docs/stable/tensorboard.html?highlight=summarywriter)_\n- Tensorboard tutorial content in the [PyTorch.org Tutorials](https://pytorch.org/tutorials/)_ \n- For more information about TensorBoard, see the [TensorBoard\n documentation](https://www.tensorflow.org/tensorboard)_\n\n" + "Now if you switch to TensorBoard and select the PROJECTOR tab, you\nshould see a 3D representation of the projection. You can rotate and\nzoom the model. Examine it at large and small scales, and see whether\nyou can spot patterns in the projected data and the clustering of\nlabels.\n\nFor better visibility, it\u2019s recommended to:\n\n- Select \u201clabel\u201d from the \u201cColor by\u201d drop-down on the left.\n- Toggle the Night Mode icon along the top to place the\n light-colored images on a dark background.\n\n## Other Resources\n\nFor more information, have a look at:\n\n- PyTorch documentation on [torch.utils.tensorboard.SummaryWriter](https://pytorch.org/docs/stable/tensorboard.html?highlight=summarywriter)_\n- Tensorboard tutorial content in the [PyTorch.org Tutorials](https://tutorials.pytorch.kr/)_ \n- For more information about TensorBoard, see the [TensorBoard\n documentation](https://www.tensorflow.org/tensorboard)_\n\n" ] } ], diff --git a/docs/_downloads/fe726e041160526cf828806536922cf6/modelsyt_tutorial.ipynb b/docs/_downloads/fe726e041160526cf828806536922cf6/modelsyt_tutorial.ipynb index d8cfcad55..1b6dde6b4 100644 --- a/docs/_downloads/fe726e041160526cf828806536922cf6/modelsyt_tutorial.ipynb +++ b/docs/_downloads/fe726e041160526cf828806536922cf6/modelsyt_tutorial.ipynb @@ -87,7 +87,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "The constructor has four arguments:\n\n- ``vocab_size`` is the number of words in the input vocabulary. Each\n word is a one-hot vector (or unit vector) in a\n ``vocab_size``-dimensional space.\n- ``tagset_size`` is the number of tags in the output set.\n- ``embedding_dim`` is the size of the *embedding* space for the\n vocabulary. An embedding maps a vocabulary onto a low-dimensional\n space, where words with similar meanings are close together in the\n space.\n- ``hidden_dim`` is the size of the LSTM\u2019s memory.\n\nThe input will be a sentence with the words represented as indices of\none-hot vectors. The embedding layer will then map these down to an\n``embedding_dim``-dimensional space. The LSTM takes this sequence of\nembeddings and iterates over it, fielding an output vector of length\n``hidden_dim``. The final linear layer acts as a classifier; applying\n``log_softmax()`` to the output of the final layer converts the output\ninto a normalized set of estimated probabilities that a given word maps\nto a given tag.\n\nIf you\u2019d like to see this network in action, check out the [Sequence\nModels and LSTM\nNetworks](https://pytorch.org/tutorials/beginner/nlp/sequence_models_tutorial.html)_\ntutorial on pytorch.org.\n\n### Transformers\n\n*Transformers* are multi-purpose networks that have taken over the state\nof the art in NLP with models like BERT. A discussion of transformer\narchitecture is beyond the scope of this video, but PyTorch has a\n``Transformer`` class that allows you to define the overall parameters\nof a transformer model - the number of attention heads, the number of\nencoder & decoder layers, dropout and activation functions, etc. (You\ncan even build the BERT model from this single class, with the right\nparameters!) The ``torch.nn.Transformer`` class also has classes to\nencapsulate the individual components (``TransformerEncoder``,\n``TransformerDecoder``) and subcomponents (``TransformerEncoderLayer``,\n``TransformerDecoderLayer``). For details, check out the\n[documentation](https://pytorch.org/docs/stable/nn.html#transformer-layers)_\non transformer classes, and the relevant\n[tutorial](https://pytorch.org/tutorials/beginner/transformer_tutorial.html)_\non pytorch.org.\n\n## Other Layers and Functions\n\n### Data Manipulation Layers\n\nThere are other layer types that perform important functions in models,\nbut don\u2019t participate in the learning process themselves.\n\n**Max pooling** (and its twin, min pooling) reduce a tensor by combining\ncells, and assigning the maximum value of the input cells to the output\ncell (we saw this). For example:\n\n\n" + "The constructor has four arguments:\n\n- ``vocab_size`` is the number of words in the input vocabulary. Each\n word is a one-hot vector (or unit vector) in a\n ``vocab_size``-dimensional space.\n- ``tagset_size`` is the number of tags in the output set.\n- ``embedding_dim`` is the size of the *embedding* space for the\n vocabulary. An embedding maps a vocabulary onto a low-dimensional\n space, where words with similar meanings are close together in the\n space.\n- ``hidden_dim`` is the size of the LSTM\u2019s memory.\n\nThe input will be a sentence with the words represented as indices of\none-hot vectors. The embedding layer will then map these down to an\n``embedding_dim``-dimensional space. The LSTM takes this sequence of\nembeddings and iterates over it, fielding an output vector of length\n``hidden_dim``. The final linear layer acts as a classifier; applying\n``log_softmax()`` to the output of the final layer converts the output\ninto a normalized set of estimated probabilities that a given word maps\nto a given tag.\n\nIf you\u2019d like to see this network in action, check out the [Sequence\nModels and LSTM\nNetworks](https://tutorials.pytorch.kr/beginner/nlp/sequence_models_tutorial.html)_\ntutorial on pytorch.org.\n\n### Transformers\n\n*Transformers* are multi-purpose networks that have taken over the state\nof the art in NLP with models like BERT. A discussion of transformer\narchitecture is beyond the scope of this video, but PyTorch has a\n``Transformer`` class that allows you to define the overall parameters\nof a transformer model - the number of attention heads, the number of\nencoder & decoder layers, dropout and activation functions, etc. (You\ncan even build the BERT model from this single class, with the right\nparameters!) The ``torch.nn.Transformer`` class also has classes to\nencapsulate the individual components (``TransformerEncoder``,\n``TransformerDecoder``) and subcomponents (``TransformerEncoderLayer``,\n``TransformerDecoderLayer``). For details, check out the\n[documentation](https://pytorch.org/docs/stable/nn.html#transformer-layers)_\non transformer classes, and the relevant\n[tutorial](https://tutorials.pytorch.kr/beginner/transformer_tutorial.html)_\non pytorch.org.\n\n## Other Layers and Functions\n\n### Data Manipulation Layers\n\nThere are other layer types that perform important functions in models,\nbut don\u2019t participate in the learning process themselves.\n\n**Max pooling** (and its twin, min pooling) reduce a tensor by combining\ncells, and assigning the maximum value of the input cells to the output\ncell (we saw this). For example:\n\n\n" ] }, { diff --git a/docs/advanced/generic_join.html b/docs/advanced/generic_join.html index 781034f19..edd9629c1 100644 --- a/docs/advanced/generic_join.html +++ b/docs/advanced/generic_join.html @@ -6,39 +6,39 @@ - + - + - + - + - + - + - + - + Distributed Training with Uneven Inputs Using the Join Context Manager — 파이토치 한국어 튜토리얼 (PyTorch tutorials in Korean) - - - + + + - - - - - - - - + + + + + + + + @@ -57,9 +57,9 @@ - + + - @@ -116,9 +116,9 @@ - - + +
Table of Contents @@ -129,21 +129,21 @@
@@ -314,7 +314,7 @@
- + @@ -333,30 +333,30 @@
    - +
  • - + Tutorials - + >
    • - +
  • Distributed Training with Uneven Inputs Using the Join Context Manager
    • - - + +
  • - - + + - - + +
    • - +
- +
@@ -368,7 +368,7 @@
- +
advanced/generic_join
@@ -390,14 +390,14 @@
- - + +
- +
- +

Distributed Training with Uneven Inputs Using the Join Context Manager

Author: Andrew Gu

@@ -425,13 +425,13 @@

Distributed Training with Uneven Inputs Using the Join Context ManagerRequirements

What is Join?

-

In Getting Started with Distributed Data Parallel - Basic Use Case, you saw +

In Getting Started with Distributed Data Parallel - Basic Use Case, you saw the general skeleton for using DistributedDataParallel to perform data parallel training. This implicitly schedules all-reduces in each backward pass to synchronize gradients across ranks. Such collective communications require participation @@ -826,21 +826,21 @@

Making a Toy Class Work with - +

- + - - + +
더 궁금하시거나 개선할 내용이 있으신가요? 커뮤니티에 참여해보세요!
@@ -860,7 +860,7 @@

Making a Toy Class Work with

- +

@@ -868,11 +868,11 @@

Making a Toy Class Work with

- +
Built with Sphinx using a theme provided by Read the Docs.
- +
@@ -913,12 +913,12 @@

Making a Toy Class Work with

- - - - + + + + @@ -932,8 +932,8 @@

Making a Toy Class Work with - - + + @@ -945,7 +945,7 @@

Making a Toy Class Work with - + @@ -114,9 +114,9 @@ - - + +
Table of Contents @@ -127,21 +127,21 @@
@@ -312,7 +312,7 @@
- + @@ -331,30 +331,30 @@
- +
@@ -366,7 +366,7 @@
- +
beginner/ddp_series_multigpu
@@ -388,14 +388,14 @@
- - + +
- +
- +

Introduction || What is DDP || Single-Node Multi-GPU Training || Fault Tolerance || Multi-Node training || minGPT Training

@@ -541,7 +541,7 @@

Saving model checkpointshere

+DDP here

- ckp = self.model.state_dict()
 + ckp = self.model.module.state_dict()
@@ -597,7 +597,7 @@ 
Further Reading
 
  • Fault Tolerant distributed training  (next tutorial in this series)
    • 
 
  • Intro to DDP (previous tutorial in this series)
    • 
-
  • Getting Started with DDP
    • 
+
  • Getting Started with DDP
    • 
 
  • Process Group
 initialization
    • 
 
@@ -606,12 +606,12 @@ 
    Further Reading
     
    
       
    더 궁금하시거나 개선할 내용이 있으신가요? 커뮤니티에 참여해보세요!
    
@@ -631,7 +631,7 @@ 
    Further Reading
-  
+
 
   
-     
+
 
 
 
@@ -682,12 +682,12 @@ 
    Further Reading
          
          
@@ -701,8 +701,8 @@ 
    Further Reading
          
          
-     
-  
+
+
 
   
   
@@ -714,7 +714,7 @@ 
    Further Reading 
 
   
-  
+
 
-  
+
 
-  
+
 
-  
+
 
-  
+
 
-  
+
 
-  
+
 
   
-  
+
   What is Distributed Data Parallel (DDP) — 파이토치 한국어 튜토리얼 (PyTorch tutorials in Korean)
-  
 
-  
-  
+
+
+
     
-  
-  
-  
 
-  
 
-  
-  
-    
 
-  
+
+
+
+
+
+
+
+
 
   
   
@@ -55,9 +55,9 @@
   
   
     
-     
+    
+
 
-  
   
 
   
@@ -114,9 +114,9 @@
 
 
 
-   
 
-    
+
+
 
     
    
       Table of Contents
@@ -127,21 +127,21 @@
       
    
         
       
    
     
@@ -312,7 +312,7 @@
     
    
       
    
         
    
-          
+
 
 
 
@@ -331,30 +331,30 @@
 
    
 
   
      
-    
+
       
    • 
         
-          
+
             Tutorials
-          
+
          >
       
      • 
 
-        
+
       
    • What is Distributed Data Parallel (DDP)
      • 
-    
-    
+
+
       
    • 
-        
-            
+
+
             
-          
-        
+
+
       
      • 
-    
+
   
    
 
-  
+
 
    
         
    
 
@@ -366,7 +366,7 @@
       
    
         
    
 
-        
+
 
           
    
             
    beginner/ddp_series_theory
    
@@ -388,14 +388,14 @@
             
    
           
    
 
-        
 
-          
+
+
           
    
-          
+
             
    
              
    
 
 
    
 
    
@@ -490,12 +490,12 @@ 
    Further Reading
     
    
       
    더 궁금하시거나 개선할 내용이 있으신가요? 커뮤니티에 참여해보세요!
    
@@ -515,7 +515,7 @@ 
    Further Reading
-  
+
 
   
-     
+
 
 
 
@@ -558,12 +558,12 @@ 
    Further Reading
          
          
@@ -577,8 +577,8 @@ 
    Further Reading
          
          
-     
-  
+
+
 
   
   
@@ -590,7 +590,7 @@ 
    Further Reading 
 
   
-  
+
 
-  
+
 
-  
+
 
-  
+
 
-  
+
 
-  
+
 
-  
+
 
   
-  
+
   Building Models with PyTorch — 파이토치 한국어 튜토리얼 (PyTorch tutorials in Korean)
-  
 
-  
-  
+
+
+
     
-  
-  
-  
 
-  
 
-  
-  
-    
 
-  
+
+
+
+
+
+
+
+
 
   
   
@@ -57,9 +57,9 @@
     
     
     
-     
+    
+
 
-  
   
 
   
@@ -116,9 +116,9 @@
 
 
 
-   
 
-    
+
+
 
     
    
       Table of Contents
@@ -129,21 +129,21 @@
       
    
         
       
    
     
@@ -314,7 +314,7 @@
     
    
       
    
         
    
-          
+
 
 
 
@@ -333,30 +333,30 @@
 
    
 
   
 
-  
+
 
    
         
    
 
@@ -368,7 +368,7 @@
       
    
         
    
 
-        
+
 
           
    
             
    beginner/introyt/modelsyt_tutorial
    
@@ -390,14 +390,14 @@
             
    
           
    
 
-        
 
-          
+
+
           
    
-          
+
             
    
              
    
-              
+
   
    
 
    참고
    
 
    Click here
@@ -754,7 +754,7 @@ 
    Recurrent Layerslog_softmax() to the output of the final layer converts the output
 into a normalized set of estimated probabilities that a given word maps
 to a given tag.
    
-
    If you’d like to see this network in action, check out the Sequence
+
    If you’d like to see this network in action, check out the Sequence
 Models and LSTM
 Networks
 tutorial on pytorch.org.
    
@@ -774,7 +774,7 @@ 
    TransformersTransformerDecoderLayer). For details, check out the
 documentation
 on transformer classes, and the relevant
-tutorial
+tutorial
 on pytorch.org.
    
 
    
 
    
@@ -911,21 +911,21 @@ 
    Loss Functions
-      
+
         
-      
-      
+
+
         
-      
+
     
    
-  
 
-  
+
+
     
    
     
    
       
    더 궁금하시거나 개선할 내용이 있으신가요? 커뮤니티에 참여해보세요!
    
@@ -945,7 +945,7 @@ 
    Loss Functions
-  
+
 
   
-     
+
 
 
 
@@ -1000,12 +1000,12 @@ 
    Loss Functions
          
          
@@ -1019,8 +1019,8 @@ 
    Loss Functions
          
          
-     
-  
+
+
 
   
   
@@ -1032,7 +1032,7 @@ 
    Loss Functions 
 
   
-  
+
 
-  
+
 
-  
+
 
-  
+
 
-  
+
 
-  
+
 
-  
+
 
   
-  
+
   PyTorch TensorBoard Support — 파이토치 한국어 튜토리얼 (PyTorch tutorials in Korean)
-  
 
-  
-  
+
+
+
     
-  
-  
-  
 
-  
 
-  
-  
-    
 
-  
+
+
+
+
+
+
+
+
 
   
   
@@ -57,9 +57,9 @@
     
     
     
-     
+    
+
 
-  
   
 
   
@@ -116,9 +116,9 @@
 
 
 
-   
 
-    
+
+
 
     
    
       Table of Contents
@@ -129,21 +129,21 @@
       
    
         
       
    
     
@@ -314,7 +314,7 @@
     
    
       
    
         
    
-          
+
 
 
 
@@ -333,30 +333,30 @@
 
    
 
   
 
-  
+
 
    
         
    
 
@@ -368,7 +368,7 @@
       
    
         
    
 
-        
+
 
           
    
             
    beginner/introyt/tensorboardyt_tutorial
    
@@ -390,14 +390,14 @@
             
    
           
    
 
-        
 
-          
+
+
           
    
-          
+
             
    
              
    
-              
+
   
    
 
    참고
    
 
    Click here
@@ -693,7 +693,7 @@ 
    Other Resources
 
  • PyTorch documentation on torch.utils.tensorboard.SummaryWriter
    • 
-
  • Tensorboard tutorial content in the PyTorch.org Tutorials
    • 
+
  • Tensorboard tutorial content in the PyTorch.org Tutorials
    • 
 
  • For more information about TensorBoard, see the TensorBoard
 documentation
    • 
 
@@ -712,21 +712,21 @@ 
    Other Resources
-      
+
         
-      
-      
+
+
         
-      
+
     
    
-  
 
-  
+
+
     
    
     
    
       
    더 궁금하시거나 개선할 내용이 있으신가요? 커뮤니티에 참여해보세요!
    
@@ -746,7 +746,7 @@ 
    Other Resources
-  
+
 
   
-     
+
 
 
 
@@ -794,12 +794,12 @@ 
    Other Resources
          
          
@@ -813,8 +813,8 @@ 
    Other Resources
          
          
-     
-  
+
+
 
   
   
@@ -826,7 +826,7 @@ 
    Other Resources 
 
   
-  
+
 
-  
+
 
-  
+
 
-  
+
 
-  
+
 
-  
+
 
-  
+
 
   
-  
+
   Distributed and Parallel Training Tutorials — 파이토치 한국어 튜토리얼 (PyTorch tutorials in Korean)
-  
 
-  
-  
+
+
+
     
-  
-  
-  
 
-  
 
-  
-  
-    
 
-  
+
+
+
+
+
+
+
+
 
   
   
@@ -57,9 +57,9 @@
     
     
     
-     
+    
+
 
-  
   
 
   
@@ -116,9 +116,9 @@
 
 
 
-   
 
-    
+
+
 
     
    
       Table of Contents
@@ -129,21 +129,21 @@
       
    
         
       
    
     
@@ -314,7 +314,7 @@
     
    
       
    
         
    
-          
+
 
 
 
@@ -333,30 +333,30 @@
 
    
 
   
      
-    
+
       
    • 
         
-          
+
             Tutorials
-          
+
          >
       
      • 
 
-        
+
       
    • Distributed and Parallel Training Tutorials
      • 
-    
-    
+
+
       
    • 
-        
-            
+
+
             
-          
-        
+
+
       
      • 
-    
+
   
    
 
-  
+
 
    
         
    
 
@@ -368,7 +368,7 @@
       
    
         
    
 
-        
+
 
           
    
             
    distributed/home
    
@@ -390,14 +390,14 @@
             
    
           
    
 
-        
 
-          
+
+
           
    
-          
+
             
    
              
    
-              
+
   
    
 
    Distributed and Parallel Training Tutorials
    
 
    Distributed training is a model training paradigm that involves
@@ -431,7 +431,7 @@ 
    Distributed and Parallel Training Tutorials
 
     Code   Video
    
 
    
-
    
+
    
 
    
 
    
 
    
@@ -445,7 +445,7 @@ 
    Distributed and Parallel Training Tutorials
 
     Code
    
 
    
-
    
+
    
 
    
 
    
 
    
@@ -460,7 +460,7 @@ 
    Distributed and Parallel Training Tutorials
 
     Code
    
 
    
-
    
+
    
 
    
 
    
 
    
@@ -481,7 +481,7 @@ 
    Distributed and Parallel Training Tutorials
 
     Code
    
 
    
-
    
+
    
 
    
 
    
 
    
@@ -495,7 +495,7 @@ 
    Distributed and Parallel Training Tutorials
 
     Code
    
 
    
-
    
+
    
 
    
 
    
 
    
@@ -516,7 +516,7 @@ 
    Distributed and Parallel Training Tutorials
 
     Code
    
 
    
-
    
+
    @@ -530,7 +530,7 @@

    Distributed and Parallel Training Tutorials

    Code

    -
    +

    @@ -544,7 +544,7 @@

    Distributed and Parallel Training Tutorials

    Code

    -
    +

    @@ -562,7 +562,7 @@

    Distributed and Parallel Training Tutorials

    Code

    -
    +
    @@ -584,7 +584,7 @@

    Distributed and Parallel Training Tutorials

    Code

    -     + @@ -593,21 +593,21 @@

    Distributed and Parallel Training Tutorials - + - - + + - + - - + +
    더 궁금하시거나 개선할 내용이 있으신가요? 커뮤니티에 참여해보세요!
    @@ -627,7 +627,7 @@

    Distributed and Parallel Training Tutorials - + - + @@ -672,12 +672,12 @@

    Distributed and Parallel Training Tutorials @@ -691,8 +691,8 @@

    Distributed and Parallel Training Tutorials - - + + @@ -704,7 +704,7 @@

    Distributed and Parallel Training Tutorials - + - + - + - + - + - + - + - + Advanced Model Training with Fully Sharded Data Parallel (FSDP) — 파이토치 한국어 튜토리얼 (PyTorch tutorials in Korean) - - - + + + - - - - - - - - + + + + + + + + @@ -57,9 +57,9 @@ - + + - @@ -116,9 +116,9 @@ - - + +
    Table of Contents @@ -129,21 +129,21 @@
    @@ -314,7 +314,7 @@
    - + @@ -333,30 +333,30 @@
      - +
    • - + Tutorials - + >
      • - +
    • Advanced Model Training with Fully Sharded Data Parallel (FSDP)
      • - - + +
    • - - + + - - + +
      • - +
    - +
    @@ -368,7 +368,7 @@
    - +
    intermediate/FSDP_adavnced_tutorial
    @@ -390,21 +390,21 @@
    - - + +
    - +
    - +

    Advanced Model Training with Fully Sharded Data Parallel (FSDP)

    Author: Hamid Shojanazeri, Less Wright, Rohan Varma, Yanli Zhao

    This tutorial introduces more advanced features of Fully Sharded Data Parallel (FSDP) as part of the PyTorch 1.12 release. To get familiar with FSDP, please -refer to the FSDP getting started tutorial.

    +refer to the FSDP getting started tutorial.

    In this tutorial, we fine-tune a HuggingFace (HF) T5 model with FSDP for text summarization as a working example.

    The example uses Wikihow and for simplicity, we will showcase the training on a @@ -818,7 +818,7 @@

    Fine-tuning HF T5

    Transformer Wrapping Policy

    -

    As discussed in the previous tutorial, +

    As discussed in the previous tutorial, auto_wrap_policy is one of the FSDP features that make it easy to automatically shard a given model and put the model, optimizer and gradient shards into distinct FSDP units.

    @@ -1019,21 +1019,21 @@

    Summary - + - - + + - +

    - - + +
    더 궁금하시거나 개선할 내용이 있으신가요? 커뮤니티에 참여해보세요!
    @@ -1053,7 +1053,7 @@

    Summary - + - + @@ -1104,12 +1104,12 @@

    Summary @@ -1123,8 +1123,8 @@

    Summary - - + + @@ -1136,7 +1136,7 @@

    Summary - + - + - + - + - + - + - + - + Distributed Pipeline Parallelism Using RPC — 파이토치 한국어 튜토리얼 (PyTorch tutorials in Korean) - - - + + + - - - - - - - - + + + + + + + + @@ -57,9 +57,9 @@ - + + - @@ -116,9 +116,9 @@ - - + +
    Table of Contents @@ -129,21 +129,21 @@
    @@ -314,7 +314,7 @@
    - + @@ -333,30 +333,30 @@
      - +
    • - + Tutorials - + >
      • - +
    • Distributed Pipeline Parallelism Using RPC
      • - - + +
    • - - + + - - + +
      • - +
    - +
    @@ -368,7 +368,7 @@
    - +
    intermediate/dist_pipeline_parallel_tutorial
    @@ -390,14 +390,14 @@
    - - + +
    - +
    - +

    Distributed Pipeline Parallelism Using RPC

    Author: Shen Li

    @@ -408,8 +408,8 @@

    Distributed Pipeline Parallelism Using RPC

  • PyTorch Distributed Overview

    • -

  • Single-Machine Model Parallel Best Practices

    • -

  • Getting started with Distributed RPC Framework

    • +

  • Single-Machine Model Parallel Best Practices

    • +

  • Getting started with Distributed RPC Framework

  • RRef helper functions: RRef.rpc_sync(), RRef.rpc_async(), and @@ -728,21 +728,21 @@

    Step 4: Launch RPC Processes - + - - + + - +

    • - - + +
    더 궁금하시거나 개선할 내용이 있으신가요? 커뮤니티에 참여해보세요!
    @@ -762,7 +762,7 @@

    Step 4: Launch RPC Processes - + - + @@ -808,12 +808,12 @@

    Step 4: Launch RPC Processes @@ -827,8 +827,8 @@

    Step 4: Launch RPC Processes - - + + @@ -840,7 +840,7 @@

    Step 4: Launch RPC Processes - + - + - + - + - + - + - + - + (beta) Building a Simple CPU Performance Profiler with FX — 파이토치 한국어 튜토리얼 (PyTorch tutorials in Korean) - - - + + + - - - - - - - - + + + + + + + + @@ -57,9 +57,9 @@ - + + - @@ -116,9 +116,9 @@ - - + +
    Table of Contents @@ -129,21 +129,21 @@
    @@ -314,7 +314,7 @@
    - + @@ -333,30 +333,30 @@
      - +
    • - + Tutorials - + >
      • - +
    • (beta) Building a Simple CPU Performance Profiler with FX
      • - - + +
    • - - + + - - + +
      • - +
    - +
    @@ -368,7 +368,7 @@
    - +
    intermediate/fx_profiling_tutorial
    @@ -390,14 +390,14 @@
    - - + +
    - +
    - +

    참고

    Click here @@ -841,7 +841,7 @@

    Investigating the Performance of ResNet18https://github.com/pytorch/pytorch/issues/51393

  • BatchNorm2d also takes up significant time. We can continue this line of thinking and optimize this in the Conv-BN Fusion with FX -tutorial.

    • +tutorial.

    @@ -871,21 +871,21 @@

    Conclusion - + - - + + - +

    - - + +
    더 궁금하시거나 개선할 내용이 있으신가요? 커뮤니티에 참여해보세요!
    @@ -905,7 +905,7 @@

    Conclusion - + - + @@ -950,12 +950,12 @@

    Conclusion @@ -969,8 +969,8 @@

    Conclusion - - + + @@ -982,7 +982,7 @@

    Conclusion - + - + - + - + - + - + - + - + Customize Process Group Backends Using Cpp Extensions — 파이토치 한국어 튜토리얼 (PyTorch tutorials in Korean) - - - + + + - - - - - - - - + + + + + + + + @@ -57,9 +57,9 @@ - + + - @@ -116,9 +116,9 @@ - - + +
    Table of Contents @@ -129,21 +129,21 @@
    @@ -314,7 +314,7 @@
    - + @@ -333,30 +333,30 @@
      - +
    • - + Tutorials - + >
      • - +
    • Customize Process Group Backends Using Cpp Extensions
      • - - + +
    • - - + + - - + +
      • - +
    - +
    @@ -368,7 +368,7 @@
    - +
    intermediate/process_group_cpp_extension_tutorial
    @@ -390,14 +390,14 @@
    - - + +
    - +
    - +

    Customize Process Group Backends Using Cpp Extensions

    Author: Feng Tian, Shen Li, Min Si

    @@ -410,7 +410,7 @@

    Customize Process Group Backends Using Cpp Extensions

    PyTorch Distributed Overview

  • PyTorch Collective Communication Package

  • PyTorch Cpp Extension

    • -

  • Writing Distributed Applications with PyTorch

    • +

  • Writing Distributed Applications with PyTorch

    • This tutorial demonstrates how to implement a custom ProcessGroup backend and plug that into @@ -673,21 +673,21 @@

    Step 4: Use The Extension in Application - + - - + + - +

    - - + +
    더 궁금하시거나 개선할 내용이 있으신가요? 커뮤니티에 참여해보세요!
    @@ -707,7 +707,7 @@

    Step 4: Use The Extension in Application - + - + @@ -753,12 +753,12 @@

    Step 4: Use The Extension in Application @@ -772,8 +772,8 @@

    Step 4: Use The Extension in Application - - + + @@ -785,7 +785,7 @@

    Step 4: Use The Extension in Application - + - + - + - + - + - + - + - + Implementing Batch RPC Processing Using Asynchronous Executions — 파이토치 한국어 튜토리얼 (PyTorch tutorials in Korean) - - - + + + - - - - - - - - + + + + + + + + @@ -57,9 +57,9 @@ - + + - @@ -116,9 +116,9 @@ - - + +
    Table of Contents @@ -129,21 +129,21 @@
    @@ -314,7 +314,7 @@
    - + @@ -333,30 +333,30 @@
      - +
    • - + Tutorials - + >
      • - +
    • Implementing Batch RPC Processing Using Asynchronous Executions
      • - - + +
    • - - + + - - + +
      • - +
    - +
    @@ -368,7 +368,7 @@
    - +
    intermediate/rpc_async_execution
    @@ -390,14 +390,14 @@
    - - + +
    - +
    - +

    Implementing Batch RPC Processing Using Asynchronous Executions

    Author: Shen Li

    @@ -584,7 +584,7 @@

    Batch-Updating Parameter Server@rpc.functions.async_execution decorator. In the next section, we re-implement the reinforcement learning example in the previous -Getting started with Distributed RPC Framework +Getting started with Distributed RPC Framework tutorial using batch processing, and demonstrate its impact on the training speed.

    @@ -651,7 +651,7 @@

    Batch-Processing CartPole SolverGetting started with Distributed RPC Framework, +Getting started with Distributed RPC Framework, observers behave a little differently. Instead of exiting when the environment is stopped, it always runs n_steps iterations in every episode. When the environment returns, the observer simply resets the environment and start over @@ -900,21 +900,21 @@

    Learn More - + - - + + - +

    - - + +
    더 궁금하시거나 개선할 내용이 있으신가요? 커뮤니티에 참여해보세요!
    @@ -934,7 +934,7 @@

    Learn More - + - + @@ -979,12 +979,12 @@

    Learn More @@ -998,8 +998,8 @@

    Learn More - - + + @@ -1011,7 +1011,7 @@

    Learn More - + - + - + - + - + - + - + - + Implementing a Parameter Server Using Distributed RPC Framework — 파이토치 한국어 튜토리얼 (PyTorch tutorials in Korean) - - - + + + - - - - - - - - + + + + + + + + @@ -57,9 +57,9 @@ - + + - @@ -116,9 +116,9 @@ - - + +
    Table of Contents @@ -129,21 +129,21 @@
    @@ -314,7 +314,7 @@
    - + @@ -333,30 +333,30 @@
      - +
    • - + Tutorials - + >
      • - +
    • Implementing a Parameter Server Using Distributed RPC Framework
      • - - + +
    • - - + + - - + +
      • - +
    - +
    @@ -368,7 +368,7 @@
    - +
    intermediate/rpc_param_server_tutorial
    @@ -390,14 +390,14 @@
    - - + +
    - +
    - +

    Implementing a Parameter Server Using Distributed RPC Framework

    Author: Rohan Varma

    @@ -412,7 +412,7 @@

    Implementing a Parameter Server Using Distributed RPC FrameworkDistributed RPC framework. The parameter server framework is a paradigm in which a set of servers store parameters, such as large embedding tables, and several trainers query the parameter servers in order to retrieve the most up to date parameters. These trainers can run a training loop locally and occasionally synchronize with the parameter server to get the latest parameters. For more reading on the parameter server approach, check out this paper.

    Using the Distributed RPC Framework, we’ll build an example where multiple trainers use RPC to communicate with the same parameter server and use RRef to access states on the remote parameter server instance. Each trainer will launch its dedicated backward pass in a distributed fashion through stitching of the autograd graph across multiple nodes using distributed autograd.

    -

    Note: This tutorial covers the use of the Distributed RPC Framework, which is useful for splitting a model onto multiple machines, or for implementing a parameter-server training strategy where network trainers fetch parameters hosted on a different machine. If instead you are looking for replicating your model across many GPUs, please see the Distributed Data Parallel tutorial. There is also another RPC tutorial that covers reinforcement learning and RNN use cases.

    +

    Note: This tutorial covers the use of the Distributed RPC Framework, which is useful for splitting a model onto multiple machines, or for implementing a parameter-server training strategy where network trainers fetch parameters hosted on a different machine. If instead you are looking for replicating your model across many GPUs, please see the Distributed Data Parallel tutorial. There is also another RPC tutorial that covers reinforcement learning and RNN use cases.

    Let’s start with the familiar: importing our required modules and defining a simple ConvNet that will train on the MNIST dataset. The below network is largely adopted from the network defined in the pytorch/examples repo.

    import argparse
 import os
@@ -763,21 +763,21 @@ 
    Implementing a Parameter Server Using Distributed RPC Framework
-      
+
         
-      
-      
+
+
         
-      
+
    - - + +
    더 궁금하시거나 개선할 내용이 있으신가요? 커뮤니티에 참여해보세요!
    @@ -797,7 +797,7 @@

    Implementing a Parameter Server Using Distributed RPC Framework - + - + @@ -836,12 +836,12 @@

    Implementing a Parameter Server Using Distributed RPC Framework @@ -855,8 +855,8 @@

    Implementing a Parameter Server Using Distributed RPC Framework - - + + @@ -868,7 +868,7 @@

    Implementing a Parameter Server Using Distributed RPC Framework - + - + - + - + - + - + - + - + TorchRec 소개 — 파이토치 한국어 튜토리얼 (PyTorch tutorials in Korean) - - - + + + - - - - - - - - + + + + + + + + @@ -57,9 +57,9 @@ - + + - @@ -116,9 +116,9 @@ - - + +
    Table of Contents @@ -129,21 +129,21 @@
    @@ -314,7 +314,7 @@
    - + @@ -333,30 +333,30 @@
    - +
    @@ -368,7 +368,7 @@
    - +
    intermediate/torchrec_tutorial
    @@ -390,14 +390,14 @@
    - - + +
    - +
    - +

    TorchRec 소개

    @@ -433,7 +433,7 @@

    개요

    분산 설정

    torch.distributed를 사용하여 환경을 설정합니다. 분산에 대한 자세한 내용은 이 -튜토리얼 을 참고하세요.

    +튜토리얼 을 참고하세요.

    여기서는 1개의 colab GPU에 대응하는 1개의 랭크(colab 프로세스)를 사용합니다.

    import os
 import torch
@@ -573,21 +573,21 @@ 
    추가 자료
-      
+
         
-      
-      
+
+
         
-      
+
    - - + +
    더 궁금하시거나 개선할 내용이 있으신가요? 커뮤니티에 참여해보세요!
    @@ -607,7 +607,7 @@

    추가 자료 - + - + @@ -659,12 +659,12 @@

    추가 자료 @@ -678,8 +678,8 @@

    추가 자료 - - + + @@ -691,7 +691,7 @@

    추가 자료 - + - + - + - + - + - + - + - + Grokking PyTorch Intel CPU performance from first principles — 파이토치 한국어 튜토리얼 (PyTorch tutorials in Korean) - - - + + + - - - - - - - - + + + + + + + + @@ -57,9 +57,9 @@ - + + - @@ -116,9 +116,9 @@ - - + +
    Table of Contents @@ -129,21 +129,21 @@
    @@ -314,7 +314,7 @@
    - + @@ -333,30 +333,30 @@
      - +
    • - + Tutorials - + >
      • - +
    • Grokking PyTorch Intel CPU performance from first principles
      • - - + +
    • - - + + - - + +
      • - +
    - +
    @@ -368,7 +368,7 @@
    - +
    intermediate/torchserve_with_ipex
    @@ -390,14 +390,14 @@
    - - + +
    - +
    - +

    Grokking PyTorch Intel CPU performance from first principles

    A case study on the TorchServe inference framework optimized with Intel® Extension for PyTorch*.

    @@ -440,7 +440,7 @@

    Grokking PyTorch Intel CPU performance from first principlesCPU Performance Tuning Guide, explain how they work, how to profile them and how to integrate them within a model serving framework like TorchServe via an easy to use launch script which we’ve integrated 1 natively.

    +

    In this blog, we’ll walk you through the important runtime configurations you should be aware of from CPU Performance Tuning Guide, explain how they work, how to profile them and how to integrate them within a model serving framework like TorchServe via an easy to use launch script which we’ve integrated 1 natively.

    We’ll explain all of these ideas visually from first principles with lots of profiles and show you how we applied our learnings to make out of the box CPU performance on TorchServe better.

    1. The feature has to be explicitly enabled by setting cpu_launcher_enable=true in config.properties.

      2. @@ -700,7 +700,7 @@

      Conclusion -

    2. CPU specific optimizations

      3. +

    3. CPU specific optimizations

    4. Maximize Performance of Intel® Software Optimization for PyTorch* on CPU

    5. Performance Tuning Guide

    6. Launch Script Usage Guide

      7. @@ -719,21 +719,21 @@

      Acknowledgement - + - - + + - +

    - - + +
    더 궁금하시거나 개선할 내용이 있으신가요? 커뮤니티에 참여해보세요!
    @@ -753,7 +753,7 @@

    Acknowledgement - + - + @@ -802,12 +802,12 @@

    Acknowledgement @@ -821,8 +821,8 @@

    Acknowledgement - - + + @@ -834,7 +834,7 @@

    Acknowledgement - + - + - + - + - + - + - + - + Grokking PyTorch Intel CPU performance from first principles (Part 2) — 파이토치 한국어 튜토리얼 (PyTorch tutorials in Korean) - - - + + + - - - - - - - - + + + + + + + + @@ -57,9 +57,9 @@ - + + - @@ -116,9 +116,9 @@ - - + +
    Table of Contents @@ -129,21 +129,21 @@
    @@ -314,7 +314,7 @@
    - + @@ -333,30 +333,30 @@
      - +
    • - + Tutorials - + >
      • - +
    • Grokking PyTorch Intel CPU performance from first principles (Part 2)
      • - - + +
    • - - + + - - + +
      • - +
    - +
    @@ -368,7 +368,7 @@
    - +
    intermediate/torchserve_with_ipex_2
    @@ -390,18 +390,18 @@
    - - + +
    - +
    - +

    Grokking PyTorch Intel CPU performance from first principles (Part 2)

    Authors: Min Jean Cho, Jing Xu, Mark Saroufim

    -

    In the Grokking PyTorch Intel CPU Performance From First Principles tutorial +

    In the Grokking PyTorch Intel CPU Performance From First Principles tutorial , we have introduced how to tune CPU runtime configurations, how to profile them, and how to integrate them into TorchServe for optimized CPU performance.

    In this tutorial, we will demonstrate boosting performance with memory allocator via the Intel® Extension for PyTorch* Launcher , and optimized kernels on CPU via Intel® Extension for PyTorch* @@ -430,7 +430,7 @@

    Tune for the Back End Bound

    Intel® VTune™ Profiler’s Instrumentation and Tracing Technology (ITT)

    -

    The ITT APIs of Intel® VTune Profiler is a useful tool to annotate a region of your workload for tracing to profile and visualize at a finer granularity of your annotation – OP/function/sub-function granularity. By annotating at the granularity of your PyTorch model’s OPs, Intel® VTune Profiler’s ITT enables op-level profiling. Intel® VTune Profiler’s ITT has been integrated into PyTorch Autograd Profiler. 1

    +

    The ITT APIs of Intel® VTune Profiler is a useful tool to annotate a region of your workload for tracing to profile and visualize at a finer granularity of your annotation – OP/function/sub-function granularity. By annotating at the granularity of your PyTorch model’s OPs, Intel® VTune Profiler’s ITT enables op-level profiling. Intel® VTune Profiler’s ITT has been integrated into PyTorch Autograd Profiler. 1

    1. The feature has to be explicitly enabled by with torch.autograd.profiler.emit_itt().

    @@ -787,21 +787,21 @@

    Acknowledgement - + - - + + - +

    - - + +
    더 궁금하시거나 개선할 내용이 있으신가요? 커뮤니티에 참여해보세요!
    @@ -821,7 +821,7 @@

    Acknowledgement - + - + @@ -901,12 +901,12 @@

    Acknowledgement @@ -920,8 +920,8 @@

    Acknowledgement - - + + @@ -933,7 +933,7 @@

    Acknowledgement - + - + - + - + - + - + - + - + (prototype) FX Graph Mode Post Training Static Quantization — 파이토치 한국어 튜토리얼 (PyTorch tutorials in Korean) - - - + + + - - - - - - - - + + + + + + + + @@ -55,9 +55,9 @@ - + + - @@ -114,9 +114,9 @@ - - + +
    Table of Contents @@ -127,21 +127,21 @@
    @@ -312,7 +312,7 @@
    - + @@ -331,30 +331,30 @@
      - +
    • - + Tutorials - + >
      • - +
    • (prototype) FX Graph Mode Post Training Static Quantization
      • - - + +
    • - - + + - - + +
      • - +
    - +
    @@ -366,7 +366,7 @@
    - +
    prototype/fx_graph_mode_ptq_static
    @@ -388,14 +388,14 @@
    - - + +
    - +
    - +

    (prototype) FX Graph Mode Post Training Static Quantization

    Author: Jerry Zhang

    @@ -404,7 +404,7 @@

    (prototype) FX Graph Mode Post Training Static Quantizationtorch.fx), we’ll have a separate tutorial to show how to make the part of the model we want to quantize compatibble with FX Graph Mode Quantization. -We also have a tutorial for FX Graph Mode Post Training Dynamic Quantization. +We also have a tutorial for FX Graph Mode Post Training Dynamic Quantization. tldr; The FX Graph Mode API looks like the following:

    import torch
 from torch.ao.quantization import get_default_qconfig
@@ -424,7 +424,7 @@ 
    (prototype) FX Graph Mode Post Training Static Quantization
 
    1. Motivation of FX Graph Mode Quantization
    
-
    Currently PyTorch only has eager mode quantization: Static Quantization with Eager Mode in PyTorch.
    
+
    Currently PyTorch only has eager mode quantization: Static Quantization with Eager Mode in PyTorch.
    
 
    We can see there are multiple manual steps involved in the process, including:

    9. Debugging Quantized Model

    @@ -804,12 +804,12 @@

    10. Comparison with Baseline Float Model and Eager Mode Quantization - +

    @@ -880,12 +880,12 @@

    10. Comparison with Baseline Float Model and Eager Mode Quantization

    - - - - + + + + @@ -899,8 +899,8 @@

    10. Comparison with Baseline Float Model and Eager Mode Quantization - - + + @@ -912,7 +912,7 @@

    10. Comparison with Baseline Float Model and Eager Mode Quantization - + @@ -114,9 +114,9 @@ - - + +
    Table of Contents @@ -127,21 +127,21 @@
    @@ -312,7 +312,7 @@
    - + @@ -331,30 +331,30 @@
      - +
    • - + Tutorials - + >
      • - +
    • (Prototype) Efficiently writing 《sparse》 semantics for Adagrad with MaskedTensor
      • - - + +
    • - - + + - - + +
      • - +
    - +
    @@ -366,7 +366,7 @@
    - +
    prototype/maskedtensor_adagrad
    @@ -388,14 +388,14 @@
    - - + +
    - +
    - +

    참고

    Click here @@ -404,8 +404,8 @@

    (Prototype) Efficiently writing 《sparse》 semantics for Adagrad with MaskedTensor

    Before working through this tutorial, please review the MaskedTensor -Overview and -Sparsity tutorials.

    +Overview and +Sparsity tutorials.

    Introduction and Motivation

    Issue 1369 discussed the additional lines of code @@ -613,7 +613,7 @@

    Conclusion

    Further Reading

    To continue learning more, you can find our final review (for now) on -MaskedTensor Advanced Semantics +MaskedTensor Advanced Semantics to see some of the differences in design decisions between MaskedTensor and NumPy’s MaskedArray, as well as reduction semantics.

    Total running time of the script: ( 0 minutes 0.027 seconds)

    @@ -631,12 +631,12 @@

    Further Reading
    더 궁금하시거나 개선할 내용이 있으신가요? 커뮤니티에 참여해보세요!
    @@ -656,7 +656,7 @@

    Further Reading - + - + @@ -704,12 +704,12 @@

    Further Reading @@ -723,8 +723,8 @@

    Further Reading - - + + @@ -736,7 +736,7 @@

    Further Reading - + - + - + - - - + + + - + - + - + (Prototype) MaskedTensor Advanced Semantics — 파이토치 한국어 튜토리얼 (PyTorch tutorials in Korean) - - - + + + - - - - - - - - + + + + + + + + @@ -55,9 +55,9 @@ - + + - @@ -114,9 +114,9 @@ - - + +
    Table of Contents @@ -127,21 +127,21 @@
    @@ -312,7 +312,7 @@
    - + @@ -331,30 +331,30 @@
      - +
    • - + Tutorials - + >
      • - +
    • (Prototype) MaskedTensor Advanced Semantics
      • - - + +
    • - - + + - - + +
      • - +
    - +
    @@ -366,7 +366,7 @@
    - +
    prototype/maskedtensor_advanced_semantics
    @@ -388,14 +388,14 @@
    - - + +
    - +
    - +

    참고

    Click here @@ -404,7 +404,7 @@

    (Prototype) MaskedTensor Advanced Semantics

    Before working on this tutorial, please make sure to review our -MaskedTensor Overview tutorial <https://pytorch.org/tutorials/prototype/maskedtensor_overview.html>.

    +MaskedTensor Overview tutorial <https://tutorials.pytorch.kr/prototype/maskedtensor_overview.html>.

    The purpose of this tutorial is to help users understand how some of the advanced semantics work and how they came to be. We will focus on two particular ones:

    *. Differences between MaskedTensor and NumPy’s MaskedArray @@ -501,7 +501,7 @@

    MaskedTensor vs NumPy’s MaskedArray

    Reduction Semantics

    -

    Recall in MaskedTensor’s Overview tutorial +

    Recall in MaskedTensor’s Overview tutorial we discussed 《Implementing missing torch.nan* ops》. Those are examples of reductions – operators that remove one (or more) dimensions from a Tensor and then aggregate the result. In this section, we will use reduction semantics to motivate our strict requirements around matching masks from above.

    @@ -636,12 +636,12 @@

    Conclusion
    더 궁금하시거나 개선할 내용이 있으신가요? 커뮤니티에 참여해보세요!
    @@ -661,7 +661,7 @@

    Conclusion - + - + @@ -706,12 +706,12 @@

    Conclusion @@ -725,8 +725,8 @@

    Conclusion - - + + @@ -738,7 +738,7 @@

    Conclusion - + - + - + - + - + - + - + - + (Prototype) MaskedTensor Overview — 파이토치 한국어 튜토리얼 (PyTorch tutorials in Korean) - - - + + + - - - - - - - - + + + + + + + + @@ -55,9 +55,9 @@ - + + - @@ -114,9 +114,9 @@ - - + +
    Table of Contents @@ -127,21 +127,21 @@
    @@ -312,7 +312,7 @@
    - + @@ -331,30 +331,30 @@
    - +
    @@ -366,7 +366,7 @@
    - +
    prototype/maskedtensor_overview
    @@ -388,14 +388,14 @@
    - - + +
    - +
    - +

    참고

    Click here @@ -799,7 +799,7 @@

    Conclusion

    Further Reading

    To continue learning more, you can find our -MaskedTensor Sparsity tutorial +MaskedTensor Sparsity tutorial to see how MaskedTensor enables sparsity and the different storage formats we currently support.

    Total running time of the script: ( 0 minutes 0.034 seconds)