-
Notifications
You must be signed in to change notification settings - Fork 225
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
PyTorch v2.0 ๋ฐ์, pytorch/tutorials@9efe789b (#626)
- Loading branch information
Showing
190 changed files
with
8,384 additions
and
8,589 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,107 @@ | ||
| from typing import Any, Dict, List, Optional, Tuple | ||
| import json | ||
| import os | ||
| from pathlib import Path | ||
| # from remove_runnable_code import remove_runnable_code | ||
|
|
||
|
|
||
| # Calculate repo base dir | ||
| REPO_BASE_DIR = Path(__file__).absolute().parent.parent | ||
|
|
||
|
|
||
| def get_all_files() -> List[str]: | ||
| sources = [x.relative_to(REPO_BASE_DIR) for x in REPO_BASE_DIR.glob("*_source/**/*.py") if 'data' not in x.parts] | ||
| return [str(x) for x in sources] | ||
|
|
||
|
|
||
| def read_metadata() -> Dict[str, Any]: | ||
| with (REPO_BASE_DIR / ".jenkins" / "metadata.json").open() as fp: | ||
| return json.load(fp) | ||
|
|
||
|
|
||
| def calculate_shards(all_files: List[str], num_shards: int = 20) -> List[List[str]]: | ||
| sharded_files: List[Tuple[float, List[str]]] = [(0.0, []) for _ in range(num_shards)] | ||
| metadata = read_metadata() | ||
|
|
||
| def get_duration(file: str) -> int: | ||
| # tutorials not listed in the metadata.json file usually take | ||
| # <3min to run, so we'll default to 1min if it's not listed | ||
| return metadata.get(file, {}).get("duration", 60) | ||
|
|
||
| def get_needs_machine(file: str) -> Optional[str]: | ||
| return metadata.get(file, {}).get("needs", None) | ||
|
|
||
| def add_to_shard(i, filename): | ||
| shard_time, shard_jobs = sharded_files[i] | ||
| shard_jobs.append(filename) | ||
| sharded_files[i] = ( | ||
| shard_time + get_duration(filename), | ||
| shard_jobs, | ||
| ) | ||
|
|
||
| all_other_files = all_files.copy() | ||
| needs_gpu_nvidia_small_multi = list( | ||
| filter(lambda x: get_needs_machine(x) == "gpu.nvidia.small.multi", all_files,) | ||
| ) | ||
| needs_gpu_nvidia_medium = list( | ||
| filter(lambda x: get_needs_machine(x) == "gpu.nvidia.large", all_files,) | ||
| ) | ||
| for filename in needs_gpu_nvidia_small_multi: | ||
| # currently, the only job that uses gpu.nvidia.small.multi is the 0th worker, | ||
| # so we'll add all the jobs that need this machine to the 0th worker | ||
| add_to_shard(0, filename) | ||
| all_other_files.remove(filename) | ||
| for filename in needs_gpu_nvidia_medium: | ||
| # currently, the only job that uses gpu.nvidia.large is the 1st worker, | ||
| # so we'll add all the jobs that need this machine to the 1st worker | ||
| add_to_shard(1, filename) | ||
| all_other_files.remove(filename) | ||
|
|
||
| sorted_files = sorted(all_other_files, key=get_duration, reverse=True,) | ||
|
|
||
| for filename in sorted_files: | ||
| min_shard_index = sorted(range(num_shards), key=lambda i: sharded_files[i][0])[ | ||
| 0 | ||
| ] | ||
| add_to_shard(min_shard_index, filename) | ||
| return [x[1] for x in sharded_files] | ||
|
|
||
|
|
||
| def compute_files_to_keep(files_to_run: List[str]) -> List[str]: | ||
| metadata = read_metadata() | ||
| files_to_keep = list(files_to_run) | ||
| for file in files_to_run: | ||
| extra_files = metadata.get(file, {}).get("extra_files", []) | ||
| files_to_keep.extend(extra_files) | ||
| return files_to_keep | ||
|
|
||
|
|
||
| def remove_other_files(all_files, files_to_keep) -> None: | ||
|
|
||
| for file in all_files: | ||
| if file not in files_to_keep: | ||
| remove_runnable_code(file, file) | ||
|
|
||
|
|
||
| def parse_args() -> Any: | ||
| from argparse import ArgumentParser | ||
| parser = ArgumentParser("Select files to run") | ||
| parser.add_argument("--dry-run", action="store_true") | ||
| parser.add_argument("--num-shards", type=int, default=int(os.environ.get("NUM_WORKERS", 20))) | ||
| parser.add_argument("--shard-num", type=int, default=int(os.environ.get("WORKER_ID", 0))) | ||
| return parser.parse_args() | ||
|
|
||
|
|
||
| def main() -> None: | ||
| args = parse_args() | ||
|
|
||
| all_files = get_all_files() | ||
| files_to_run = calculate_shards(all_files, num_shards=args.num_shards)[args.shard_num] | ||
| if not args.dry_run: | ||
| remove_other_files(all_files, compute_files_to_keep(files_to_run)) | ||
| stripped_file_names = [Path(x).stem for x in files_to_run] | ||
| print(" ".join(stripped_file_names)) | ||
|
|
||
|
|
||
| if __name__ == "__main__": | ||
| main() |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,13 @@ | ||
| from pathlib import Path | ||
| from typing import List | ||
|
|
||
| from get_files_to_run import get_all_files | ||
| from validate_tutorials_built import NOT_RUN | ||
|
|
||
|
|
||
| def get_files_for_sphinx() -> List[str]: | ||
| all_py_files = get_all_files() | ||
| return [x for x in all_py_files if all(y not in x for y in NOT_RUN)] | ||
|
|
||
|
|
||
| SPHINX_SHOULD_RUN = "|".join(get_files_for_sphinx()) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1 @@ | ||
| <mxfile host="app.diagrams.net" modified="2022-10-01T16:00:40.980Z" agent="5.0 (X11)" etag="_qbqVrrm3wUvm_i0-Q9T" version="20.4.0" type="device"><diagram id="aSXDm0BvLjt-Za0vl2Tv" name="Page-1">5Vpbc+MmFP41nmkfmpGEpMiPjTftzrTZZtbbbbYvHSxhiRQJFeHb/vqChG4gx95ElqfTeCaGwwEO37lwDskMLNL9zwzmyQONEJk5VrSfgXczx7GtuSW+JOVQUXzXrggxw5FiaglL/BXVMxV1gyNU9Bg5pYTjvE8MaZahkPdokDG667OtKenvmsMYGYRlCIlJ/QNHPKmogWe19PcIx0m9s22pkRTWzIpQJDCiuw4J3M/AglHKq1a6XyAiwatxqeb9dGS0EYyhjJ8z4flQfCient3PwfKXVfTn40P6/vMPapUtJBt14I8oJ/AgaA8opeygZOeHGhBGN1mE5JrWDNztEszRMoehHN0JExC0hKdE9GzRXGNCFpRQVs4Fa09+BD1mMMJC7s7YqvyIsYIz+jfqjPjljxhRsiLG0f4oCHYDrbBJRFPE5SEsNcGt1aPMMVDdXatbu1ZY0tGrr2hQmVPcrNwiLhoK9G9QgGMo4Lec41T6gWN9535v4C/WFLaPTmM/AlrebR8t2z0TruBScAEDLgaziKaCtoI8TAy4wg3bltYqIUFZ9KOMCaIbElgUOOxD1rftai0UGQFCg0/sRzcsRKfdjEMWI37KGkx1dOD2BtCuaQwRyPG2L+6QCtQOjxSLgxz1DRdoWqyOqWZ1I42+kNdfCOjmUOFgLCR0U0aemi2XDMVxgQ3ztK0X5fJtjR/0+EWjkqA1z0YHr7dYz7DYR0pwKM/5AfFRw2sEUbAOh8PrLYI+sgbDaxig1foy4dWxrh1fAzOACodeqi5lPKExzSC5b6laGGh5fqU0V8g/I84PKluBG06HQu8okcN/W+Q4OyS8CWTfsPFPlbz/Cxu/eg5hm0nEmPcg2mP+JNs3nup96Yy823c7h/HvTu8/cXfqV9H8lXen5xxJuUa+O91A2yd4+e709LvTP8Hvvsh/mbvWNtPDUR0hE+I9NVNF50vrFrLb+kLZq52hcaCO+9hTuY9zpvt413Qf3Vqc29e6z1xbyJkm9TSKb51fS4mdKVJP2zXc4fc8grwsLb3rlpaODog3cItaU96ijUdNmirWscHuXq03jjdVeLDnZ8aHI+qcJsGspewFdp8Iee8ivJU7Ehxn5YD/z0a+qN0RtOZtT7Ri9Q1Tac3ZqsjLvvWJQZzhLBbNUmtqXSFnuXQzb5zd7Bvxa5FQWkgvbB4vJDxCgXVlCEOOaXZhURwpylJQiRQFZdsL7wfkfh9RSFkkGql6XrQ2KiRddG9X7t2+rF10L6/ElUpu5VZ/ZWUt1D/piuk76/K8pWyq5S+lHiVi23oGaA9E7PlAxG4Yxw/ZZr4X1q5Vu9AE6V8wP5UAyt4jYlgcG7HrlUhVGL1WkgeO5EDf/r5oDdcuo9dIeqUPXk7ygK/xn3iPNACxJkgKHTMpHNVJBmod6+Z2snzmqmWMrlCgVx/nWjjQLc+7jIUDvYw5ZeFA43emsFjzCf0iYd2ava6q7z2LTVbX18XdyaDvX9UjNIMBevl2tkdo71VATyrG8ghd4LcV6qLb/oW/Ym//TwLc/ws=</diagram></mxfile> |
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.