From 62633f60870cc8f1e96c49d4275eecaf8c21325d Mon Sep 17 00:00:00 2001 From: jirka Date: Fri, 26 Jul 2024 09:36:56 +0200 Subject: [PATCH 1/2] ci: pre-install in separate step --- .actions/assistant.py | 5 +++++ .azure/ipynb-publish.yml | 7 ++++--- .azure/ipynb-validate.yml | 5 +++-- 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/.actions/assistant.py b/.actions/assistant.py index 66fc78483..770ddce2d 100644 --- a/.actions/assistant.py +++ b/.actions/assistant.py @@ -308,6 +308,11 @@ def _parse_requirements(folder: str) -> Tuple[str, str]: return " ".join([f'"{req}"' for req in requires]), " ".join(pip_args) + @staticmethod + def pip_install(folder: str) -> str: + """Parse all notebook requirements to be pre-installed.""" + return " ".join(AssistantCLI._parse_requirements(folder)) + @staticmethod def _bash_download_data(folder: str) -> List[str]: """Generate sequence of commands for optional downloading dataset specified in the meta file. diff --git a/.azure/ipynb-publish.yml b/.azure/ipynb-publish.yml index ef9964ec4..6f1bdfe40 100644 --- a/.azure/ipynb-publish.yml +++ b/.azure/ipynb-publish.yml @@ -143,9 +143,10 @@ jobs: - bash: | set -e pip --version - # todo: export requirements for notebooks to file and execute # todo: adjust torch ecosystem versions pip install -r requirements.txt + # todo: export requirements for notebooks to file and execute + pip install $(python .actions/assistant.py pip-install --folder=$(notebook)) displayName: "Install dependencies" timeoutInMinutes: "15" @@ -156,14 +157,14 @@ jobs: python -m papermill --version displayName: "Sanity check" - - bash: python .actions/assistant.py convert-ipynb $(notebook) + - bash: python .actions/assistant.py convert-ipynb --folder=$(notebook) displayName: "Generate notebook" timeoutInMinutes: "5" - bash: | set -e mkdir $(PATH_DATASETS) - python .actions/assistant.py bash-render $(notebook) + python .actions/assistant.py bash-render --folder=$(notebook) cat .actions/_ipynb-render.sh bash .actions/_ipynb-render.sh git status diff --git a/.azure/ipynb-validate.yml b/.azure/ipynb-validate.yml index 8f9f3c081..276de239f 100644 --- a/.azure/ipynb-validate.yml +++ b/.azure/ipynb-validate.yml @@ -85,6 +85,7 @@ jobs: set -e pip --version pip install -r requirements.txt + pip install $(python .actions/assistant.py pip-install --folder=$(notebook)) pip list displayName: "Install dependencies" @@ -94,13 +95,13 @@ jobs: python -c "import torch ; mgpu = torch.cuda.device_count() ; assert mgpu > 0, f'GPU: {mgpu}'" displayName: "Sanity check" - - bash: python .actions/assistant.py convert-ipynb $(notebook) + - bash: python .actions/assistant.py convert-ipynb --folder=$(notebook) displayName: "Generate notebook" - bash: | set -e mkdir $(PATH_DATASETS) - python .actions/assistant.py bash-validate $(notebook) + python .actions/assistant.py bash-validate --folder=$(notebook) cat .actions/_ipynb-validate.sh bash .actions/_ipynb-validate.sh env: From bebf9cbdad6788026db820ee861b03d6004a7106 Mon Sep 17 00:00:00 2001 From: jirka Date: Fri, 26 Jul 2024 10:13:32 +0200 Subject: [PATCH 2/2] update --- .actions/assistant.py | 22 +++++++++++++++------- .azure/ipynb-publish.yml | 5 +++-- .azure/ipynb-validate.yml | 4 ++-- 3 files changed, 20 insertions(+), 11 deletions(-) diff --git a/.actions/assistant.py b/.actions/assistant.py index 770ddce2d..db6ba97a8 100644 --- a/.actions/assistant.py +++ b/.actions/assistant.py @@ -8,7 +8,7 @@ from datetime import datetime from shutil import copyfile from textwrap import wrap -from typing import Any, Dict, List, Optional, Sequence, Tuple +from typing import Any, Dict, List, Optional, Sequence, Tuple, Union from warnings import warn import fire @@ -283,11 +283,12 @@ def _valid_accelerator(folder: str) -> bool: return any(ac in meta_accels for ac in device_accels) @staticmethod - def _parse_requirements(folder: str) -> Tuple[str, str]: + def _parse_requirements(folder: str, formatted: bool = True) -> Union[Tuple[str, str], Tuple[list, list]]: """Parse standard requirements from meta file. Args: folder: path to the folder with python script, meta and artefacts + formatted: format it into two strings """ meta = AssistantCLI._load_meta(folder) @@ -298,20 +299,27 @@ def _parse_requirements(folder: str) -> Tuple[str, str]: for k, v in meta.items() if k.startswith(AssistantCLI._META_PIP_KEY) } - pip_args = ['--extra-index-url="https://download.pytorch.org/whl/"' + _RUNTIME_VERSIONS.get("DEVICE")] + pip_args = [f'--extra-index-url="https://download.pytorch.org/whl/{_RUNTIME_VERSIONS.get("DEVICE")}"'] for pip_key in meta_pip_args: if not isinstance(meta_pip_args[pip_key], (list, tuple, set)): meta_pip_args[pip_key] = [meta_pip_args[pip_key]] for arg in meta_pip_args[pip_key]: arg = arg % _RUNTIME_VERSIONS pip_args.append(f"--{pip_key} {arg}") - - return " ".join([f'"{req}"' for req in requires]), " ".join(pip_args) + if formatted: + return " ".join([f'"{req}"' for req in requires]), " ".join(pip_args) + return list(requires), pip_args @staticmethod def pip_install(folder: str) -> str: - """Parse all notebook requirements to be pre-installed.""" - return " ".join(AssistantCLI._parse_requirements(folder)) + """Print all notebook requirements to be pre-installed in format of requirements file. + + Args: + folder: path to the folder with python script, meta and artefacts + + """ + req, args = AssistantCLI._parse_requirements(folder, formatted=False) + return os.linesep.join(req) + os.linesep + os.linesep.join(args) @staticmethod def _bash_download_data(folder: str) -> List[str]: diff --git a/.azure/ipynb-publish.yml b/.azure/ipynb-publish.yml index 6f1bdfe40..4002e702e 100644 --- a/.azure/ipynb-publish.yml +++ b/.azure/ipynb-publish.yml @@ -145,8 +145,9 @@ jobs: pip --version # todo: adjust torch ecosystem versions pip install -r requirements.txt - # todo: export requirements for notebooks to file and execute - pip install $(python .actions/assistant.py pip-install --folder=$(notebook)) + # export requirements for notebooks to file and execute + python .actions/assistant.py pip-install --folder=$(notebook) > notebook.txt + pip install -r notebook.txt displayName: "Install dependencies" timeoutInMinutes: "15" diff --git a/.azure/ipynb-validate.yml b/.azure/ipynb-validate.yml index 276de239f..b91d8b5d3 100644 --- a/.azure/ipynb-validate.yml +++ b/.azure/ipynb-validate.yml @@ -85,8 +85,8 @@ jobs: set -e pip --version pip install -r requirements.txt - pip install $(python .actions/assistant.py pip-install --folder=$(notebook)) - pip list + python .actions/assistant.py pip-install --folder=$(notebook) > notebook.txt + pip install -r notebook.txt displayName: "Install dependencies" - bash: |