Merge branch 'main' into docs

.actions/assistant.py

@@ -113,7 +113,7 @@ def get_running_cuda_version() -> str:
         return ""
 
 
-def get_running_torch_version():
+def get_running_torch_version() -> str:
     """Extract the version of actual PyTorch for this runtime."""
     try:
         import torch
@@ -154,7 +154,7 @@ class AssistantCLI:
         "lightning_examples": "Lightning-Examples",
         "flash_tutorials": "Kaggle",
     }
-    _BASH_SCRIPT_BASE = ("#!/bin/bash", "set -e", "")
+    _BASH_SCRIPT_BASE = ("#!/bin/bash", "set -ex", "")
     _EXT_ARCHIVE_ZIP = (".zip",)
     _EXT_ARCHIVE_TAR = (".tar", ".gz")
     _EXT_ARCHIVE = _EXT_ARCHIVE_ZIP + _EXT_ARCHIVE_TAR
@@ -322,7 +322,13 @@ def bash_render(folder: str, output_file: str = PATH_SCRIPT_RENDER) -> Optional[
             # dry run does not execute the notebooks just takes them as they are
             cmd.append(f"cp {ipynb_file} {pub_ipynb}")
             # copy and add meta config
-            cmd += [f"cp {meta_file} {pub_meta}", f"cat {pub_meta}", f"git add {pub_meta}"]
+            cmd += [
+                f"cp {meta_file} {pub_meta}",
+                'echo "#====== START OF YAML FILE ======#"',
+                f"cat {pub_meta}",
+                'echo "#======= END OF YAML FILE =======#"',
+                f"git add {pub_meta}",
+            ]
         else:
             pip_req, pip_args = AssistantCLI._parse_requirements(folder)
             cmd += [f"pip install {pip_req} --quiet {pip_args}", "pip list"]
@@ -335,7 +341,13 @@ def bash_render(folder: str, output_file: str = PATH_SCRIPT_RENDER) -> Optional[
             # Export the actual packages used in runtime
             cmd.append(f"meta_file=$(python .actions/assistant.py update-env-details {folder})")
             # copy and add to version the enriched meta config
-            cmd += ["echo $meta_file", "cat $meta_file", "git add $meta_file"]
+            cmd += [
+                "echo $meta_file",
+                'echo "#====== START OF YAML FILE ======#"',
+                "cat $meta_file",
+                'echo "#======= END OF YAML FILE =======#"',
+                "git add $meta_file",
+            ]
         # if thumb image is linked to the notebook, copy and version it too
         if thumb_file:
             cmd += [f"cp {thumb_file} {pub_thumb}", f"git add {pub_thumb}"]
@@ -347,7 +359,7 @@ def bash_render(folder: str, output_file: str = PATH_SCRIPT_RENDER) -> Optional[
             fopen.write(os.linesep.join(cmd))
 
     @staticmethod
-    def bash_test(folder: str, output_file: str = PATH_SCRIPT_TEST) -> Optional[str]:
+    def bash_test(folder: str, output_file: str = PATH_SCRIPT_TEST, virtualenv: bool = False) -> Optional[str]:
         """Prepare bash script for running tests of a particular notebook.
 
         Args:
@@ -364,11 +376,12 @@ def bash_test(folder: str, output_file: str = PATH_SCRIPT_TEST) -> Optional[str]
 
         # prepare isolated environment with inheriting the global packages
         path_venv = os.path.join(folder, "venv")
-        cmd += [
-            f"python -m virtualenv --system-site-packages {path_venv}",
-            f"source {os.path.join(path_venv, 'bin', 'activate')}",
-            "pip --version",
-        ]
+        if virtualenv:
+            cmd += [
+                f"python -m virtualenv --system-site-packages {path_venv}",
+                f"source {os.path.join(path_venv, 'bin', 'activate')}",
+                "pip --version",
+            ]
 
         cmd.append(f"# available: {AssistantCLI.DEVICE_ACCELERATOR}")
         if AssistantCLI._valid_accelerator(folder):
@@ -378,21 +391,30 @@ def bash_test(folder: str, output_file: str = PATH_SCRIPT_TEST) -> Optional[str]
             # Export the actual packages used in runtime
             cmd.append(f"meta_file=$(python .actions/assistant.py update-env-details {folder} --base_path .)")
             # show created meta config
-            cmd += ["echo $meta_file", "cat $meta_file"]
-            cmd.append(f"python -m pytest {ipynb_file} -v --nbval --nbval-cell-timeout=300")
+            cmd += [
+                "echo $meta_file",
+                'echo "#====== START OF YAML FILE ======#"',
+                "cat $meta_file",
+                'echo "#======= END OF YAML FILE =======#"',
+            ]
+            # use standard jupyter's executable via CMD
+            cmd.append(f"jupyter execute {ipynb_file} --inplace")
         else:
             pub_ipynb = os.path.join(DIR_NOTEBOOKS, f"{folder}.ipynb")
             pub_meta = pub_ipynb.replace(".ipynb", ".yaml")
             # copy and add meta config
             cmd += [
                 f"mkdir -p {os.path.dirname(pub_meta)}",
                 f"cp {meta_file} {pub_meta}",
+                'echo "#====== START OF YAML FILE ======#"',
                 f"cat {pub_meta}",
+                'echo "#======= END OF YAML FILE =======#"',
                 f"git add {pub_meta}",
             ]
             warn("Invalid notebook's accelerator for this device. So no tests will be run!!!", RuntimeWarning)
         # deactivate and clean local environment
-        cmd += ["deactivate", f"rm -rf {os.path.join(folder, 'venv')}"]
+        if virtualenv:
+            cmd += ["deactivate", f"rm -rf {os.path.join(folder, 'venv')}"]
         if not output_file:
             return os.linesep.join(cmd)
         with open(output_file, "w") as fopen:
@@ -504,15 +526,15 @@ def group_folders(
         with open(fpath_gitdiff) as fopen:
             changed = [ln.strip() for ln in fopen.readlines()]
         dirs = [os.path.dirname(ln) for ln in changed]
-        # not empty paths
-        dirs = [ln for ln in dirs if ln]
 
         if fpath_actual_dirs:
             assert isinstance(fpath_actual_dirs, list)
             assert all(os.path.isfile(p) for p in fpath_actual_dirs)
             dir_sets = [{ln.strip() for ln in open(fp).readlines()} for fp in fpath_actual_dirs]
             # get only different
             dirs += list(set.union(*dir_sets) - set.intersection(*dir_sets))
+        # not empty paths
+        dirs = [ln for ln in dirs if ln]
 
         if root_path:
             dirs = [os.path.join(root_path, d) for d in dirs]
@@ -626,6 +648,7 @@ def copy_notebooks(
         path_docs_images: str = "_static/images",
         patterns: Sequence[str] = (".", "**"),
         ignore: Optional[Sequence[str]] = None,
+        strict: bool = True,
     ) -> None:
         """Copy all notebooks from a folder to doc folder.
 
@@ -636,12 +659,16 @@ def copy_notebooks(
             path_docs_images: destination path to the images' location relative to ``docs_root``
             patterns: patterns to use when glob-ing notebooks
             ignore: ignore some specific notebooks even when the given string is in path
+            strict: raise exception if copy fails
 
         """
-        all_ipynb = []
-        for pattern in patterns:
-            all_ipynb += glob.glob(os.path.join(path_root, DIR_NOTEBOOKS, pattern, "*.ipynb"))
         os.makedirs(os.path.join(docs_root, path_docs_ipynb), exist_ok=True)
+        all_ipynb = [
+            os.path.realpath(ipynb)
+            for pattern in patterns
+            for ipynb in glob.glob(os.path.join(path_root, DIR_NOTEBOOKS, pattern, "*.ipynb"))
+        ]
+        print(f"Copy following notebooks to docs folder: {all_ipynb}")
         if ignore and not isinstance(ignore, (list, set, tuple)):
             ignore = [ignore]
         elif not ignore:
@@ -661,8 +688,11 @@ def copy_notebooks(
                     path_docs_images=path_docs_images,
                 )
             except Exception as ex:
-                warnings.warn(f"Failed to copy notebook: {path_ipynb}\n{ex}", ResourceWarning)
-                continue
+                msg = f"Failed to copy notebook: {path_ipynb}\n{ex}"
+                if not strict:
+                    warnings.warn(msg, ResourceWarning)
+                    continue
+                raise FileNotFoundError(msg)
             ipynb_content.append(os.path.join(path_docs_ipynb, path_ipynb_in_dir))
 
     @staticmethod
@@ -707,7 +737,10 @@ def update_env_details(folder: str, base_path: str = DIR_NOTEBOOKS) -> str:
 
         Args:
              folder: path to the folder
-             base_path:
+             base_path: base path with notebooks
+
+        Returns:
+            path the updated YAML file
 
         """
         meta = AssistantCLI._load_meta(folder)

.actions/git-diff-sync.sh

@@ -1,13 +1,12 @@
 #!/bin/bash
 
-set -e
+set -ex
 printf "Detect changes for: $1 >> $2\n\n"
 
 b1="${1//'/'/'_'}"
-printf "Branch alias: $b1\n"
 # list all dirs in source branch
 python .actions/assistant.py list_dirs > "dirs-$b1.txt"
-cat "dirs-$b1.txt"
+printf "Branch alias: $b1\n" && cat "dirs-$b1.txt"
 
 head=$(git rev-parse origin/$2)
 git diff --name-only $head --output=target-diff.txt
@@ -19,19 +18,15 @@ cp -r .actions/ _TEMP/.actions/
 
 git checkout $2
 b2="${2//'/'/'_'}"
-printf "Branch alias: $b2\n"
 # recover the original CLI
 #rm -rf .actions && mv _TEMP/.actions .actions
 # list all dirs in target branch
 python _TEMP/.actions/assistant.py list_dirs ".notebooks" --include_file_ext=".ipynb"  > "dirs-$b2.txt"
-cat "dirs-$b2.txt"
+printf "Branch alias: $b2\n" && cat "dirs-$b2.txt"
 
 printf "\n\n"
 git merge --ff -s resolve origin/$1
 
 python _TEMP/.actions/assistant.py group-folders target-diff.txt --fpath_actual_dirs "['dirs-$b1.txt', 'dirs-$b2.txt']"
-printf "\n\nChanged folders:\n"
-cat changed-folders.txt
-printf "\n\nDropped folders:\n"
-cat dropped-folders.txt
-printf "\n"
+printf "\n================\nChanged folders:\n----------------\n" && cat changed-folders.txt
+printf "\n================\nDropped folders:\n----------------\n" && cat dropped-folders.txt

.actions/requires.txt

@@ -1,6 +1,6 @@
 Fire
 tqdm
-PyYAML
+PyYAML <5.4  # todo: racing issue with cython compile
 wcmatch
 requests
 pip

.azure/ipynb-publish.yml

@@ -12,6 +12,8 @@ jobs:
   - job: sync_pub
     pool:
       vmImage: "Ubuntu-20.04"
+    workspace:
+      clean: all
     variables:
       ACCELERATOR: CPU,GPU
       PUB_BRANCH: publication
@@ -24,6 +26,7 @@ jobs:
           printf "commit hash:\n $(COMMIT_HASH)\n"
           printf "commit message:\n $(COMMIT_MSG)\n"
         displayName: "Set Git user"
+        timeoutInMinutes: "5"
       - bash: |
           set -e
           git fetch --all
@@ -36,7 +39,7 @@ jobs:
         displayName: "Install dependencies"
       - bash: |
           current_branch=$(cut -d '/' -f3- <<< $(Build.SourceBranch))
-          printf "$current_branch\n"
+          printf "Current branch: $current_branch\n"
           bash .actions/git-diff-sync.sh $current_branch $(PUB_BRANCH)
         displayName: "Compare changes & sync"
 
@@ -58,15 +61,19 @@ jobs:
           git status
           git commit -m "prune: $(COMMIT_HASH)"
         condition: gt(variables['dropped.folders'], 0)
-        displayName: "Prune notebook"
+        displayName: "Prune notebooks"
+        timeoutInMinutes: "10"
 
       - bash: |
+          set -ex
           git status
           git push https://$(PAT_GHOST)@github.com/Lightning-AI/tutorials.git $(PUB_BRANCH)
-        displayName: "Finish push"
+        displayName: "Finish and push"
 
   - job: papermill
     dependsOn: sync_pub
+    # run if the initial job succeeded and the strategy matrix is not empty
+    condition: and(succeeded(), ne(dependencies.sync_pub.outputs['mtrx.dirs'], ''))
     strategy:
       # generated matrix with changed notebooks, include fields: "notebook", "agent-pool" and "docker-image"
       matrix: $[ dependencies.sync_pub.outputs['mtrx.dirs'] ]
@@ -85,6 +92,8 @@ jobs:
     container:
       image: $(docker-image)
       options: "--gpus=all --shm-size=32g -v /usr/bin/docker:/tmp/docker:ro"
+    workspace:
+      clean: all
 
     variables:
       ACCELERATOR: CPU,GPU
@@ -94,8 +103,6 @@ jobs:
       COMMIT_HASH: "$(Build.SourceVersion)"
       DEVICES: $( python -c 'print("$(Agent.Name)".split("_")[-1])' )
 
-    condition: ne(dependencies.sync_pub.outputs['mtrx.dirs'], '')
-
     steps:
       - bash: |
           echo "##vso[task.setvariable variable=CUDA_VISIBLE_DEVICES]$(DEVICES)"
@@ -115,9 +122,11 @@ jobs:
       - bash: |
           git config --global user.email "[email protected]"
           git config --global user.name "Azure Pipelines"
+          git config --global pull.ff only
           printf "commit hash:\n $(COMMIT_HASH)\n"
           printf "commit message:\n $(COMMIT_MSG)\n"
         displayName: "Set Git user"
+        timeoutInMinutes: "5"
       - bash: |
           set -e
           git fetch --all
@@ -128,6 +137,7 @@ jobs:
           git show-ref $(PUB_BRANCH)
           git pull
         displayName: "Git check & switch branch"
+        timeoutInMinutes: "5"
 
       - bash: |
           set -e
@@ -136,6 +146,7 @@ jobs:
           # todo: adjust torch ecosystem versions
           pip install -r requirements.txt -r _requirements/data.txt
         displayName: "Install dependencies"
+        timeoutInMinutes: "15"
 
       - bash: |
           set -e
@@ -146,6 +157,7 @@ jobs:
 
       - bash: python .actions/assistant.py convert-ipynb $(notebook)
         displayName: "Generate notebook"
+        timeoutInMinutes: "5"
 
       - bash: |
           set -e
@@ -161,7 +173,8 @@ jobs:
         displayName: "Render notebook"
 
       - bash: |
+          set -ex
           git status
           git show-ref $(PUB_BRANCH)
           git push https://$(PAT_GHOST)@github.com/Lightning-AI/tutorials.git $(PUB_BRANCH)
-        displayName: "Finish push"
+        displayName: "Finish and push"