Adds Jupyter notebooks and MATLAB live scripts to the examples (#30)

* rearranged examples to make way for python

* script for getting tutorials

* fixed matlab images and made setup tutorial simpler

* split tutorials folder into two

* split tutorials folder into two

* added livescripts to matlab examples

* added nbsphinx to requirements and install pandoc in CI

* removed python-rat data from repo

* made notebooks part of build process

* fix ci

* added prolog to notebooks

* matlab live scripts now have outputs

* updated readme, fixed typo, added examples to header

* update notebooks readme

* tidied up header

* complete rebase (whoops!)

* review fixes

.github/workflows/build_docs.yml

@@ -21,8 +21,17 @@ jobs:
         python-version: '3.11' 
     - name: Build docs
       run: |
+        # create an x11 display because otherwise MATLAB refuses to export live scripts...
+        sudo apt install -y xvfb
+        export DISPLAY=':99.0'
+        Xvfb :99 -screen 0 1024x768x24 > /dev/null 2>&1 &
+
         wget https://github.com/RascalSoftware/RAT/releases/download/nightly/Linux.zip
         unzip Linux.zip -d API/
+
+        # we get pandoc from web as apt version is outdated
+        wget https://github.com/jgm/pandoc/releases/download/3.6.2/pandoc-3.6.2-1-amd64.deb
+        sudo apt install -y ./pandoc-3.6.2-1-amd64.deb
         python -m pip install --upgrade pip
         pip install matlabengine==24.1.*
         pip install -r requirements.txt

.gitignore

@@ -19,3 +19,7 @@ source/_outputs/
 
 # autogenerated documentation
 source/reference/python/RATapi.*
+source/python_examples/data/
+source/matlab_examples/*.html
+source/python_examples/notebooks/*
+!source/python_examples/notebooks/README.md

README.md

@@ -17,6 +17,8 @@ Download the appropriate version of RAT from the GitHub [release](https://github
     wget https://github.com/RascalSoftware/RAT/releases/download/nightly/Linux.zip
     unzip Linux.zip -d API/
 
+You also must have `pandoc` installed to build the Python example Jupyter notebooks. See the installation instructions [here](https://pandoc.org/installing.html).
+
 
 To build the HTML docs, type the following into a terminal with access to the Python executable:  
 

requirements.txt

@@ -3,6 +3,8 @@ sphinxcontrib-matlabdomain==0.18
 pydata-sphinx-theme==0.15.2
 sphinx_design==0.6.0
 sphinx-copybutton==0.5.2
+jupyter==1.0.0
+nbsphinx==0.9.6
 RATapi==0.0.0.dev4
 autodoc_pydantic==2.2.0
 enum-tools[sphinx]==0.12.0

source/conf.py

@@ -7,8 +7,13 @@
 # documentation root, use os.path.abspath to make it absolute, like shown here.
 import os
 import sys
+import shutil
 import datetime
+import zipfile
+from importlib import metadata
 from urllib.parse import urljoin
+from urllib.request import urlretrieve
+from pathlib import Path
 # -- Project information -----------------------------------------------------
 # Add any Sphinx extension module names here, as strings. They can be
 # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
@@ -33,17 +38,60 @@
 sys.path.insert(0, os.path.dirname(os.path.abspath("..")))
 from version import get_doc_version
 doc_version = get_doc_version()
-    
+
 # -- General configuration ---------------------------------------------------
 
 # add extensions path for snippets
 sys.path.append(os.path.abspath("./_ext"))
 
-extensions = ['sphinxcontrib.matlab', 'sphinx.ext.napoleon', 'sphinx.ext.autodoc', 'sphinx.ext.autosummary', 'sphinxcontrib.autodoc_pydantic', 'sphinx_design', 'sphinx_copybutton', 'snippets', 'enum_tools.autoenum']
+extensions = ['sphinxcontrib.matlab', 'sphinx.ext.napoleon', 'sphinx.ext.autodoc', 'sphinx.ext.autosummary', 'sphinxcontrib.autodoc_pydantic', 'sphinx_design', 'sphinx_copybutton', 'snippets', 'enum_tools.autoenum', 'nbsphinx']
 
 # Add any paths that contain templates here, relative to this directory.
 templates_path = ['_templates']
 
+# -- Setup example files -----------------------------------------------------
+PYTHON_RAT_RELEASE = metadata.version("RATapi")
+
+if not os.path.isdir("./python_examples/data"):
+    zip_dir, headers = urlretrieve(f"https://github.com/RascalSoftware/python-RAT/archive/refs/tags/{PYTHON_RAT_RELEASE}.zip")
+    with zipfile.ZipFile(zip_dir) as zf:
+        zf.extractall()
+    print("Copying Jupyter notebooks...")
+    for directory in ['normal_reflectivity', 'domains', 'absorption']:
+        for file in Path(f"./python-RAT-{PYTHON_RAT_RELEASE}/RATapi/examples/{directory}/").glob('*'):
+            shutil.copy(file, "./python_examples/notebooks/")
+
+    shutil.copytree(f"./python-RAT-{PYTHON_RAT_RELEASE}/RATapi/examples/data", "./python_examples/data", dirs_exist_ok=True)
+
+    shutil.rmtree(f"./python-RAT-{PYTHON_RAT_RELEASE}")
+
+if not os.path.isfile("./matlab_examples/standardLayersDSPCSheet.html"):
+    try:
+        from matlab.engine import start_matlab
+    except ImportError:
+        print("Could not copy MATLAB live scripts as MATLAB is not installed.")
+    else:
+        print("Starting MATLAB Engine...")
+        eng = start_matlab()
+        matlab_examples_path = Path("./matlab_examples").resolve()
+        eng.eval("cd('../API'); addPaths;", nargout=0)
+        for sheet in ['normalReflectivity/standardLayers/standardLayersDSPCSheet', 
+                      'normalReflectivity/customLayers/customLayersDSPCSheet', 
+                      'normalReflectivity/customXY/customXYDSPCSheet', 
+                      'domains/standardLayers/domainsStandardLayersSheet', 
+                      'domains/customLayers/domainsCustomLayersSheet',
+                      'domains/customXY/domainsCustomXYSheet',
+                      'miscellaneous/convertRascal1Project/convertRascal',
+                      'miscellaneous/alternativeLanguages/customModelLanguagesSheet',]:
+            filename = Path(sheet).name
+            folder = str(Path(sheet).parent)
+            print(f"exporting {sheet}")
+            eng.cd(f"examples/{folder}", nargout=0)
+            eng.matlab.internal.liveeditor.executeAndSave(str(Path(f"../API/examples/{sheet}.mlx").resolve()), nargout=0)
+            eng.export(f"{filename}.mlx", str(matlab_examples_path / f"{filename}.html"), nargout=0)
+            eng.cd("../../../", nargout=0)
+
+
 # -- Options for HTML output -------------------------------------------------
 #set primary_domain = 'matlab'
 primary_domain = None
@@ -84,6 +132,30 @@
 
 autodoc_typehints = "description"
 
+nbsphinx_prolog = r"""
+{% set docname = 'doc/' + env.doc2path(env.docname, base=None)|string %}
+
+.. raw:: html
+
+    <div class="admonition note">
+      This page was generated from the notebook {{ env.docname.split('/')|last|e + '.ipynb' }} found in 
+      <a class="reference external" href="https://github.com/RascalSoftware/python-RAT/blob/"""+PYTHON_RAT_RELEASE+r"""/RATapi/examples/">the Python-RAT repository</a>.
+      <a href="{{ env.docname.split('/')|last|e + '.ipynb' }}" class="reference download internal" download>Download notebook</a>.
+    </div>
+
+.. note::
+
+    To get the output project and results from this example in your Python session, run:
+
+    .. code-block:: python
+
+        from RATapi.examples import {{ env.docname.split('/')|last|e }}
+        project, results = {{ env.docname.split('/')|last|e }}() 
+
+-------------------------------------------------------------------------------------
+
+"""
+
 ### autodoc_pydantic settings
 # hide JSON schemas by default
 autodoc_pydantic_model_show_json = False

source/examples.rst

@@ -0,0 +1,21 @@
+Examples
+========
+
+Several examples are provided for RAT in both Python and MATLAB:
+
+
+Matlab
+------
+
+.. toctree::
+   :maxdepth: 3
+
+   matlab_examples/index
+
+Python
+------
+
+.. toctree::
+   :maxdepth: 3
+
+   python_examples/index