From 044c31fbab638d0725db6314774bc1fdb4674d85 Mon Sep 17 00:00:00 2001 From: David Feltell Date: Fri, 30 Aug 2024 18:58:55 +0100 Subject: [PATCH] [Docs] Add hybrid plugin system notebook Part of OpenAssetIO/OpenAssetIO#1202. Add a Jupyter Notebook illustrating the design and usage of the hybrid plugin system. Update CI to ensure SimpleCppManager is built with OpenAssetIO, and use a dedicated job for the hybrid plugin system notebook to supply the built SimpleCppManager. Bump the minimum versions of OpenAssetIO and BAL in `requirements.txt` to support the required features for the notebook to run. Signed-off-by: David Feltell --- .github/build_openassetio/action.yml | 2 +- .github/workflows/examples.yml | 39 +- examples/hybrid_plugin_system.ipynb | 396 ++++++++++++++++++ .../SimpleCppManager/README.md | 10 + .../hybrid_plugin_system/bal_database.json | 42 ++ .../openassetio_config.toml | 44 ++ examples/resources/requirements.txt | 4 +- 7 files changed, 533 insertions(+), 4 deletions(-) create mode 100644 examples/hybrid_plugin_system.ipynb create mode 100644 examples/resources/hybrid_plugin_system/SimpleCppManager/README.md create mode 100644 examples/resources/hybrid_plugin_system/bal_database.json create mode 100644 examples/resources/hybrid_plugin_system/openassetio_config.toml diff --git a/.github/build_openassetio/action.yml b/.github/build_openassetio/action.yml index 0e34aa6..5f28b2c 100644 --- a/.github/build_openassetio/action.yml +++ b/.github/build_openassetio/action.yml @@ -22,7 +22,7 @@ runs: run: | cd openassetio-checkout mkdir build - cmake -G Ninja -S . -B build + cmake -G Ninja -S . -B build -DOPENASSETIO_ENABLE_SIMPLECPPMANAGER=ON cmake --build build cmake --install build - uses: actions/upload-artifact@v3 diff --git a/.github/workflows/examples.yml b/.github/workflows/examples.yml index 23777ca..3eb670e 100644 --- a/.github/workflows/examples.yml +++ b/.github/workflows/examples.yml @@ -18,6 +18,9 @@ jobs: matrix: os: ["windows-2022", "ubuntu-22.04", "macos-13"] python: ["3.10", "3.11"] + defaults: + run: + shell: bash steps: - uses: actions/checkout@v3 - uses: actions/setup-python@v4 @@ -28,4 +31,38 @@ jobs: python -m pip install . python -m pip install -r examples/resources/requirements.txt - name: Test Notebooks - run: jupyter nbconvert --to html --execute examples/*.ipynb + # Execute all the notebooks apart from the Hybrid Plugin System, + # which requires a build of SimpleCppManager + run: > + find examples -maxdepth 1 -name "*.ipynb" + ! -name "hybrid_plugin_system.ipynb" + -exec jupyter nbconvert --to html --execute {} \; + + test-cpp-notebooks: + # A special job just for the Hybrid Plugin System because it needs a + # build of SimpleCppManager. Use the OpenAssetIO Docker container to + # get a build. If/when OpenAssetIO publishes SimpleCppManager with + # its release artifacts, this could be simplified. + name: Test Hybrid Plugin System notebook + runs-on: ubuntu-latest + container: + image: ghcr.io/openassetio/openassetio-build + steps: + - uses: actions/checkout@v3 + - name: Install Dependencies + run: | + python -m pip install . + python -m pip install -r examples/resources/requirements.txt + + - name: Build SimpleCppManager + uses: ./.github/build_openassetio + + - name: Copy SimpleCppManager to expected location for notebook + run: > + cp openassetio-checkout/build/dist/SimpleCppManager.so + examples/resources/hybrid_plugin_system/SimpleCppManager/ + + - name: Test notebook + run: jupyter nbconvert --to html --execute examples/hybrid_plugin_system.ipynb + + diff --git a/examples/hybrid_plugin_system.ipynb b/examples/hybrid_plugin_system.ipynb new file mode 100644 index 0000000..54166a6 --- /dev/null +++ b/examples/hybrid_plugin_system.ipynb @@ -0,0 +1,396 @@ +{ + "cells": [ + { + "metadata": {}, + "cell_type": "markdown", + "source": [ + "# Hybrid Plugin System\n", + "\n", + "This notebook illustrates usage of the Hybrid Plugin System. We combine a Python manager plugin and a C++ manager plugin, and dispatch to the appropriate plugin based on its capabilities and priority. \n", + "\n", + "The primary use-case for this feature is to allow performance critical functionality to be written in performant C++, whilst less performance critical functionality can be written in more flexible Python.\n", + "\n", + "The hybrid plugin system is not limited to this use-case, however. You are free to combine any number of plugins in any supported language (only C++ and Python are available in the core library at time of writing). The hybrid plugin system also provides a more convenient abstraction for working with multiple plugin systems in general.\n", + "\n", + "## How it works\n", + "\n", + "OpenAssetIO manager plugins must advertise a unique identifier. An OpenAssetIO plugin system (e.g. C++ or Python) maps unique identifiers to a plugin instance. A consequence of this is if multiple plugins advertise the same unique identifier, only one of those plugins can be chosen _by that plugin system_. However, if multiple plugin systems are in use,\n", + "then each plugin system has their own mapping of unique identifier to plugin instance. This means there _can_ be multiple plugins with the same identifier, as long as they are discovered by different plugin systems.\n", + "\n", + "As a motivating example, a manager plugin developed in Python, loaded by the Python plugin system, could advertise the same identifier as a manager plugin developed in C++, loaded by the C++ plugin system.\n", + "\n", + "This is the essence of how the hybrid plugin system discovers plugins. If two (or more) plugins from two (or more) different plugin systems advertise the same unique identifier, then we assume they are related and can be composed.\n", + "\n", + "The hybrid plugin system therefore wraps a list of child plugin systems, such that they present as a single plugin system to the host.\n", + "\n", + "Given that the hybrid plugin system has discovered two (or more) composable plugins, we then need a mechanism to dispatch API calls to the appropriate plugin. I.e. we need a way to choose which plugin is the best to use for a particular API call. This is where the `hasCapability` API method comes in.\n", + "\n", + "OpenAssetIO API methods are grouped under \"capabilities\", e.g. `\"resolution\"`, `\"publishing\"`, `\"relationshipQueries\"`, etc (these are stringified representations of the `Capability` enumeration). A manager plugin advertises which capabilities it supports by overriding the `hasCapability` method of the base `ManagerInterface` class.\n", + "\n", + "Therefore, we can dispatch an API call to the appropriate manager plugin by finding the plugin that advertises the associated capability for that API call.\n", + "\n", + "If multiple managers advertise that they support the required capability, then which plugin to use is determined by the original order that the child plugin systems were provided to the hybrid plugin system. For example, if the hybrid plugin system was constructed with a list containing the C++ plugin system followed by the Python plugin system, and C++ and Python plugins have been discovered and composed, and both advertise that they support the required capability for a particular API call, then the C++ plugin will be chosen for that API call.\n", + "\n", + "All the OpenAssetIO _required_ capabilities (i.e. `\"entityReferenceIdentification\"`, `\"managementPolicyQueries\"`, `\"entityTraitIntrospection\"`) must be satisfied by at least one of the composed plugins.\n", + "\n", + "If only one child factory locates a plugin with the desired identifier, then that plugin is used directly (i.e. the plugin is not wrapped). In this way, host applications making use of the hybrid plugin system don't lose out on any functionality or performance. \n", + "\n", + "This leads to an important point: _the hybrid plugin system should be used by default by most hosts_, even if they don't make use of all of its functionality." + ], + "id": "6ebc020b4c31f1b8" + }, + { + "metadata": {}, + "cell_type": "markdown", + "source": "## Example", + "id": "959ede09b6700a1e" + }, + { + "metadata": {}, + "cell_type": "markdown", + "source": [ + "### Preamble\n", + "\n", + "First lets get some boilerplate out of the way. See the \"Hello OpenAssetIO\" notebook for more details." + ], + "id": "5b0c4ae1f1b914e6" + }, + { + "metadata": { + "ExecuteTime": { + "end_time": "2024-08-30T17:20:42.358486Z", + "start_time": "2024-08-30T17:20:42.353611Z" + } + }, + "cell_type": "code", + "source": [ + "import os\n", + "\n", + "\n", + "try:\n", + " import openassetio\n", + " import openassetio_mediacreation\n", + "except ImportError:\n", + " print(\n", + " \"This notebook requires the packages listed in `resources/requirements.txt` to be installed\")\n", + " raise\n", + "\n", + "from resources import helpers\n", + "\n", + "from openassetio.hostApi import HostInterface, ManagerFactory\n", + "from openassetio.log import ConsoleLogger, SeverityFilter\n", + "\n", + "\n", + "class NotebookHostInterface(HostInterface):\n", + " def identifier(self):\n", + " return \"org.jupyter.notebook\"\n", + "\n", + " def displayName(self):\n", + " return \"Jupyter Notebook\"\n", + "\n", + "\n", + "host_interface = NotebookHostInterface()\n", + "\n", + "logger = SeverityFilter(ConsoleLogger())" + ], + "id": "db2739c3f0a96d70", + "outputs": [], + "execution_count": 16 + }, + { + "metadata": {}, + "cell_type": "markdown", + "source": [ + "\n", + "### The two example plugins\n", + "\n", + "In order to illustrate the hybrid plugin system, we'll make use of two ready-made example plugins - the \"Basic Asset Libary\" (aka BAL), a pure Python manager/plugin, and the \"Simple C++ Manager\" (aka SimpleCppManager), a pure C++ manager/plugin. \n", + "\n", + "BAL should be installed into the Python environment of this notebook (see `resources/requirements.txt`), and so will be trivially discoverable by OpenAssetIO.\n", + "\n", + "SimpleCppManager is more complex, and must be built with a compiler toolchain compatible with the OpenAssetIO libraries in the Python environment of this notebook. See `resources/hybrid_plugin_system/SimpleCppManager/README.md` for more details. We assume it is installed into `resources/hybrid_plugin_system/SimpleCppManager`, and will be discovered by adding this location to the standard `OPENASSETIO_PLUGIN_PATH` environment variable." + ], + "id": "791f52d06eea9aea" + }, + { + "metadata": { + "ExecuteTime": { + "end_time": "2024-08-30T17:20:42.410841Z", + "start_time": "2024-08-30T17:20:42.407963Z" + } + }, + "cell_type": "code", + "source": [ + "os.environ[\"OPENASSETIO_PLUGIN_PATH\"] = os.path.join(\n", + " \"resources\", \"hybrid_plugin_system\", \"SimpleCppManager\")" + ], + "id": "e3a94c5be66ad34e", + "outputs": [], + "execution_count": 17 + }, + { + "metadata": {}, + "cell_type": "markdown", + "source": [ + "\n", + "\n", + "These two plugins each advertise a different unique identifier, so will not work with the hybrid plugin system out of the box. Luckily, they are both designed to be used in testing situations and are fully configurable. We can control the identifier that they will advertise using environment variables:\n" + ], + "id": "e19f60f60b11d7f9" + }, + { + "metadata": { + "ExecuteTime": { + "end_time": "2024-08-30T17:20:42.470606Z", + "start_time": "2024-08-30T17:20:42.467879Z" + } + }, + "cell_type": "code", + "source": [ + "os.environ[\"OPENASSETIO_BAL_IDENTIFIER\"] = \"org.openassetio.examples.manager.hybrid\"\n", + "os.environ[\"OPENASSETIO_SIMPLECPPMANAGER_IDENTIFIER\"] = \"org.openassetio.examples.manager.hybrid\"" + ], + "id": "6c95ee177f2a9028", + "outputs": [], + "execution_count": 18 + }, + { + "metadata": {}, + "cell_type": "markdown", + "source": [ + "Other configuration is provided by the configuration file `resources/hybrid_plugin_system/openassetio_config.toml`. Note that with hybrid plugins, the same configuration file is used for all the constituent plugins.\n", + "\n", + "Let's try to initialise the two managers separately and see what happens.\n" + ], + "id": "2332a406713d0e76" + }, + { + "metadata": { + "ExecuteTime": { + "end_time": "2024-08-30T17:20:42.534046Z", + "start_time": "2024-08-30T17:20:42.523535Z" + } + }, + "cell_type": "code", + "source": [ + "from openassetio.errors import ConfigurationException\n", + "from openassetio.pluginSystem import (\n", + " CppPluginSystemManagerImplementationFactory, PythonPluginSystemManagerImplementationFactory)\n", + "\n", + "\n", + "cpp_factory = CppPluginSystemManagerImplementationFactory(logger)\n", + "\n", + "try:\n", + " _ = ManagerFactory.defaultManagerForInterface(\n", + " \"resources/hybrid_plugin_system/openassetio_config.toml\",\n", + " host_interface,\n", + " cpp_factory,\n", + " logger)\n", + "\n", + "except ConfigurationException as exc:\n", + " helpers.display_result(f\"C++ plugin error: {exc}\")\n", + "\n", + "python_factory = PythonPluginSystemManagerImplementationFactory(logger)\n", + "\n", + "try:\n", + " _ = ManagerFactory.defaultManagerForInterface(\n", + " \"resources/hybrid_plugin_system/openassetio_config.toml\",\n", + " host_interface,\n", + " python_factory,\n", + " logger)\n", + "\n", + "except ConfigurationException as exc:\n", + " helpers.display_result(f\"Python plugin error: {exc}\")\n" + ], + "id": "129439ba58f8e81f", + "outputs": [ + { + "data": { + "text/markdown": "> **Result:**\n> `C++ plugin error: Manager implementation for 'org.openassetio.examples.manager.hybrid' does not support the required capabilities: managementPolicyQueries`" + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001B[0;33m warning: PythonPluginSystem: Use of top-level 'plugin' variable is deprecated, use `openassetioPlugin` instead. /home/dave/workspace/cloud/assetapi/OpenAssetIO-MediaCreation/.venv-conda-vfx24/lib/python3.11/site-packages/openassetio_manager_bal/__init__.py\u001B[0m\n" + ] + }, + { + "data": { + "text/markdown": "> **Result:**\n> `Python plugin error: Manager implementation for 'org.openassetio.examples.manager.hybrid' does not support the required capabilities: entityReferenceIdentification`" + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "execution_count": 19 + }, + { + "metadata": {}, + "cell_type": "markdown", + "source": "Neither of them advertise all the required capabilities! At least, on their own...", + "id": "9328df2151a26971" + }, + { + "metadata": {}, + "cell_type": "markdown", + "source": [ + "### The hybrid plugin system\n", + "\n", + "Given the two `ManagerImplementationFactoryInterface` instances (`cpp_factory` and `python_factory`), we can create a hybrid factory.\n", + "\n", + "For illustration purposes, we will create two managers from two different hybrid plugin system instances. The only difference being whether the C++ plugin system or the Python plugin system was provided first." + ], + "id": "5fd4a729153276f9" + }, + { + "metadata": { + "ExecuteTime": { + "end_time": "2024-08-30T17:20:42.651515Z", + "start_time": "2024-08-30T17:20:42.648800Z" + } + }, + "cell_type": "code", + "source": [ + "from openassetio.pluginSystem import HybridPluginSystemManagerImplementationFactory\n", + "\n", + "\n", + "hybrid_factory = HybridPluginSystemManagerImplementationFactory(\n", + " [cpp_factory, python_factory], logger)\n", + "\n", + "manager = ManagerFactory.defaultManagerForInterface(\n", + " \"resources/hybrid_plugin_system/openassetio_config.toml\",\n", + " host_interface,\n", + " hybrid_factory,\n", + " logger)" + ], + "id": "87c3d7e701b095d9", + "outputs": [], + "execution_count": 20 + }, + { + "metadata": {}, + "cell_type": "markdown", + "source": "Success! Now lets retrieve some data.", + "id": "52d907bb3a19db77" + }, + { + "metadata": { + "ExecuteTime": { + "end_time": "2024-08-30T17:20:42.704684Z", + "start_time": "2024-08-30T17:20:42.701225Z" + } + }, + "cell_type": "code", + "source": [ + "from openassetio.access import EntityTraitsAccess\n", + "\n", + "context = manager.createContext()\n", + "entity_ref = manager.createEntityReference(\"examplehybrid:///project_artwork/logos/openassetio\")\n", + "\n", + "trait_set = manager.entityTraits(entity_ref, EntityTraitsAccess.kRead, context)\n", + "\n", + "helpers.display_result(trait_set)\n" + ], + "id": "35b7fc091048a457", + "outputs": [ + { + "data": { + "text/markdown": "> **Result:**\n> `{'openassetio-mediacreation:usage.Entity', 'openassetio-mediacreation:twoDimensional.Image', 'openassetio-mediacreation:identity.DisplayName'}`" + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "execution_count": 21 + }, + { + "metadata": {}, + "cell_type": "markdown", + "source": [ + "So this is the entity's trait set, according to the (hybrid) manager. \n", + "\n", + "Or is it?\n", + "\n", + "What if both plugins advertise the same capability? In this case, the first plugin in the list supplied to the `HybridPluginSystemManagerImplementationFactory` constructor will be used." + ], + "id": "85d2a4a633cc9e82" + }, + { + "metadata": {}, + "cell_type": "markdown", + "source": "Now lets try a different configuration of plugins - note that this time `python_factory` is ordered before `cpp_factory`.", + "id": "76fa30f9d2f31969" + }, + { + "metadata": { + "ExecuteTime": { + "end_time": "2024-08-30T17:20:42.765994Z", + "start_time": "2024-08-30T17:20:42.751980Z" + } + }, + "cell_type": "code", + "source": [ + "alt_hybrid_factory = HybridPluginSystemManagerImplementationFactory(\n", + " [python_factory, cpp_factory], logger)\n", + "\n", + "alt_manager = ManagerFactory.defaultManagerForInterface(\n", + " \"resources/hybrid_plugin_system/openassetio_config.toml\",\n", + " host_interface,\n", + " alt_hybrid_factory,\n", + " logger)\n", + "\n", + "trait_set = alt_manager.entityTraits(entity_ref, EntityTraitsAccess.kRead, context)\n", + "\n", + "helpers.display_result(trait_set)" + ], + "id": "e8008d8183fb262d", + "outputs": [ + { + "data": { + "text/markdown": "> **Result:**\n> `{'openassetio-mediacreation:content.LocatableContent', 'openassetio-mediacreation:identity.DisplayName', 'openassetio-mediacreation:twoDimensional.Image', 'openassetio-mediacreation:lifecycle.Version', 'openassetio-mediacreation:usage.Entity', 'openassetio-mediacreation:timeDomain.FrameRanged'}`" + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "execution_count": 22 + }, + { + "metadata": {}, + "cell_type": "markdown", + "source": [ + "The trait set of the entity is different! \n", + "\n", + "This is because both BAL and SimpleCppManager support the `\"entityTraitIntrospection\"` capability, and in the `alt_manager` case the Python plugin system was first in the list provided to the hybrid factory constructor. The plugin loaded by the Python factory therefore gets priority for API calls, and the value returned from `entityTraits` by BAL is different than the value returned by SimpleCppManager.\n", + "\n", + "Note that in real-world usage, it is strongly recommended that constituent plugins return the same results when they support the same capability!" + ], + "id": "1fb66b7c6fd38071" + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 2 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython2", + "version": "2.7.6" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/examples/resources/hybrid_plugin_system/SimpleCppManager/README.md b/examples/resources/hybrid_plugin_system/SimpleCppManager/README.md new file mode 100644 index 0000000..bdb479d --- /dev/null +++ b/examples/resources/hybrid_plugin_system/SimpleCppManager/README.md @@ -0,0 +1,10 @@ +# Simple C++ Manager + +This directory is where the hybrid_plugin_system.ipynb notebook will +look for the SimpleCppManager plugin. + +In order for that notebook to run, the SimpleCppManager plugin must be +built and installed here. + +See the [SimpleCppManager README](https://github.com/OpenAssetIO/OpenAssetIO/blob/main/examples/manager/SimpleCppManager/README.md) +for more details. \ No newline at end of file diff --git a/examples/resources/hybrid_plugin_system/bal_database.json b/examples/resources/hybrid_plugin_system/bal_database.json new file mode 100644 index 0000000..f3894d2 --- /dev/null +++ b/examples/resources/hybrid_plugin_system/bal_database.json @@ -0,0 +1,42 @@ +{ + "capabilities": ["managementPolicyQueries","entityTraitIntrospection"], + "managementPolicy": { + "read": { + "default": { + "openassetio-mediacreation:managementPolicy.Managed": {}, + "openassetio-mediacreation:content.LocatableContent": {}, + "openassetio-mediacreation:identity.DisplayName": {}, + "openassetio-mediacreation:lifecycle.Version": {} + } + }, + "write": { + "default": { + "openassetio-mediacreation:managementPolicy.Managed": {}, + "openassetio-mediacreation:content.LocatableContent": {}, + "openassetio-mediacreation:identity.DisplayName": {} + } + }, + "managerDriven": { + "default": { + "openassetio-mediacreation:managementPolicy.Managed": {}, + "openassetio-mediacreation:content.LocatableContent": {}, + "openassetio-mediacreation:identity.DisplayName": {} + } + } + }, + "entities": { + "project_artwork/logos/openassetio": { + "versions": [ + { + "traits": { + "openassetio-mediacreation:usage.Entity": {}, + "openassetio-mediacreation:twoDimensional.Image": {}, + "openassetio-mediacreation:identity.DisplayName": {}, + "openassetio-mediacreation:content.LocatableContent": {}, + "openassetio-mediacreation:timeDomain.FrameRanged": {} + } + } + ] + } + } +} diff --git a/examples/resources/hybrid_plugin_system/openassetio_config.toml b/examples/resources/hybrid_plugin_system/openassetio_config.toml new file mode 100644 index 0000000..98d7314 --- /dev/null +++ b/examples/resources/hybrid_plugin_system/openassetio_config.toml @@ -0,0 +1,44 @@ +[manager] +# Identifier advertised by BAL and SimpleCppManager, overridden using +# environment variables. +identifier = "org.openassetio.examples.manager.hybrid" + +[manager.settings] + +######################################################################## +# BAL settings + +# BAL JSON library containing its settings/database. +# +# In the JSON config file we: +# * Set the capabilities to not advertise entityReferenceIdentification, +# or resolution, since SimpleCppManager will satisfy those +# capabilities (see below). +# * Note that entityTraitIntrospection is advertised by both managers, +# so which manager is used for `entityTraits(...)` API calls depends +# on priority order. +library_path = "${config_dir}/bal_database.json" + +# Entity reference format should match. Overridden similarly for +# SimpleCppManager, below. +entity_reference_url_scheme = "examplehybrid" + +######################################################################## +# SimpleCppManager settings: + +# Entity reference format should match. Overridden similarly for BAL, +# above. +prefix = "examplehybrid:///" + +# SimpleCppManager does not advertise "managementPolicyQueries", which +# is a required capability for a manager. But when used as part of a +# hybrid plugin, the capability can be satisfied by the other +# constituent plugins(s). +capabilities = "entityReferenceIdentification,resolution,entityTraitIntrospection" + +# The database of entities, as regurgitated by SimpleCppManager. +read_traits = ''' +examplehybrid:///project_artwork/logos/openassetio,openassetio-mediacreation:usage.Entity +examplehybrid:///project_artwork/logos/openassetio,openassetio-mediacreation:twoDimensional.Image +examplehybrid:///project_artwork/logos/openassetio,openassetio-mediacreation:identity.DisplayName,name,The OpenAssetIO Logo +''' \ No newline at end of file diff --git a/examples/resources/requirements.txt b/examples/resources/requirements.txt index 4342f3a..fab8fa3 100644 --- a/examples/resources/requirements.txt +++ b/examples/resources/requirements.txt @@ -1,5 +1,5 @@ jupyter -openassetio>=v1.0.0b2rev0 -openassetio-manager-bal>=v1.0.0a14 +openassetio>=v1.0.0rc1.rev0 +openassetio-manager-bal>=v1.0.0b1.rev0 openassetio-mediacreation Pillow