|
| 1 | +from typing import List, Optional, Tuple, Type, TypeVar |
| 2 | + |
| 3 | +import docutils.nodes as nodes |
| 4 | +from dagster._annotations import is_deprecated, is_public |
| 5 | +from sphinx.addnodes import versionmodified |
| 6 | +from sphinx.application import Sphinx |
| 7 | +from sphinx.environment import BuildEnvironment |
| 8 | +from sphinx.ext.autodoc import ( |
| 9 | + ClassDocumenter, |
| 10 | + ObjectMembers, |
| 11 | + Options as AutodocOptions, |
| 12 | +) |
| 13 | +from sphinx.util import logging |
| 14 | +from typing_extensions import Literal, TypeAlias |
| 15 | + |
| 16 | +from dagster_sphinx.configurable import ConfigurableDocumenter |
| 17 | + |
| 18 | +logger = logging.getLogger(__name__) |
| 19 | + |
| 20 | +##### Useful links for Sphinx documentation |
| 21 | +# |
| 22 | +# [Event reference] https://www.sphinx-doc.org/en/master/extdev/appapi.html#sphinx-core-events |
| 23 | +# These events are emitted during the build and can be hooked into during the |
| 24 | +# build process. |
| 25 | +# [autodoc] https://www.sphinx-doc.org/en/master/usage/extensions/autodoc.html |
| 26 | +# Autodoc is not sphinx itself, but it is the central extension that reads |
| 27 | +# docstrings. |
| 28 | +# [Sphinx extensions API] https://www.sphinx-doc.org/en/master/extdev/index.html |
| 29 | +# Root page for learning about writing extensions. |
| 30 | + |
| 31 | + |
| 32 | +# See: https://www.sphinx-doc.org/en/master/usage/extensions/autodoc.html#docstring-preprocessing |
| 33 | +# Autodoc doesn't provide it's own alias. |
| 34 | +AutodocObjectType: TypeAlias = Literal[ |
| 35 | + "module", "class", "exception", "function", "method", "attribute" |
| 36 | +] |
| 37 | + |
| 38 | + |
| 39 | +def record_error(env: BuildEnvironment, message: str) -> None: |
| 40 | + """Record an error in the Sphinx build environment. The error list is |
| 41 | + globally available during the build. |
| 42 | + """ |
| 43 | + logger.info(message) |
| 44 | + if not hasattr(env, "dagster_errors"): |
| 45 | + setattr(env, "dagster_errors", []) |
| 46 | + getattr(env, "dagster_errors").append(message) |
| 47 | + |
| 48 | + |
| 49 | +# ######################## |
| 50 | +# ##### CHECKS |
| 51 | +# ######################## |
| 52 | + |
| 53 | + |
| 54 | +def check_public_method_has_docstring(env: BuildEnvironment, name: str, obj: object) -> None: |
| 55 | + if name != "__init__" and not obj.__doc__: |
| 56 | + message = ( |
| 57 | + f"Docstring not found for {object.__name__}.{name}. " |
| 58 | + "All public methods and properties must have docstrings." |
| 59 | + ) |
| 60 | + record_error(env, message) |
| 61 | + |
| 62 | + |
| 63 | +class DagsterClassDocumenter(ClassDocumenter): |
| 64 | + """Overrides the default autodoc ClassDocumenter to adds some extra options.""" |
| 65 | + |
| 66 | + objtype = "class" |
| 67 | + |
| 68 | + def get_object_members(self, want_all: bool) -> Tuple[bool, ObjectMembers]: |
| 69 | + _, unfiltered_members = super().get_object_members(want_all) |
| 70 | + # Use form `is_public(self.object, attr_name) if possible, because to access a descriptor |
| 71 | + # object (returned by e.g. `@staticmethod`) you need to go in through |
| 72 | + # `self.object.__dict__`-- the value provided in the member list is _not_ the descriptor! |
| 73 | + filtered_members = [ |
| 74 | + m |
| 75 | + for m in unfiltered_members |
| 76 | + if m[0] in self.object.__dict__ and is_public(self.object.__dict__[m[0]]) |
| 77 | + ] |
| 78 | + for member in filtered_members: |
| 79 | + check_public_method_has_docstring(self.env, member[0], member[1]) |
| 80 | + return False, filtered_members |
| 81 | + |
| 82 | + |
| 83 | +# This is a hook that will be executed for every processed docstring. It modifies the lines of the |
| 84 | +# docstring in place. |
| 85 | +def process_docstring( |
| 86 | + app: Sphinx, |
| 87 | + what: AutodocObjectType, |
| 88 | + name: str, |
| 89 | + obj: object, |
| 90 | + options: AutodocOptions, |
| 91 | + lines: List[str], |
| 92 | +) -> None: |
| 93 | + assert app.env is not None |
| 94 | + |
| 95 | + # Insert a "deprecated" sphinx directive (this is built-in to autodoc) for objects flagged with |
| 96 | + # @deprecated. |
| 97 | + if is_deprecated(obj): |
| 98 | + # Note that these are in reversed order from how they will appear because we insert at the |
| 99 | + # front. We insert the <placeholder> string because the directive requires an argument that |
| 100 | + # we can't supply (we would have to know the version at which the object was deprecated). |
| 101 | + # We discard the "<placeholder>" string in `substitute_deprecated_text`. |
| 102 | + for line in ["", ".. deprecated:: <placeholder>"]: |
| 103 | + lines.insert(0, line) |
| 104 | + |
| 105 | + |
| 106 | +T_Node = TypeVar("T_Node", bound=nodes.Node) |
| 107 | + |
| 108 | + |
| 109 | +def get_child_as(node: nodes.Node, index: int, node_type: Type[T_Node]) -> T_Node: |
| 110 | + child = node.children[index] |
| 111 | + assert isinstance( |
| 112 | + child, node_type |
| 113 | + ), f"Docutils node not of expected type. Expected `{node_type}`, got `{type(child)}`." |
| 114 | + return child |
| 115 | + |
| 116 | + |
| 117 | +def substitute_deprecated_text(app: Sphinx, doctree: nodes.Element, docname: str) -> None: |
| 118 | + # The `.. deprecated::` directive is rendered as a `versionmodified` node. |
| 119 | + # Find them all and replace the auto-generated text, which requires a version argument, with a |
| 120 | + # plain string "Deprecated". |
| 121 | + for node in doctree.findall(versionmodified): |
| 122 | + paragraph = get_child_as(node, 0, nodes.paragraph) |
| 123 | + inline = get_child_as(paragraph, 0, nodes.inline) |
| 124 | + text = get_child_as(inline, 0, nodes.Text) |
| 125 | + inline.replace(text, nodes.Text("Deprecated")) |
| 126 | + |
| 127 | + |
| 128 | +def check_custom_errors(app: Sphinx, exc: Optional[Exception] = None) -> None: |
| 129 | + dagster_errors = getattr(app.env, "dagster_errors", []) |
| 130 | + if len(dagster_errors) > 0: |
| 131 | + for error_msg in dagster_errors: |
| 132 | + logger.info(error_msg) |
| 133 | + raise Exception( |
| 134 | + f"Bulid failed. Found {len(dagster_errors)} violations of docstring requirements." |
| 135 | + ) |
| 136 | + |
| 137 | + |
| 138 | +def setup(app): |
| 139 | + app.setup_extension("sphinx.ext.autodoc") # Require autodoc extension |
| 140 | + app.add_autodocumenter(ConfigurableDocumenter) |
| 141 | + # override allows `.. autoclass::` to invoke DagsterClassDocumenter instead of default |
| 142 | + app.add_autodocumenter(DagsterClassDocumenter, override=True) |
| 143 | + app.connect("autodoc-process-docstring", process_docstring) |
| 144 | + app.connect("doctree-resolved", substitute_deprecated_text) |
| 145 | + app.connect("build-finished", check_custom_errors) |
| 146 | + |
| 147 | + return { |
| 148 | + "version": "0.1", |
| 149 | + "parallel_read_safe": True, |
| 150 | + "parallel_write_safe": True, |
| 151 | + } |
0 commit comments