diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 0af3107c..0433749a 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -110,9 +110,14 @@ jobs: name: doc-bundles path: ~/.papyri/data/ + - name: Test with pytest + run: | + pip install scipy + pytest --cov=papyri --cov-append + ls -al - name: Misc cli run: | - coverage run -m papyri --help + coverage run -a -m papyri --help ls -al - name: Ingest run: | @@ -121,11 +126,6 @@ jobs: coverage run -a -m papyri render --ascii ls -al - - name: Test with pytest - run: | - pip install scipy - pytest --cov=papyri --cov-append - ls -al - name: Convert .coverage sql to xml for upload run: | coverage report diff --git a/examples/IPython.toml b/examples/IPython.toml index ca6f526f..c3d8ce86 100644 --- a/examples/IPython.toml +++ b/examples/IPython.toml @@ -11,9 +11,18 @@ execute_exclude_patterns = [ exclude = [] [global.expected_errors] IncorrectInternalDocsLen = [ - "IPython.core.magics.namespace:NamespaceMagics.xdel", + "IPython.core.magics.namespace:NamespaceMagics.xdel", +] +TextSignatureParsingFailed = [ + "IPython.core.guarded_eval:SelectivePolicy", + "IPython.core.oinspect:OInfo", + "IPython.terminal.shortcuts:RuntimeBinding", + "IPython.terminal.shortcuts:BaseBinding", + "IPython.terminal.shortcuts:Binding", + "IPython.utils.timing:clock2", + "IPython.utils.timing:timings", + "IPython.utils.timing:timings_out", ] - [global.implied_imports] get_ipython = 'IPython:get_ipython' diff --git a/examples/scipy.toml b/examples/scipy.toml index 1e533e53..c7d3ab9f 100644 --- a/examples/scipy.toml +++ b/examples/scipy.toml @@ -65,6 +65,7 @@ pypi = 'scipy' homepage = 'https://scipy.org/' docspage = 'https://docs.scipy.org/doc/scipy/' [global.implied_imports] +scipy = 'scipy' ua = 'scipy._lib.uarray' array = 'numpy:array' arange = 'numpy:arange' @@ -100,6 +101,10 @@ VisitCitationReferenceNotImplementedError = [ "scipy.optimize._nonlin:Anderson", "scipy.spatial._spherical_voronoi:calculate_solid_angles", "scipy.special._orthogonal:_pbcf", + "scipy.special._orthogonal", + "scipy.optimize._lsq.dogbox", + "scipy.optimize._lsq.trf", + "scipy.stats._levy_stable:levy_stable_gen", ] IncorrectInternalDocsLen = [ "scipy.signal._spline:symiirorder1", @@ -146,6 +151,8 @@ NumpydocParseError = [ "scipy.stats._discrete_distns:betabinom_gen", "scipy.stats._discrete_distns:geom_gen", "scipy.stats._discrete_distns:planck_gen", + "scipy.special:btdtr", + "scipy.special:btdtri", ] ExampleError1 = [ "scipy.stats._qmc:PoissonDisk", diff --git a/papyri/__init__.py b/papyri/__init__.py index a83568b1..bea8cf15 100644 --- a/papyri/__init__.py +++ b/papyri/__init__.py @@ -127,7 +127,7 @@ points via aliases; in the intro one link as a ``name `` syntax which is also not yet recognized. -`scipy.signal.filter_design.zpk2sos`: +`scipy.signal.zpk2sos`: multi blocks in enumerated list `scipy.signal.filter_design.zpk2sos`: @@ -141,7 +141,7 @@ `numpy.einsum`: one of the longest numpy docstring/document, or at least one of the longest to render, with - `scipy.signal.windows.windows.dpss` , `scipy.optimize._minimize.minimize` and + `scipy.signal.windows.dpss` , `scipy.optimize._minimize.minimize` and `scipy.optimize._basinhopping.basinhopping` diff --git a/papyri/ascii.tpl.j2 b/papyri/ascii.tpl.j2 index def99fdb..9bb80d17 100644 --- a/papyri/ascii.tpl.j2 +++ b/papyri/ascii.tpl.j2 @@ -139,7 +139,7 @@ {{-green("green")}} refer to items within the same document, {{blue("blue")}} external {{underline("known")}} entities, {{red("red")}} entities we were not able to find and {{yellow("yellow")}} for code, and other "raw" items, typically between double backticks. {% if doc.signature %} - |{{ bold(underline(doc.signature.value))}} + |{{bold(underline(name))}}{{bold(underline(doc.signature.to_signature()))}} {% endif %} {% for section in doc.content if doc.content[section] %} diff --git a/papyri/crosslink.py b/papyri/crosslink.py index fc6f43bb..15865ef1 100644 --- a/papyri/crosslink.py +++ b/papyri/crosslink.py @@ -16,13 +16,13 @@ from .config import ingest_dir from .gen import DocBlob, normalise_ref from .graphstore import GraphStore, Key +from .signature import SignatureNode from .take2 import ( Param, RefInfo, Fig, Section, SeeAlsoItem, - Signature, encoder, TocTree, ) @@ -87,7 +87,7 @@ class IngestedBlobs(Node): aliases: List[str] example_section_data: Section see_also: List[SeeAlsoItem] # see also data - signature: Signature + signature: Optional[SignatureNode] references: Optional[List[str]] qa: str arbitrary: List[Section] diff --git a/papyri/errors.py b/papyri/errors.py index 568bb199..0067383b 100644 --- a/papyri/errors.py +++ b/papyri/errors.py @@ -2,6 +2,10 @@ class IncorrectInternalDocsLen(AssertionError): pass +class TextSignatureParsingFailed(ValueError): + pass + + class NumpydocParseError(ValueError): pass diff --git a/papyri/examples.py b/papyri/examples.py index a596d788..f542ef17 100644 --- a/papyri/examples.py +++ b/papyri/examples.py @@ -39,8 +39,12 @@ """ +from typing import Optional, Union -def example1(pos, only, /, var, args, *, kwargs, also=None): + +def example1( + pos: int, only: None, /, var: Union[float, bool], args=1, *, kwargs, also=None +) -> Optional[str]: """ first example. @@ -56,6 +60,7 @@ def example1(pos, only, /, var, args, *, kwargs, also=None): ... plt.show() """ + return "ok" def example2(): diff --git a/papyri/gen.py b/papyri/gen.py index 91174f5a..0d8b4389 100644 --- a/papyri/gen.py +++ b/papyri/gen.py @@ -53,9 +53,10 @@ IncorrectInternalDocsLen, NumpydocParseError, UnseenError, + TextSignatureParsingFailed, ) from .miscs import BlockExecutor, DummyP -from .signature import Signature as ObjectSignature +from .signature import Signature as ObjectSignature, SignatureNode from .take2 import ( Code, Fig, @@ -69,7 +70,6 @@ RefInfo, Section, SeeAlsoItem, - Signature, parse_rst_section, ) from .toc import make_tree @@ -536,7 +536,7 @@ def gen_main( g = Gen(dummy_progress=dummy_progress, config=config) g.log.info("Will write data to %s", target_dir) if debug: - g.log.setLevel("DEBUG") + g.log.setLevel(logging.DEBUG) g.log.debug("Log level set to debug") g.collect_package_metadata( @@ -730,6 +730,20 @@ class DocBlob(Node): as well as link to external references, like images generated. """ + __slots__ = ( + "content", + "example_section_data", + "ordered_sections", + "item_file", + "item_line", + "item_type", + "aliases", + "see_also", + "signature", + "references", + "arbitrary", + ) + @classmethod def _deserialise(cls, **kwargs): # print_("will deserialise", cls) @@ -769,8 +783,8 @@ def _deserialise(cls, **kwargs): item_type: Optional[str] aliases: List[str] see_also: List[SeeAlsoItem] # see also data - signature: Signature - references: Optional[List[str]] + signature: Optional[SignatureNode] + references: Optional[Section] arbitrary: List[Section] def __repr__(self): @@ -792,30 +806,7 @@ def slots(self): @classmethod def new(cls): - return cls({}, None, None, None, None, None, [], [], Signature(None), None, []) - - # def __init__( - # self, - # content, - # example_section_data, - # ordered_sections, - # item_file, - # item_line, - # item_type, - # aliases, - # see_also, - # signature, - # references, - # arbitrary, - # ): - # self.content = content - # self.example_section_data = example_section_data - # self.ordered_sections = ordered_sections - # self.item_file = item_file - # self.item_line = item_line - # self.item_type = item_type - # self.aliases = aliases - # self.signature = signature + return cls({}, None, None, None, None, None, [], [], None, None, []) def _numpy_data_to_section(data: List[Tuple[str, str, List[str]]], title: str, qa): @@ -868,14 +859,25 @@ class APIObjectInfo: kind: str docstring: str - signature: Optional[Signature] + signature: Optional[ObjectSignature] name: str - def __init__(self, kind, docstring, signature, name, qa): + def __repr__(self): + return f"" + + def __init__( + self, + kind: str, + docstring: str, + signature: Optional[ObjectSignature], + name: str, + qa: str, + ): + assert isinstance(signature, (ObjectSignature, type(None))) self.kind = kind self.name = name self.docstring = docstring - self.parsed = [] + self.parsed: List[Any] = [] self.signature = signature self._qa = qa @@ -895,9 +897,14 @@ def __init__(self, kind, docstring, signature, name, qa): assert isinstance(section, Section) self.parsed.append(section) elif title in _numpydoc_sections_with_text: - docs = ts.parse("\n".join(ndoc[title]).encode(), qa) + predoc = "\n".join(ndoc[title]) + docs = ts.parse(predoc.encode(), qa) if len(docs) != 1: - raise IncorrectInternalDocsLen("\n".join(ndoc[title]), docs) + # TODO + # potential reasons + # Summary and Extended Summary should be parsed as one. + # References with ` : ` in them fail parsing.Issue opened in Tree-sitter. + raise IncorrectInternalDocsLen(predoc, docs) section = docs[0] assert isinstance(section, Section), section self.parsed.append(section) @@ -933,7 +940,7 @@ def validate(self): p.validate() -def _normalize_see_also(see_also: List[Any], qa): +def _normalize_see_also(see_also: Section, qa: str): """ numpydoc is complex, the See Also fields can be quite complicated, so here we sort of try to normalise them. @@ -1347,7 +1354,7 @@ def collect_narrative_docs(self): blob.aliases = [] blob.example_section_data = Section([], None) blob.see_also = [] - blob.signature = Signature(None) + blob.signature = None blob.references = None blob.validate() titles = [s.title for s in blob.arbitrary if s.title] @@ -1419,11 +1426,13 @@ def put_raw(self, path: str, data: bytes): """ self.bdata[path] = data - def _transform_1(self, blob, ndoc): + def _transform_1(self, blob: DocBlob, ndoc) -> DocBlob: blob.content = {k: v for k, v in ndoc._parsed_data.items()} + for k, v in blob.content.items(): + assert isinstance(v, (str, list, dict)), type(v) return blob - def _transform_2(self, blob, target_item, qa): + def _transform_2(self, blob: DocBlob, target_item, qa: str) -> DocBlob: # try to find relative path WRT site package. # will not work for dev install. Maybe an option to set the root location ? item_file = find_file(target_item) @@ -1492,7 +1501,7 @@ def prepare_doc_for_one_object( qa: str, config: Config, aliases: List[str], - api_object, + api_object: APIObjectInfo, ) -> Tuple[DocBlob, List]: """ Get documentation information for one python object @@ -1509,7 +1518,7 @@ def prepare_doc_for_one_object( current configuratin aliases : sequence other aliases for cuttent object. - api_object : + api_object : APIObjectInfo Returns @@ -1525,7 +1534,7 @@ def prepare_doc_for_one_object( collect_api_docs """ assert isinstance(aliases, list) - blob = DocBlob.new() + blob: DocBlob = DocBlob.new() blob = self._transform_1(blob, ndoc) blob = self._transform_2(blob, target_item, qa) @@ -1533,13 +1542,25 @@ def prepare_doc_for_one_object( item_type = str(type(target_item)) if blob.content["Signature"]: - blob.signature = Signature(blob.content.pop("Signature")) + try: + # the type ignore below is wrong and need to be refactored. + # we basically modify blob.content in place, but should not. + sig = ObjectSignature.from_str(blob.content.pop("Signature")) # type: ignore + if sig is not None: + blob.signature = sig.to_node() + except TextSignatureParsingFailed: + # this really fails often when the first line is not Signature. + # or when numpy has the def f(,...[a,b,c]) optional parameter. + pass else: assert blob is not None assert api_object is not None - - blob.signature = Signature(api_object.signature) + if api_object.signature is None: + blob.signature = None + else: + blob.signature = api_object.signature.to_node() del blob.content["Signature"] + self.log.debug("SIG %r", blob.signature) if api_object.special("Examples"): # warnings this is true only for non-modules @@ -1651,7 +1672,7 @@ def prepare_doc_for_one_object( else: blob.content[s] = Section([], None) - blob.see_also = _normalize_see_also(blob.content.get("See Also", None), qa) + blob.see_also = _normalize_see_also(blob.content.get("See Also", Section()), qa) del blob.content["See Also"] return blob, figs @@ -1794,7 +1815,7 @@ def collect_examples_out(self): def helper_1( self, *, qa: str, target_item: Any - ) -> Tuple[Optional[str], List[Section], Optional[APIObjectInfo]]: + ) -> Tuple[Optional[str], List[Section], APIObjectInfo]: """ Parameters ---------- @@ -1814,11 +1835,9 @@ def helper_1( "module", item_docstring, None, target_item.__name__, qa ) elif isinstance(target_item, (FunctionType, builtin_function_or_method)): - sig: Optional[str] + sig: Optional[ObjectSignature] try: - sig = str(ObjectSignature(target_item)) - # sig = qa.split(":")[-1] + sig - # sig = re.sub("at 0x[0-9a-f]+", "at 0x0000000", sig) + sig = ObjectSignature(target_item) except (ValueError, TypeError): sig = None try: @@ -1948,7 +1967,7 @@ def collect_api_docs(self, root: str, limit_to: List[str]): # taskp = p2.add_task(description="parsing", total=len(collected)) failure_collection: Dict[str, List[str]] = defaultdict(lambda: []) - + api_object: APIObjectInfo for qa, target_item in collected.items(): self.log.debug("treating %r", qa) @@ -1957,6 +1976,7 @@ def collect_api_docs(self, root: str, limit_to: List[str]): qa=qa, target_item=target_item, ) + self.log.debug("APIOBJECT %r", api_object) if ecollector.errored: if ecollector._errors.keys(): self.log.warning( @@ -1965,7 +1985,6 @@ def collect_api_docs(self, root: str, limit_to: List[str]): else: self.log.info("only expected error with %s", qa) continue - assert api_object is not None, ecollector.errored try: if item_docstring is None: @@ -1999,7 +2018,6 @@ def collect_api_docs(self, root: str, limit_to: List[str]): ex = False # TODO: ndoc-placeholder : make sure ndoc placeholder handled here. - assert api_object is not None with error_collector(qa=qa) as c: doc_blob, figs = self.prepare_doc_for_one_object( target_item, @@ -2009,6 +2027,7 @@ def collect_api_docs(self, root: str, limit_to: List[str]): aliases=collector.aliases[qa], api_object=api_object, ) + del api_object if c.errored: continue _local_refs: List[str] = [] @@ -2086,6 +2105,8 @@ def collect_api_docs(self, root: str, limit_to: List[str]): doc_blob.validate() except Exception as e: raise type(e)(f"Error in {qa}") + self.log.debug(doc_blob.signature) + self.log.debug(doc_blob.to_dict()) self.put(qa, doc_blob) if figs: self.log.debug("Found %s figures", len(figs)) diff --git a/papyri/html.tpl.j2 b/papyri/html.tpl.j2 index c4b52eb4..073937de 100644 --- a/papyri/html.tpl.j2 +++ b/papyri/html.tpl.j2 @@ -30,8 +30,8 @@ -{% if doc.signature.value -%} - {{doc.signature.value}} +{% if doc.signature -%} + {{name}}{{doc.signature.to_signature()}} {%- endif -%} {%- for section in doc.content.keys() if doc.content[section] -%} diff --git a/papyri/miniserde.py b/papyri/miniserde.py index b9e4bf22..b26dd070 100644 --- a/papyri/miniserde.py +++ b/papyri/miniserde.py @@ -148,6 +148,9 @@ def serialize(instance, annotation): ) from e +_sentinel = object() + + # type_ and annotation are _likely_ duplicate here as an annotation is likely a type, or a List, Union, ....) def deserialize(type_, annotation, data): # assert type_ is annotation @@ -208,8 +211,8 @@ def deserialize(type_, annotation, data): try: real_type = real_type[0] except IndexError: - breakpoint() - if data.get("data"): + raise + if data.get("data", _sentinel) is not _sentinel: data_ = data["data"] else: data_ = {k: v for k, v in data.items() if k != "type"} diff --git a/papyri/render.py b/papyri/render.py index aabd9c9c..1e5305ca 100644 --- a/papyri/render.py +++ b/papyri/render.py @@ -197,6 +197,10 @@ def GET(ref_map, key, cpath): return siblings +def _uuid(): + return uuid.uuid4().hex + + class HtmlRenderer: def __init__(self, store: GraphStore, *, sidebar, prefix, trailing_html): assert prefix.startswith("/") @@ -220,7 +224,7 @@ def __init__(self, store: GraphStore, *, sidebar, prefix, trailing_html): self.env.globals["unreachable"] = unreachable self.env.globals["sidebar"] = sidebar self.env.globals["dothtml"] = extension - self.env.globals["uuid"] = lambda: uuid.uuid4().hex + self.env.globals["uuid"] = _uuid def compute_graph( self, backrefs: Set[Key], refs: Set[Key], key: Key @@ -382,7 +386,7 @@ class S: backrefs=[[], []], module="*", doc=doc, - parts={"*": []}, + parts=list({"*": []}.items()), version="*", ext="", current_type="", @@ -477,7 +481,8 @@ class D: meta=meta, figmap=figmap, module=package, - parts=parts, + parts_mods=parts.get(package, []), + parts=list(parts.items()), version=version, parts_links=defaultdict(lambda: ""), doc=doc, @@ -503,7 +508,8 @@ class D: logo=logo, meta=meta, module=package, - parts={}, + parts=list({}.items()), + parts_mods=[], version=version, parts_links=defaultdict(lambda: ""), doc=doc, @@ -571,15 +577,17 @@ def render_one( doc.content[k] = self.LR.visit(v) doc.arbitrary = [self.LR.visit(x) for x in doc.arbitrary] + module = qa.split(".")[0] return template.render( current_type=current_type, doc=doc, logo=meta.get("logo", None), - qa=qa, version=meta["version"], - module=qa.split(".")[0], + module=module, + name=qa.split(":")[-1].split(".")[-1], backrefs=backrefs, - parts=parts, + parts_mods=parts.get(module, []), + parts=list(parts.items()), parts_links=parts_links, graph=graph, meta=meta, @@ -654,8 +662,8 @@ async def _route_data(self, ref, version, known_refs): async def _route( self, - ref, - version=None, + ref: str, + version: Optional[str] = None, ): assert not ref.endswith(".html") assert version is not None @@ -727,6 +735,7 @@ async def _write_api_file( template = self.env.get_template("html.tpl.j2") gfiles = list(self.store.glob((None, None, "module", None))) random.shuffle(gfiles) + for _, key in progress(gfiles, description="Rendering API..."): module, version = key.module, key.version if config.ascii: @@ -782,11 +791,12 @@ async def _copy_dir(self, src_dir: Path, dest_dir: Path): async def copy_static(self, output_dir): here = Path(os.path.dirname(__file__)) _static = here / "static" - output_dir.mkdir(exist_ok=True) - static = output_dir.parent / "static" - static.mkdir(exist_ok=True) - await self._copy_dir(_static, static) - (static / "pygments.css").write_bytes(await pygment_css().get_data()) + if output_dir is not None: + output_dir.mkdir(exist_ok=True) + static = output_dir.parent / "static" + static.mkdir(exist_ok=True) + await self._copy_dir(_static, static) + (static / "pygments.css").write_bytes(await pygment_css().get_data()) async def copy_assets(self, config): """ @@ -870,7 +880,7 @@ class Doc: meta=meta, logo=meta["logo"], module=package, - parts=parts, + parts=list(parts.items()), version=version, parts_links=defaultdict(lambda: ""), doc=doc, @@ -898,7 +908,8 @@ class Doc: meta=meta, logo=logo, module=module, - parts=parts, + parts_mods=parts.get(module, []), + parts=list(parts.items()), version=version, parts_links=defaultdict(lambda: ""), doc=doc, @@ -1126,7 +1137,7 @@ def old_render_one( *, current_type, backrefs, - parts=(), + parts: Dict[str, List[Tuple[str, str]]], parts_links=(), graph="{}", meta, @@ -1185,11 +1196,11 @@ def old_render_one( current_type=current_type, doc=doc, logo=meta.get("logo", None), - qa=qa, + name=qa.split(":")[-1].split(".")[-1], version=meta["version"], module=qa.split(".")[0], backrefs=backrefs, - parts=parts, + parts=list(parts.items()), parts_links=parts_links, graph=graph, meta=meta, @@ -1247,6 +1258,7 @@ async def _ascii_render(key: Key, store: GraphStore, known_refs=None, template=N return old_render_one( store, + parts={}, current_type="API", meta=meta, template=template, diff --git a/papyri/signature.py b/papyri/signature.py index ac92b1db..a46eaf34 100644 --- a/papyri/signature.py +++ b/papyri/signature.py @@ -1,35 +1,78 @@ import inspect from dataclasses import dataclass -from typing import Optional, List +from typing import List, Any, Dict, Union from .common_ast import Node +from .errors import TextSignatureParsingFailed +from .common_ast import register +import json + + +@register(4031) +class Empty(Node): + pass + + +_empty = Empty() + +NoneType = type(None) + + +@register(4030) @dataclass class ParameterNode(Node): name: str # we likely want to make sure annotation is a structured object in the long run - annotation: Optional[str] + annotation: Union[str, NoneType, Empty] kind: str - default: Optional[str] + default: Union[str, NoneType, Empty] - def to_parameter(self): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + def to_parameter(self) -> inspect.Parameter: return inspect.Parameter( name=self.name, kind=getattr(inspect._ParameterKind, self.kind), - default=self.default, - annotation=self.annotation, + default=inspect._empty if isinstance(self.default, Empty) else None, + annotation=inspect._empty + if isinstance(self.annotation, Empty) + else self.annotation, ) +@register(4029) class SignatureNode(Node): kind: str # maybe enum, is it a function, async generator, generator, etc. parameters: List[ParameterNode] # of pairs, we don't use dict because of ordering + return_annotation: Union[Empty, str] + + def to_signature(self): + return inspect.Signature([p.to_parameter() for p in self.parameters]) class Signature: """A wrapper around inspect utilities.""" + @classmethod + def from_str(cls, sig: str, /) -> "Signature": + """ + Create signature from a string version. + + Of course this is slightly incorrect as all the isgerator and CO are going to wrong + + """ + glob: Dict[str, Any] = {} + oname = sig.split("(")[0] + toexec = f"def {sig}:pass" + try: + exec(toexec, {}, glob) + except Exception as e: + raise TextSignatureParsingFailed(f"Unable to parse {toexec}") from e + return cls(glob[oname]) + def __init__(self, target_item): """ Initialize the class. @@ -43,12 +86,14 @@ def __init__(self, target_item): self.target_item = target_item self._sig = inspect.signature(target_item) - def to_node(self): - if inspect.isfunction(self.target_item): + def to_node(self) -> SignatureNode: + if inspect.isbuiltin(self.target_item): + kind = "builtins" + elif inspect.isfunction(self.target_item): kind = "function" - elif self.is_generator(): + elif self.is_generator: kind = "generator" - elif self.is_async_generator(): + elif self.is_async_generator: kind = "async_generator" elif inspect.iscoroutinefunction(self.target_item): kind = "coroutine_function" @@ -58,40 +103,65 @@ def to_node(self): parameters = [] for param in self.parameters.values(): + annotation: Union[Empty, str] + if param.annotation is inspect._empty: + annotation = _empty + elif isinstance(param.annotation, str): + annotation = param.annotation + else: + # TODO: Keep the original annotation object somewhere + annotation = inspect.formatannotation(param.annotation) parameters.append( ParameterNode( - param.name, - None if param.annotation is inspect._empty else param.annotation, - param.kind.name, - None if param.default else str(param.default), + name=param.name, + annotation=annotation, + kind=param.kind.name, + default=_empty + if param.default is inspect._empty + else str(param.default), ) ) - - return SignatureNode(kind, parameters) + assert isinstance(kind, str) + return SignatureNode( + kind=kind, + parameters=parameters, + return_annotation=_empty + if self._sig.return_annotation is inspect._empty + else str(self._sig.return_annotation), + ) @property def parameters(self): return self._sig.parameters @property - def is_async_function(self): + def is_async_function(self) -> bool: return inspect.iscoroutinefunction(self.target_item) @property - def is_async_generator(self): + def is_async_generator(self) -> bool: return inspect.isasyncgenfunction(self.target_item) @property - def is_generator(self): + def is_generator(self) -> bool: return inspect.isgenerator(self.target_item) def param_default(self, param): return self.parameters.get(param).default @property - def annotations(self): + def annotations(self) -> bool: return self.target_item.__annotations__ + @property + def return_annotation(self) -> Union[Empty, str]: + return_annotation = self._sig.return_annotation + return ( + _empty + if return_annotation is inspect._empty + else inspect.formatannotation(return_annotation) + ) + @property def is_public(self) -> bool: return not self.target_item.__name__.startswith("_") @@ -124,5 +194,25 @@ def keyword_only_parameter_count(self): else: return None + def to_dict(self) -> dict: + """ + Output self as JSON (Python dict), using the same format as Griffe + """ + json_data = self.to_node().to_dict() + + # Use human-readable names for parameter kinds + for param in json_data["parameters"]: + param["kind"] = getattr(inspect._ParameterKind, param["kind"]).description + + json_data["returns"] = self.return_annotation + + return json_data + + def to_json(self) -> bytes: + """ + Output self as JSON, using the same format as Griffe + """ + return json.dumps(self.to_dict(), indent=2, sort_keys=True).encode() + def __str__(self): return str(self._sig) diff --git a/papyri/skeleton.tpl.j2 b/papyri/skeleton.tpl.j2 index fdb647db..2016dff6 100644 --- a/papyri/skeleton.tpl.j2 +++ b/papyri/skeleton.tpl.j2 @@ -6,7 +6,7 @@
{{module}} / 
- {% for link,name in parts.get(module, []) -%} + {% for link,name in parts_mods -%} {{link.path}} {% endfor %}
@@ -30,7 +30,7 @@
- {% for p,links in parts.items() %} + {% for p,links in parts %} {% if links %}
{% if p =='+' %} diff --git a/papyri/take2.py b/papyri/take2.py index 50016965..51a78b51 100644 --- a/papyri/take2.py +++ b/papyri/take2.py @@ -235,11 +235,6 @@ def from_untrusted(cls, module, version, kind, path): return cls(module, version, kind, path) -@register(4011) -class Signature(Node): - value: Optional[str] - - @register(4012) class NumpydocExample(Node): value: List[str] diff --git a/papyri/tests/test_gen.py b/papyri/tests/test_gen.py index 25cc834d..1e19af34 100644 --- a/papyri/tests/test_gen.py +++ b/papyri/tests/test_gen.py @@ -95,3 +95,62 @@ def test_numpy(module, submodules, objects): for o in objects: assert (td / "module" / f"{o}.json").exists() + + +def test_self(): + from papyri.gen import Gen, Config + + c = Config(dry_run=True, dummy_progress=True) + g = Gen(False, config=c) + g.collect_package_metadata("papyri", ".", {}) + g.collect_api_docs("papyri", {"papyri.examples:example1", "papyri"}) + assert g.data["papyri.examples:example1"].to_dict()["signature"] == { + "type": "SignatureNode", + "kind": "function", + "parameters": [ + { + "type": "ParameterNode", + "name": "pos", + "annotation": {"data": "int", "type": "str"}, + "kind": "POSITIONAL_ONLY", + "default": {"type": "Empty"}, + }, + { + "type": "ParameterNode", + "name": "only", + "annotation": {"data": "None", "type": "str"}, + "kind": "POSITIONAL_ONLY", + "default": {"type": "Empty"}, + }, + { + "type": "ParameterNode", + "name": "var", + "annotation": {"data": "Union[float, bool]", "type": "str"}, + "kind": "POSITIONAL_OR_KEYWORD", + "default": {"type": "Empty"}, + }, + { + "type": "ParameterNode", + "name": "args", + "annotation": {"type": "Empty"}, + "kind": "POSITIONAL_OR_KEYWORD", + "default": {"data": "1", "type": "str"}, + }, + { + "type": "ParameterNode", + "name": "kwargs", + "annotation": {"type": "Empty"}, + "kind": "KEYWORD_ONLY", + "default": {"type": "Empty"}, + }, + { + "type": "ParameterNode", + "name": "also", + "annotation": {"type": "Empty"}, + "kind": "KEYWORD_ONLY", + "default": {"data": "None", "type": "str"}, + }, + ], + "return_annotation": {"data": "typing.Optional[str]", "type": "str"}, + } + assert g.data["papyri"].to_dict()["signature"] is None diff --git a/papyri/tests/test_signatures.py b/papyri/tests/test_signatures.py index fdc791bb..8711a328 100644 --- a/papyri/tests/test_signatures.py +++ b/papyri/tests/test_signatures.py @@ -4,6 +4,7 @@ """ from papyri.signature import Signature as SignatureObject, SignatureNode import json +from typing import Union, Optional import pytest @@ -18,191 +19,320 @@ def add(func): @add def function_1(posonly, /, pos_or_k, pos_ok_k_d=1, *varargs, **varkwargs): - """ - { + """{ "kind": "function", "parameters": [ { - "annotation": null, - "default": null, + "annotation": { + "type": "Empty" + }, + "default": { + "type": "Empty" + }, "kind": "POSITIONAL_ONLY", "name": "posonly", "type": "ParameterNode" }, { - "annotation": null, - "default": null, + "annotation": { + "type": "Empty" + }, + "default": { + "type": "Empty" + }, "kind": "POSITIONAL_OR_KEYWORD", "name": "pos_or_k", "type": "ParameterNode" }, { - "annotation": null, - "default": null, + "annotation": { + "type": "Empty" + }, + "default": { + "data": "1", + "type": "str" + }, "kind": "POSITIONAL_OR_KEYWORD", "name": "pos_ok_k_d", "type": "ParameterNode" }, { - "annotation": null, - "default": null, + "annotation": { + "type": "Empty" + }, + "default": { + "type": "Empty" + }, "kind": "VAR_POSITIONAL", "name": "varargs", "type": "ParameterNode" }, { - "annotation": null, - "default": null, + "annotation": { + "type": "Empty" + }, + "default": { + "type": "Empty" + }, "kind": "VAR_KEYWORD", "name": "varkwargs", "type": "ParameterNode" } ], + "return_annotation": {"type": "Empty"}, "type": "SignatureNode" }""" pass @add -def async_function_1(posonly, /, pos_or_k, pos_ok_k_d=1, *varargs, **varkwargs): +def async_function_2(posonly, /, pos_or_k, pos_ok_k_d=1, *varargs, **varkwargs): """{ "kind": "function", "parameters": [ { - "annotation": null, - "default": null, + "annotation": { + "type": "Empty" + }, + "default": { + "type": "Empty" + }, "kind": "POSITIONAL_ONLY", "name": "posonly", "type": "ParameterNode" }, { - "annotation": null, - "default": null, + "annotation": { + "type": "Empty" + }, + "default": { + "type": "Empty" + }, "kind": "POSITIONAL_OR_KEYWORD", "name": "pos_or_k", "type": "ParameterNode" }, { - "annotation": null, - "default": null, + "annotation": { + "type": "Empty" + }, + "default": { + "data": "1", + "type": "str" + }, "kind": "POSITIONAL_OR_KEYWORD", "name": "pos_ok_k_d", "type": "ParameterNode" }, { - "annotation": null, - "default": null, + "annotation": { + "type": "Empty" + }, + "default": { + "type": "Empty" + }, "kind": "VAR_POSITIONAL", "name": "varargs", "type": "ParameterNode" }, { - "annotation": null, - "default": null, + "annotation": { + "type": "Empty" + }, + "default": { + "type": "Empty" + }, "kind": "VAR_KEYWORD", "name": "varkwargs", "type": "ParameterNode" } ], + "return_annotation": {"type": "Empty"}, "type": "SignatureNode" }""" pass @add -def generator_function_1(posonly, /, pos_or_k, pos_ok_k_d=1, *varargs, **varkwargs): +def generator_function_3(posonly, /, pos_or_k, pos_ok_k_d=1, *varargs, **varkwargs): """{ "kind": "function", "parameters": [ { - "annotation": null, - "default": null, + "annotation": { + "type": "Empty" + }, + "default": { + "type": "Empty" + }, "kind": "POSITIONAL_ONLY", "name": "posonly", "type": "ParameterNode" }, { - "annotation": null, - "default": null, + "annotation": { + "type": "Empty" + }, + "default": { + "type": "Empty" + }, "kind": "POSITIONAL_OR_KEYWORD", "name": "pos_or_k", "type": "ParameterNode" }, { - "annotation": null, - "default": null, + "annotation": { + "type": "Empty" + }, + "default": { + "data": "1", + "type": "str" + }, "kind": "POSITIONAL_OR_KEYWORD", "name": "pos_ok_k_d", "type": "ParameterNode" }, { - "annotation": null, - "default": null, + "annotation": { + "type": "Empty" + }, + "default": { + "type": "Empty" + }, "kind": "VAR_POSITIONAL", "name": "varargs", "type": "ParameterNode" }, { - "annotation": null, - "default": null, + "annotation": { + "type": "Empty" + }, + "default": { + "type": "Empty" + }, "kind": "VAR_KEYWORD", "name": "varkwargs", "type": "ParameterNode" } ], + "return_annotation": {"type": "Empty"}, "type": "SignatureNode" }""" yield @add -async def async_generator_function_1( +async def async_generator_function_4( posonly, /, pos_or_k, pos_ok_k_d=1, *varargs, **varkwargs ): """{ "kind": "function", "parameters": [ { - "annotation": null, - "default": null, + "annotation": { + "type": "Empty" + }, + "default": { + "type": "Empty" + }, "kind": "POSITIONAL_ONLY", "name": "posonly", "type": "ParameterNode" }, { - "annotation": null, - "default": null, + "annotation": { + "type": "Empty" + }, + "default": { + "type": "Empty" + }, "kind": "POSITIONAL_OR_KEYWORD", "name": "pos_or_k", "type": "ParameterNode" }, { - "annotation": null, - "default": null, + "annotation": { + "type": "Empty" + }, + "default": { + "data": "1", + "type": "str" + }, "kind": "POSITIONAL_OR_KEYWORD", "name": "pos_ok_k_d", "type": "ParameterNode" }, { - "annotation": null, - "default": null, + "annotation": { + "type": "Empty" + }, + "default": { + "type": "Empty" + }, "kind": "VAR_POSITIONAL", "name": "varargs", "type": "ParameterNode" }, { - "annotation": null, - "default": null, + "annotation": { + "type": "Empty" + }, + "default": { + "type": "Empty" + }, "kind": "VAR_KEYWORD", "name": "varkwargs", "type": "ParameterNode" } ], + "return_annotation": {"type": "Empty"}, "type": "SignatureNode" }""" yield +@add +def function_with_annotation5(a: int, b: Union[int, float]) -> Optional[bool]: + """ + { + "kind": "function", + "parameters": [ + { + "annotation": { + "data": "int", + "type": "str" + }, + "default": { + "type": "Empty" + }, + "kind": "POSITIONAL_OR_KEYWORD", + "name": "a", + "type": "ParameterNode" + }, + { + "annotation": { + "data": "Union[int, float]", + "type": "str" + }, + "default": { + "type": "Empty" + }, + "kind": "POSITIONAL_OR_KEYWORD", + "name": "b", + "type": "ParameterNode" + } + ], + "return_annotation": { + "data": "typing.Optional[bool]", + "type": "str" + }, + "type": "SignatureNode" + } + + """ + pass + + @pytest.mark.parametrize( "func", all_funcs, @@ -211,6 +341,8 @@ def test_f1(func): so = SignatureObject(func) node = so.to_node() bytes_ = node.to_json() - assert json.loads(bytes_) == json.loads(func.__doc__) + assert json.dumps(json.loads(bytes_), indent=2) == json.dumps( + json.loads(func.__doc__), indent=2 + ) node_back = SignatureNode.from_json(bytes_) assert node_back == node diff --git a/pyproject.toml b/pyproject.toml index 8e73c73d..d283ad0f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,7 +16,7 @@ dependencies = [ "tomli_w", "urwid", "httpx", - "jinja2<3.1", + "jinja2>=3.1.2", "ipython", "numpydoc", "cachetools", @@ -27,8 +27,8 @@ dependencies = [ "jedi", "typer", "velin", - "quart-trio", - "quart<0.14", + "quart-trio>=0.6.0", + "quart>=0.18.4", "flatlatex", # "tree_sitter", # tree sitter does not currently provide builds for all platform. tree-sitter-builds is an # alternative build that provide more wheels. diff --git a/pytest.ini b/pytest.ini index e7015006..62deafe0 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,4 +1,3 @@ [pytest] addopts = --durations=10 -trio_mode = true