diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 7dd104af..60edc1d0 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -7,6 +7,18 @@ on: branches: '*' jobs: + + pre-commit: + name: Run pre-commit formatters and linters + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Set up Python 3.8 + uses: actions/setup-python@v1 + with: + python-version: 3.8 + - uses: pre-commit/action@v2.0.0 + build-n-test-n-coverage: name: Build, test and code coverage runs-on: ${{ matrix.os }} diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 00000000..8ebb1dce --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,40 @@ +ci: + autoupdate_schedule: monthly + autofix_prs: true + +repos: + + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.0.1 + hooks: + - id: check-yaml + - id: end-of-file-fixer + - id: trailing-whitespace + # - id: check-json + + - repo: https://github.com/mgedmin/check-manifest + rev: "0.46" + hooks: + - id: check-manifest + additional_dependencies: [setuptools>=46.4.0] + + - repo: https://github.com/pycqa/isort + rev: 5.9.3 + hooks: + - id: isort + + - repo: https://github.com/asottile/pyupgrade + rev: v2.25.0 + hooks: + - id: pyupgrade + args: [--py36-plus] + + - repo: https://github.com/psf/black + rev: 21.7b0 + hooks: + - id: black + + - repo: https://gitlab.com/pycqa/flake8 + rev: 3.9.1 + hooks: + - id: flake8 diff --git a/CHANGELOG.md b/CHANGELOG.md index 8e25eb62..dfce2c1b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,3 @@ # Change Log -See the Change Log in the [nbclient documentation](https://nbclient.readthedocs.io/en/latest/changelog.html). \ No newline at end of file +See the Change Log in the [nbclient documentation](https://nbclient.readthedocs.io/en/latest/changelog.html). diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 4ec1d52d..70165cc2 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -4,16 +4,38 @@ We follow the [Jupyter Contribution Workflow](https://jupyter.readthedocs.io/en/latest/contributor/content-contributor.html) and the [IPython Contributing Guide](https://github.com/ipython/ipython/blob/master/CONTRIBUTING.md). -# Testing +## Code formatting + +Use the [pre-commit](https://pre-commit.com/) tool to format and lint the codebase: + +```console +# to apply to only staged files +$ pre-commit run +# to run against all files +$ pre-commit run --all-files +# to install so that it is run before commits +$ pre-commit install +``` + +## Testing In your environment `pip install -e '.[test]'` will be needed to be able to run all of the tests. -# Documentation +The recommended way to do this is using [tox](https://tox.readthedocs.io/en/latest/): + +```console +# to list all environments +$ tox -av +# to run all tests for a specific environment +$ tox -e py38 +``` + +## Documentation NbClient needs some PRs to copy over documentation! -# Releasing +## Releasing If you are going to release a version of `nbclient` you should also be capable of testing it and building the docs. diff --git a/MANIFEST.in b/MANIFEST.in index e3e313a9..78d74883 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -5,8 +5,8 @@ include requirements-dev.txt include *.md include .bumpversion.cfg include tox.ini -include mypy.ini include pyproject.toml +include .pre-commit-config.yaml # Code and test files recursive-include nbclient *.ipynb diff --git a/docs/Makefile b/docs/Makefile index 65fbd609..48c91ebc 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -17,4 +17,4 @@ help: # Catch-all target: route all unknown targets to Sphinx using the new # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). %: Makefile - @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) \ No newline at end of file + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/docs/changelog.md b/docs/changelog.md index 0f85a1f5..ff47aebf 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -7,7 +7,7 @@ - Support parsing of IPython dev version [#150](https://github.com/jupyter/nbclient/pull/150) ([@cphyc](https://github.com/cphyc)) - Set `IPYKERNEL_CELL_NAME = ` [#147](https://github.com/jupyter/nbclient/pull/147) ([@davidbrochart](https://github.com/davidbrochart)) - Print useful error message on exception [#142](https://github.com/jupyter/nbclient/pull/142) ([@certik](https://github.com/certik)) - + ## 0.5.3 - Fix ipykernel's `stop_on_error` value to take into account `raises-exception` tag and `force_raise_errors` [#137](https://github.com/jupyter/nbclient/pull/137) diff --git a/docs/conf.py b/docs/conf.py index 57fa6a66..a1b6bd62 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- # # nbclient documentation build configuration file, created by # sphinx-quickstart on Mon Jan 26 16:00:00 2020. @@ -20,6 +19,8 @@ import os import sys +import nbclient + sys.path.insert(0, os.path.abspath('..')) # -- General configuration ------------------------------------------------ @@ -61,7 +62,6 @@ # |version| and |release|, also used in various other places throughout the # built documents. # -import nbclient # The short X.Y version. version = '.'.join(nbclient.__version__.split('.')[0:2]) @@ -115,7 +115,7 @@ # Custom sidebar templates, must be a dictionary that maps document names # to template names. -#html_sidebars = {} +# html_sidebars = {} html_title = "nbclient" diff --git a/mypy.ini b/mypy.ini deleted file mode 100644 index c218c2df..00000000 --- a/mypy.ini +++ /dev/null @@ -1,24 +0,0 @@ -# Global options: - -[mypy] -python_version = 3.9 - -# Per-module options: - -[mypy-ipython_genutils.*] -ignore_missing_imports = True - -[mypy-nbformat.*] -ignore_missing_imports = True - -[mypy-nest_asyncio.*] -ignore_missing_imports = True - -[mypy-async_generator.*] -ignore_missing_imports = True - -[mypy-traitlets.*] -ignore_missing_imports = True - -[mypy-jupyter_client.*] -ignore_missing_imports = True diff --git a/nbclient/__init__.py b/nbclient/__init__.py index 6110e8f6..b12f9a40 100644 --- a/nbclient/__init__.py +++ b/nbclient/__init__.py @@ -1,7 +1,8 @@ -import sys import subprocess -from .client import NotebookClient, execute # noqa: F401 +import sys + from ._version import version as __version__ # noqa: F401 +from .client import NotebookClient, execute # noqa: F401 def _cleanup() -> None: diff --git a/nbclient/client.py b/nbclient/client.py index 99103087..eee687bb 100644 --- a/nbclient/client.py +++ b/nbclient/client.py @@ -1,7 +1,7 @@ import atexit +import base64 import collections import datetime -import base64 import signal from textwrap import dedent @@ -11,30 +11,29 @@ # Use the backport package async-generator for Python < 3.7. # This should be removed when nbclient drops support for Python 3.6 from async_generator import asynccontextmanager # type: ignore -from contextlib import contextmanager -from time import monotonic -from queue import Empty import asyncio import typing as t +from contextlib import contextmanager +from queue import Empty +from time import monotonic -from traitlets.config.configurable import LoggingConfigurable -from traitlets import List, Unicode, Bool, Enum, Any, Type, Dict, Integer, default - -from nbformat import NotebookNode -from nbformat.v4 import output_from_msg from jupyter_client import KernelManager from jupyter_client.client import KernelClient +from nbformat import NotebookNode +from nbformat.v4 import output_from_msg +from traitlets import Any, Bool, Dict, Enum, Integer, List, Type, Unicode, default +from traitlets.config.configurable import LoggingConfigurable from .exceptions import ( CellControlSignal, + CellExecutionComplete, + CellExecutionError, CellTimeoutError, DeadKernelError, - CellExecutionComplete, - CellExecutionError ) -from .util import run_sync, ensure_async from .output_widget import OutputWidget +from .util import ensure_async, run_sync def timestamp() -> str: @@ -295,11 +294,7 @@ def _kernel_manager_class_default(self) -> KernelManager: ) ) - def __init__( - self, - nb: NotebookNode, - km: t.Optional[KernelManager] = None, - **kw) -> None: + def __init__(self, nb: NotebookNode, km: t.Optional[KernelManager] = None, **kw) -> None: """Initializes the execution manager. Parameters @@ -317,9 +312,7 @@ def __init__( self.kc: t.Optional[KernelClient] = None self.reset_execution_trackers() self.widget_registry: t.Dict[str, t.Dict] = { - '@jupyter-widgets/output': { - 'OutputModel': OutputWidget - } + '@jupyter-widgets/output': {'OutputModel': OutputWidget} } # comm_open_handlers should return an object with a .handle_msg(msg) method or None self.comm_open_handlers: t.Dict[str, t.Any] = { @@ -327,8 +320,7 @@ def __init__( } def reset_execution_trackers(self) -> None: - """Resets any per-execution trackers. - """ + """Resets any per-execution trackers.""" self.task_poll_for_reply: t.Optional[asyncio.Future] = None self.code_cells_executed = 0 self._display_id_map = {} @@ -402,12 +394,15 @@ async def async_start_new_kernel(self, **kwargs) -> None: kwargs["cwd"] = resource_path has_history_manager_arg = any( - arg.startswith('--HistoryManager.hist_file') for arg in self.extra_arguments) - if (hasattr(self.km, 'ipykernel') - and self.km.ipykernel - and self.ipython_hist_file - and not has_history_manager_arg): - self.extra_arguments += ['--HistoryManager.hist_file={}'.format(self.ipython_hist_file)] + arg.startswith('--HistoryManager.hist_file') for arg in self.extra_arguments + ) + if ( + hasattr(self.km, 'ipykernel') + and self.km.ipykernel + and self.ipython_hist_file + and not has_history_manager_arg + ): + self.extra_arguments += [f'--HistoryManager.hist_file={self.ipython_hist_file}'] await ensure_async(self.km.start_kernel(extra_arguments=self.extra_arguments, **kwargs)) @@ -512,10 +507,7 @@ def on_signal(): except (NotImplementedError, RuntimeError): pass - async def async_execute( - self, - reset_kc: bool = False, - **kwargs) -> NotebookNode: + async def async_execute(self, reset_kc: bool = False, **kwargs) -> NotebookNode: """ Executes each code cell. @@ -584,10 +576,7 @@ def set_widgets_metadata(self) -> None: if buffers: widget['buffers'] = buffers - def _update_display_id( - self, - display_id: str, - msg: t.Dict) -> None: + def _update_display_id(self, display_id: str, msg: t.Dict) -> None: """Update outputs with a given display_id""" if display_id not in self._display_id_map: self.log.debug("display id %r not in %s", display_id, self._display_id_map) @@ -610,12 +599,13 @@ def _update_display_id( outputs[output_idx]['metadata'] = out['metadata'] async def _async_poll_for_reply( - self, - msg_id: str, - cell: NotebookNode, - timeout: t.Optional[int], - task_poll_output_msg: asyncio.Future, - task_poll_kernel_alive: asyncio.Future) -> t.Dict: + self, + msg_id: str, + cell: NotebookNode, + timeout: t.Optional[int], + task_poll_output_msg: asyncio.Future, + task_poll_kernel_alive: asyncio.Future, + ) -> t.Dict: assert self.kc is not None new_timeout: t.Optional[float] = None @@ -651,10 +641,8 @@ async def _async_poll_for_reply( await self._async_handle_timeout(timeout, cell) async def _async_poll_output_msg( - self, - parent_msg_id: str, - cell: NotebookNode, - cell_index: int) -> None: + self, parent_msg_id: str, cell: NotebookNode, cell_index: int + ) -> None: assert self.kc is not None while True: @@ -688,9 +676,8 @@ def _get_timeout(self, cell: t.Optional[NotebookNode]) -> int: return timeout async def _async_handle_timeout( - self, - timeout: int, - cell: t.Optional[NotebookNode] = None) -> None: + self, timeout: int, cell: t.Optional[NotebookNode] = None + ) -> None: self.log.error("Timeout waiting for execute reply (%is)." % timeout) if self.interrupt_on_timeout: @@ -709,9 +696,8 @@ async def _async_check_alive(self) -> None: raise DeadKernelError("Kernel died") async def async_wait_for_reply( - self, - msg_id: str, - cell: t.Optional[NotebookNode] = None) -> t.Optional[t.Dict]: + self, msg_id: str, cell: t.Optional[NotebookNode] = None + ) -> t.Optional[t.Dict]: assert self.kc is not None # wait for finish, with timeout @@ -720,9 +706,7 @@ async def async_wait_for_reply( while True: try: msg = await ensure_async( - self.kc.shell_channel.get_msg( - timeout=self.shell_timeout_interval - ) + self.kc.shell_channel.get_msg(timeout=self.shell_timeout_interval) ) except Empty: await self._async_check_alive() @@ -744,10 +728,7 @@ def _passed_deadline(self, deadline: int) -> bool: return True return False - def _check_raise_for_error( - self, - cell: NotebookNode, - exec_reply: t.Optional[t.Dict]) -> None: + def _check_raise_for_error(self, cell: NotebookNode, exec_reply: t.Optional[t.Dict]) -> None: if exec_reply is None: return None @@ -759,17 +740,19 @@ def _check_raise_for_error( cell_allows_errors = (not self.force_raise_errors) and ( self.allow_errors or exec_reply_content.get('ename') in self.allow_error_names - or "raises-exception" in cell.metadata.get("tags", [])) + or "raises-exception" in cell.metadata.get("tags", []) + ) if not cell_allows_errors: raise CellExecutionError.from_cell_and_msg(cell, exec_reply_content) async def async_execute_cell( - self, - cell: NotebookNode, - cell_index: int, - execution_count: t.Optional[int] = None, - store_history: bool = True) -> NotebookNode: + self, + cell: NotebookNode, + cell_index: int, + execution_count: t.Optional[int] = None, + store_history: bool = True, + ) -> NotebookNode: """ Executes a single code cell. @@ -814,14 +797,12 @@ async def async_execute_cell( self.log.debug("Executing cell:\n%s", cell.source) cell_allows_errors = (not self.force_raise_errors) and ( - self.allow_errors - or "raises-exception" in cell.metadata.get("tags", [])) + self.allow_errors or "raises-exception" in cell.metadata.get("tags", []) + ) parent_msg_id = await ensure_async( self.kc.execute( - cell.source, - store_history=store_history, - stop_on_error=not cell_allows_errors + cell.source, store_history=store_history, stop_on_error=not cell_allows_errors ) ) # We launched a code cell to execute @@ -831,9 +812,7 @@ async def async_execute_cell( cell.outputs = [] self.clear_before_next_output = False - task_poll_kernel_alive = asyncio.ensure_future( - self._async_poll_kernel_alive() - ) + task_poll_kernel_alive = asyncio.ensure_future(self._async_poll_kernel_alive()) task_poll_output_msg = asyncio.ensure_future( self._async_poll_output_msg(parent_msg_id, cell, cell_index) ) @@ -866,10 +845,8 @@ async def async_execute_cell( execute_cell = run_sync(async_execute_cell) def process_message( - self, - msg: t.Dict, - cell: NotebookNode, - cell_index: int) -> t.Optional[t.List]: + self, msg: t.Dict, cell: NotebookNode, cell_index: int + ) -> t.Optional[t.List]: """ Processes a kernel message, updates cell state, and returns the resulting output object that was appended to cell.outputs. @@ -932,11 +909,8 @@ def process_message( return None def output( - self, - outs: t.List, - msg: t.Dict, - display_id: str, - cell_index: int) -> t.Optional[t.List]: + self, outs: t.List, msg: t.Dict, display_id: str, cell_index: int + ) -> t.Optional[t.List]: msg_type = msg['msg_type'] @@ -971,11 +945,7 @@ def output( return out - def clear_output( - self, - outs: t.List, - msg: t.Dict, - cell_index: int) -> None: + def clear_output(self, outs: t.List, msg: t.Dict, cell_index: int) -> None: content = msg['content'] @@ -995,19 +965,13 @@ def clear_output( outs[:] = [] self.clear_display_id_mapping(cell_index) - def clear_display_id_mapping( - self, - cell_index: int) -> None: + def clear_display_id_mapping(self, cell_index: int) -> None: for display_id, cell_map in self._display_id_map.items(): if cell_index in cell_map: cell_map[cell_index] = [] - def handle_comm_msg( - self, - outs: t.List, - msg: t.Dict, - cell_index: int) -> None: + def handle_comm_msg(self, outs: t.List, msg: t.Dict, cell_index: int) -> None: content = msg['content'] data = content['data'] @@ -1056,10 +1020,7 @@ def _get_buffer_data(self, msg: t.Dict) -> t.List[t.Dict[str, str]]: ) return encoded_buffers - def register_output_hook( - self, - msg_id: str, - hook: OutputWidget) -> None: + def register_output_hook(self, msg_id: str, hook: OutputWidget) -> None: """Registers an override object that handles output/clear_output instead. Multiple hooks can be registered, where the last one will be used (stack based) @@ -1068,10 +1029,7 @@ def register_output_hook( # https://jupyterlab.github.io/jupyterlab/services/interfaces/kernel.ikernelconnection.html#registermessagehook self.output_hook_stack[msg_id].append(hook) - def remove_output_hook( - self, - msg_id: str, - hook: OutputWidget) -> None: + def remove_output_hook(self, msg_id: str, hook: OutputWidget) -> None: """Unregisters an override object that handles output/clear_output instead""" # mimics # https://jupyterlab.github.io/jupyterlab/services/interfaces/kernel.ikernelconnection.html#removemessagehook @@ -1091,10 +1049,8 @@ def on_comm_open_jupyter_widget(self, msg: t.Dict): def execute( - nb: NotebookNode, - cwd: t.Optional[str] = None, - km: t.Optional[KernelManager] = None, - **kwargs) -> NotebookClient: + nb: NotebookNode, cwd: t.Optional[str] = None, km: t.Optional[KernelManager] = None, **kwargs +) -> NotebookClient: """Execute a notebook's code, updating outputs within the notebook object. This is a convenient wrapper around NotebookClient. It returns the diff --git a/nbclient/exceptions.py b/nbclient/exceptions.py index 0376ed7e..1ffc0afa 100644 --- a/nbclient/exceptions.py +++ b/nbclient/exceptions.py @@ -9,6 +9,7 @@ class CellControlSignal(Exception): control actions (not the best model, but it's needed to cover existing behavior without major refactors). """ + pass @@ -18,17 +19,13 @@ class CellTimeoutError(TimeoutError, CellControlSignal): """ @classmethod - def error_from_timeout_and_cell( - cls, - msg: str, - timeout: int, - cell: NotebookNode): + def error_from_timeout_and_cell(cls, msg: str, timeout: int, cell: NotebookNode): if cell and cell.source: src_by_lines = cell.source.strip().split("\n") src = ( cell.source if len(src_by_lines) < 11 - else "{}\n...\n{}".format(src_by_lines[:5], src_by_lines[-5:]) + else f"{src_by_lines[:5]}\n...\n{src_by_lines[-5:]}" ) else: src = "Cell contents not found." @@ -58,12 +55,8 @@ class CellExecutionError(CellControlSignal): failures gracefully. """ - def __init__( - self, - traceback: str, - ename: str, - evalue: str) -> None: - super(CellExecutionError, self).__init__(traceback) + def __init__(self, traceback: str, ename: str, evalue: str) -> None: + super().__init__(traceback) self.traceback = traceback self.ename = ename self.evalue = evalue @@ -81,10 +74,7 @@ def __unicode__(self) -> str: return self.traceback @classmethod - def from_cell_and_msg( - cls, - cell: NotebookNode, - msg: Dict): + def from_cell_and_msg(cls, cell: NotebookNode, msg: Dict): """Instantiate from a code cell object and a message contents (message is either execute_reply or error) """ @@ -97,11 +87,11 @@ def from_cell_and_msg( evalue=msg.get('evalue', ''), ), ename=msg.get('ename', ''), - evalue=msg.get('evalue', '') + evalue=msg.get('evalue', ''), ) -exec_err_msg: str = u"""\ +exec_err_msg: str = """\ An error occurred while executing the following cell: ------------------ {cell.source} @@ -112,7 +102,7 @@ def from_cell_and_msg( """ -timeout_err_msg: str = u"""\ +timeout_err_msg: str = """\ A cell timed out while it was being executed, after {timeout} seconds. The message was: {msg}. Here is a preview of the cell contents: diff --git a/nbclient/jsonutil.py b/nbclient/jsonutil.py index 7bd52ddc..9a6e0a74 100644 --- a/nbclient/jsonutil.py +++ b/nbclient/jsonutil.py @@ -5,17 +5,16 @@ # Copyright (c) IPython Development Team. # Distributed under the terms of the Modified BSD License. -from binascii import b2a_base64 import math +import numbers import re import types +from binascii import b2a_base64 from datetime import datetime -import numbers from typing import Dict - from ipython_genutils import py3compat -from ipython_genutils.py3compat import unicode_type, iteritems +from ipython_genutils.py3compat import iteritems, unicode_type next_attr_name = '__next__' if py3compat.PY3 else 'next' diff --git a/nbclient/output_widget.py b/nbclient/output_widget.py index 0f14afc8..0be7c65c 100644 --- a/nbclient/output_widget.py +++ b/nbclient/output_widget.py @@ -1,18 +1,17 @@ -from .jsonutil import json_clean -from nbformat.v4 import output_from_msg -from typing import Dict, List, Any, Optional +from typing import Any, Dict, List, Optional from jupyter_client.client import KernelClient +from nbformat.v4 import output_from_msg + +from .jsonutil import json_clean class OutputWidget: """This class mimics a front end output widget""" + def __init__( - self, - comm_id: str, - state: Dict[str, Any], - kernel_client: KernelClient, - executor) -> None: + self, comm_id: str, state: Dict[str, Any], kernel_client: KernelClient, executor + ) -> None: self.comm_id: str = comm_id self.state: Dict[str, Any] = state @@ -22,11 +21,7 @@ def __init__( self.outputs: List = self.state['outputs'] self.clear_before_next_output: bool = False - def clear_output( - self, - outs: List, - msg: Dict, - cell_index: int) -> None: + def clear_output(self, outs: List, msg: Dict, cell_index: int) -> None: self.parent_header = msg['parent_header'] content = msg['content'] @@ -46,34 +41,32 @@ def sync_state(self) -> None: self.send(msg) def _publish_msg( - self, - msg_type: str, - data: Optional[Dict] = None, - metadata: Optional[Dict] = None, - buffers: Optional[List] = None, - **keys) -> None: + self, + msg_type: str, + data: Optional[Dict] = None, + metadata: Optional[Dict] = None, + buffers: Optional[List] = None, + **keys + ) -> None: """Helper for sending a comm message on IOPub""" data = {} if data is None else data metadata = {} if metadata is None else metadata content = json_clean(dict(data=data, comm_id=self.comm_id, **keys)) - msg = self.kernel_client.session.msg(msg_type, content=content, parent=self.parent_header, - metadata=metadata) + msg = self.kernel_client.session.msg( + msg_type, content=content, parent=self.parent_header, metadata=metadata + ) self.kernel_client.shell_channel.send(msg) def send( - self, - data: Optional[Dict] = None, - metadata: Optional[Dict] = None, - buffers: Optional[List] = None) -> None: + self, + data: Optional[Dict] = None, + metadata: Optional[Dict] = None, + buffers: Optional[List] = None, + ) -> None: self._publish_msg('comm_msg', data=data, metadata=metadata, buffers=buffers) - def output( - self, - outs: List, - msg: Dict, - display_id: str, - cell_index: int) -> None: + def output(self, outs: List, msg: Dict, display_id: str, cell_index: int) -> None: if self.clear_before_next_output: self.outputs = [] @@ -84,9 +77,11 @@ def output( if self.outputs: # try to coalesce/merge output text last_output = self.outputs[-1] - if (last_output['output_type'] == 'stream' - and output['output_type'] == 'stream' - and last_output['name'] == output['name']): + if ( + last_output['output_type'] == 'stream' + and output['output_type'] == 'stream' + and last_output['name'] == output['name'] + ): last_output['text'] += output['text'] else: self.outputs.append(output) diff --git a/nbclient/tests/base.py b/nbclient/tests/base.py index 2aa142cf..69ee12ca 100644 --- a/nbclient/tests/base.py +++ b/nbclient/tests/base.py @@ -4,7 +4,6 @@ class NBClientTestsBase(unittest.TestCase): - def build_notebook(self, with_json_outputs=False): """Build a notebook in memory for use with NotebookClient tests""" diff --git a/nbclient/tests/fake_kernelmanager.py b/nbclient/tests/fake_kernelmanager.py index 893f9176..a24daa52 100644 --- a/nbclient/tests/fake_kernelmanager.py +++ b/nbclient/tests/fake_kernelmanager.py @@ -7,14 +7,14 @@ class FakeCustomKernelManager(AsyncKernelManager): def __init__(self, *args, **kwargs): self.log.info('FakeCustomKernelManager initialized') self.expected_methods['__init__'] += 1 - super(FakeCustomKernelManager, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) async def start_kernel(self, *args, **kwargs): self.log.info('FakeCustomKernelManager started a kernel') self.expected_methods['start_kernel'] += 1 - return await super(FakeCustomKernelManager, self).start_kernel(*args, **kwargs) + return await super().start_kernel(*args, **kwargs) def client(self, *args, **kwargs): self.log.info('FakeCustomKernelManager created a client') self.expected_methods['client'] += 1 - return super(FakeCustomKernelManager, self).client(*args, **kwargs) + return super().client(*args, **kwargs) diff --git a/nbclient/tests/files/Factorials.ipynb b/nbclient/tests/files/Factorials.ipynb index 939c49f4..c63883c9 100644 --- a/nbclient/tests/files/Factorials.ipynb +++ b/nbclient/tests/files/Factorials.ipynb @@ -45,4 +45,4 @@ "metadata": {}, "nbformat": 4, "nbformat_minor": 0 -} \ No newline at end of file +} diff --git a/nbclient/tests/files/HelloWorld.ipynb b/nbclient/tests/files/HelloWorld.ipynb index 9b8f550b..6464da1e 100644 --- a/nbclient/tests/files/HelloWorld.ipynb +++ b/nbclient/tests/files/HelloWorld.ipynb @@ -23,4 +23,4 @@ "metadata": {}, "nbformat": 4, "nbformat_minor": 0 -} \ No newline at end of file +} diff --git a/nbclient/tests/files/SVG.ipynb b/nbclient/tests/files/SVG.ipynb index 2769ab2b..22f8530d 100644 --- a/nbclient/tests/files/SVG.ipynb +++ b/nbclient/tests/files/SVG.ipynb @@ -45,4 +45,4 @@ "metadata": {}, "nbformat": 4, "nbformat_minor": 0 -} \ No newline at end of file +} diff --git a/nbclient/tests/files/Unicode.ipynb b/nbclient/tests/files/Unicode.ipynb index cdbcb10a..5f1b44d4 100644 --- a/nbclient/tests/files/Unicode.ipynb +++ b/nbclient/tests/files/Unicode.ipynb @@ -23,4 +23,4 @@ "metadata": {}, "nbformat": 4, "nbformat_minor": 0 -} \ No newline at end of file +} diff --git a/nbclient/tests/test_client.py b/nbclient/tests/test_client.py index cb205dcf..f0635e7a 100644 --- a/nbclient/tests/test_client.py +++ b/nbclient/tests/test_client.py @@ -1,35 +1,30 @@ -from base64 import b64encode, b64decode +import asyncio +import concurrent.futures import copy -import io +import datetime +import functools import os import re import threading -import asyncio -import datetime import warnings +from base64 import b64decode, b64encode +from queue import Empty +from unittest.mock import MagicMock, Mock import nbformat -import sys import pytest -import functools import xmltodict - -from .base import NBClientTestsBase -from .. import NotebookClient, execute -from ..exceptions import CellExecutionError - -from traitlets import TraitError -from nbformat import NotebookNode +from ipython_genutils.py3compat import string_types from jupyter_client import KernelManager from jupyter_client.kernelspec import KernelSpecManager from nbconvert.filters import strip_ansi +from nbformat import NotebookNode from testpath import modified_env -from ipython_genutils.py3compat import string_types -import concurrent.futures - -from queue import Empty -from unittest.mock import MagicMock, Mock +from traitlets import TraitError +from .. import NotebookClient, execute +from ..exceptions import CellExecutionError +from .base import NBClientTestsBase addr_pat = re.compile(r'0x[0-9a-f]{7,9}') ipython_input_pat = re.compile(r'') @@ -43,6 +38,7 @@ class AsyncMock(Mock): def make_async(mock_value): async def _(): return mock_value + return _() @@ -60,7 +56,7 @@ def run_notebook(filename, opts, resources=None): running it and the version after running it. """ - with io.open(filename) as f: + with open(filename) as f: input_nb = nbformat.read(f, 4) cleaned_input_nb = copy.deepcopy(input_nb) @@ -94,7 +90,7 @@ async def async_run_notebook(filename, opts, resources=None): running it and the version after running it. """ - with io.open(filename) as f: + with open(filename) as f: input_nb = nbformat.read(f, 4) cleaned_input_nb = copy.deepcopy(input_nb) @@ -130,13 +126,15 @@ def shell_channel_message_mock(): # Return the message generator for # self.kc.shell_channel.get_msg => {'parent_header': {'msg_id': parent_id}} return AsyncMock( - return_value=make_async(NBClientTestsBase.merge_dicts( - { - 'parent_header': {'msg_id': parent_id}, - 'content': {'status': 'ok', 'execution_count': 1}, - }, - reply_msg or {}, - )) + return_value=make_async( + NBClientTestsBase.merge_dicts( + { + 'parent_header': {'msg_id': parent_id}, + 'content': {'status': 'ok', 'execution_count': 1}, + }, + reply_msg or {}, + ) + ) ) def iopub_messages_mock(): @@ -174,7 +172,7 @@ def test_mock_wrapper(self): iopub_channel=MagicMock(get_msg=message_mock), shell_channel=MagicMock(get_msg=shell_channel_message_mock()), execute=MagicMock(return_value=parent_id), - is_alive=MagicMock(return_value=make_async(True)) + is_alive=MagicMock(return_value=make_async(True)), ) executor.parent_id = parent_id return func(self, executor, cell_mock, message_mock) @@ -333,8 +331,7 @@ def test_async_parallel_notebooks(capfd, tmpdir): with modified_env({"NBEXECUTE_TEST_PARALLEL_TMPDIR": str(tmpdir)}): tasks = [ - async_run_notebook(input_file.format(label=label), opts, res) - for label in ("A", "B") + async_run_notebook(input_file.format(label=label), opts, res) for label in ("A", "B") ] loop = asyncio.get_event_loop() loop.run_until_complete(asyncio.gather(*tasks)) @@ -358,10 +355,7 @@ def test_many_async_parallel_notebooks(capfd): # run once, to trigger creating the original context run_notebook(input_file, opts, res) - tasks = [ - async_run_notebook(input_file, opts, res) - for i in range(4) - ] + tasks = [async_run_notebook(input_file, opts, res) for i in range(4)] loop = asyncio.get_event_loop() loop.run_until_complete(asyncio.gather(*tasks)) @@ -540,7 +534,7 @@ def timeout_func(source): def test_kernel_death_after_timeout(self): """Check that an error is raised when the kernel is_alive is false after a cell timed out""" filename = os.path.join(current_dir, 'files', 'Interrupt.ipynb') - with io.open(filename, 'r') as f: + with open(filename) as f: input_nb = nbformat.read(f, 4) res = self.build_resources() res['metadata']['path'] = os.path.dirname(filename) @@ -553,6 +547,7 @@ def test_kernel_death_after_timeout(self): async def is_alive(): return False + km.is_alive = is_alive # Will be a RuntimeError or subclass DeadKernelError depending # on if jupyter_client or nbconvert catches the dead client first @@ -564,7 +559,7 @@ def test_kernel_death_during_execution(self): execution. """ filename = os.path.join(current_dir, 'files', 'Autokill.ipynb') - with io.open(filename, 'r') as f: + with open(filename) as f: input_nb = nbformat.read(f, 4) executor = NotebookClient(input_nb) @@ -582,10 +577,7 @@ def test_allow_errors(self): with pytest.raises(CellExecutionError) as exc: run_notebook(filename, dict(allow_errors=False), res) self.assertIsInstance(str(exc.value), str) - if sys.version_info >= (3, 0): - assert u"# üñîçø∂é" in str(exc.value) - else: - assert u"# üñîçø∂é".encode('utf8', 'replace') in str(exc.value) + assert "# üñîçø∂é" in str(exc.value) def test_force_raise_errors(self): """ @@ -598,15 +590,12 @@ def test_force_raise_errors(self): with pytest.raises(CellExecutionError) as exc: run_notebook(filename, dict(force_raise_errors=True), res) self.assertIsInstance(str(exc.value), str) - if sys.version_info >= (3, 0): - assert u"# üñîçø∂é" in str(exc.value) - else: - assert u"# üñîçø∂é".encode('utf8', 'replace') in str(exc.value) + assert "# üñîçø∂é" in str(exc.value) def test_reset_kernel_client(self): filename = os.path.join(current_dir, 'files', 'HelloWorld.ipynb') - with io.open(filename) as f: + with open(filename) as f: input_nb = nbformat.read(f, 4) executor = NotebookClient( @@ -631,7 +620,7 @@ def test_reset_kernel_client(self): def test_cleanup_kernel_client(self): filename = os.path.join(current_dir, 'files', 'HelloWorld.ipynb') - with io.open(filename) as f: + with open(filename) as f: input_nb = nbformat.read(f, 4) executor = NotebookClient( @@ -653,7 +642,7 @@ def test_custom_kernel_manager(self): filename = os.path.join(current_dir, 'files', 'HelloWorld.ipynb') - with io.open(filename) as f: + with open(filename) as f: input_nb = nbformat.read(f, 4) cleaned_input_nb = copy.deepcopy(input_nb) @@ -675,14 +664,14 @@ def test_custom_kernel_manager(self): expected = FakeCustomKernelManager.expected_methods.items() for method, call_count in expected: - self.assertNotEqual(call_count, 0, '{} was called'.format(method)) + self.assertNotEqual(call_count, 0, f'{method} was called') def test_process_message_wrapper(self): outputs = [] class WrappedPreProc(NotebookClient): def process_message(self, msg, cell, cell_index): - result = super(WrappedPreProc, self).process_message(msg, cell, cell_index) + result = super().process_message(msg, cell, cell_index) if result: outputs.append(result) return result @@ -690,7 +679,7 @@ def process_message(self, msg, cell, cell_index): current_dir = os.path.dirname(__file__) filename = os.path.join(current_dir, 'files', 'HelloWorld.ipynb') - with io.open(filename) as f: + with open(filename) as f: input_nb = nbformat.read(f, 4) original = copy.deepcopy(input_nb) @@ -703,7 +692,7 @@ def test_execute_function(self): # Test the execute() convenience API filename = os.path.join(current_dir, 'files', 'HelloWorld.ipynb') - with io.open(filename) as f: + with open(filename) as f: input_nb = nbformat.read(f, 4) original = copy.deepcopy(input_nb) @@ -836,10 +825,10 @@ def test_deadline_iopub(self, executor, cell_mock, message_mock): def test_eventual_deadline_iopub(self, executor, cell_mock, message_mock): # Process a few messages before raising a timeout from iopub def message_seq(messages): - for message in messages: - yield message + yield from messages while True: yield Empty() + message_mock.side_effect = message_seq(list(message_mock.side_effect)[:-1]) executor.kc.shell_channel.get_msg = Mock( return_value=make_async({'parent_header': {'msg_id': executor.parent_id}}) diff --git a/nbclient/util.py b/nbclient/util.py index 9b672357..6cee6a5a 100644 --- a/nbclient/util.py +++ b/nbclient/util.py @@ -4,9 +4,9 @@ # Distributed under the terms of the Modified BSD License. import asyncio -import sys import inspect -from typing import Callable, Awaitable, Any, Union +import sys +from typing import Any, Awaitable, Callable, Union def check_ipython() -> None: @@ -19,8 +19,10 @@ def check_ipython() -> None: IPython_version = tuple(map(int, version_str.split('.'))) if IPython_version < (7, 0, 0): - raise RuntimeError(f'You are using IPython {IPython.__version__} ' # type: ignore - 'while we require 7.0.0+, please update IPython') + raise RuntimeError( + f'You are using IPython {IPython.__version__} ' # type: ignore + 'while we require 7.0.0+, please update IPython' + ) def check_patch_tornado() -> None: @@ -28,9 +30,11 @@ def check_patch_tornado() -> None: # original from vaex/asyncio.py if 'tornado' in sys.modules: import tornado.concurrent # type: ignore + if asyncio.Future not in tornado.concurrent.FUTURES: - tornado.concurrent.FUTURES = \ - tornado.concurrent.FUTURES + (asyncio.Future, ) # type: ignore + tornado.concurrent.FUTURES = tornado.concurrent.FUTURES + ( + asyncio.Future, + ) # type: ignore def just_run(coro: Awaitable) -> Any: @@ -52,6 +56,7 @@ def just_run(coro: Awaitable) -> Any: # to have reentrant event loops check_ipython() import nest_asyncio + nest_asyncio.apply() check_patch_tornado() return loop.run_until_complete(coro) @@ -74,8 +79,10 @@ def run_sync(coro: Callable) -> Callable: result : Whatever the coroutine returns. """ + def wrapped(*args, **kwargs): return just_run(coro(*args, **kwargs)) + wrapped.__doc__ = coro.__doc__ return wrapped diff --git a/pyproject.toml b/pyproject.toml index 8bfffb83..f6ae0b9a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -27,3 +27,21 @@ exclude = ''' )/ ''' skip-string-normalization = true + +[tool.isort] +profile = "black" +known_first_party = ["nbclient"] + +[tool.mypy] +python_version = 3.9 + +[[tool.mypy.overrides]] +module = [ + "ipython_genutils.*", + "nbformat.*", + "nest_asyncio.*", + "async_generator.*", + "traitlets.*", + "jupyter_client.*", +] +ignore_missing_imports = true diff --git a/setup.cfg b/setup.cfg index c94e7e17..25ea298b 100644 --- a/setup.cfg +++ b/setup.cfg @@ -7,3 +7,8 @@ license_file = LICENSE [check-manifest] ignore = .circleci* + +[flake8] +ignore = E203,E731,F811,W503 +max-complexity=23 +max-line-length=100 diff --git a/setup.py b/setup.py index 700316c9..1d276068 100644 --- a/setup.py +++ b/setup.py @@ -1,10 +1,10 @@ #!/usr/bin/env python -# coding: utf-8 # Copyright (c) Jupyter Development Team. # Distributed under the terms of the Modified BSD License. import os + from setuptools import setup # the name of the package @@ -18,7 +18,7 @@ def version(): - with open(here + '/nbclient/_version.py', 'r') as ver: + with open(here + '/nbclient/_version.py') as ver: for line in ver.readlines(): if line.startswith('version ='): return line.split(' = ')[-1].strip()[1:-1] @@ -26,7 +26,7 @@ def version(): def read(path): - with open(path, 'r') as fhandle: + with open(path) as fhandle: return fhandle.read() @@ -47,7 +47,9 @@ def read_reqs(fname): author='Jupyter Development Team', author_email='jupyter@googlegroups.com', url='https://jupyter.org', - description="A client library for executing notebooks. Formerly nbconvert's ExecutePreprocessor.", + description=( + "A client library for executing notebooks. Formerly nbconvert's ExecutePreprocessor." + ), long_description=long_description, long_description_content_type='text/markdown', packages=['nbclient'], diff --git a/tox.ini b/tox.ini index 4c3c70e0..b07aeabd 100644 --- a/tox.ini +++ b/tox.ini @@ -1,19 +1,13 @@ [tox] skipsdist = true -envlist = py{36,37,38, 39}, flake8, mypy, dist, manifest, docs +envlist = py{36,37,38, 39}, mypy, dist, docs [gh-actions] python = 3.6: py36 3.7: py37 3.8: py38 - 3.9: py39, flake8, mypy, dist, manifest - -# Linters -[testenv:flake8] -skip_install = true -deps = flake8 -commands = flake8 nbclient --count --ignore=E203,E731,F811,W503 --max-complexity=23 --max-line-length=100 --show-source --statistics + 3.9: py39, mypy, dist # Type check [testenv:mypy] @@ -21,12 +15,6 @@ skip_install = true deps = mypy commands = mypy nbclient/client.py nbclient/exceptions.py nbclient/__init__.py nbclient/jsonutil.py nbclient/output_widget.py nbclient/util.py nbclient/_version.py -# Manifest -[testenv:manifest] -skip_install = true -deps = check-manifest -commands = check-manifest - # Docs [testenv:docs] description = invoke sphinx-build to build the HTML docs @@ -58,9 +46,7 @@ basepython = py37: python3.7 py38: python3.8 py39: python3.9 - flake8: python3.9 mypy: python3.9 - manifest: python3.9 binder: python3.9 dist: python3.9 docs: python3.9