From 70303068e2bfc3b68bd6b931b245b9c440cb92ca Mon Sep 17 00:00:00 2001 From: Jackson Lee <86482098+jacksonlee-civis@users.noreply.github.com> Date: Tue, 11 Jun 2024 12:47:50 -0400 Subject: [PATCH] [CIVIS-2326] MAINT update repo setup and packaging (#57) * MAINT switch to pyproject.toml, add support for python 3.12, drop python 3.8 * MAINT setup_remote_docker * MAINT update ci config and add github templates * FIX include dotfiles in setuptools.package-data * REF refactor legacy python code --- .circleci/config.yml | 111 ++---- .flake8 | 3 +- .github/ISSUE_TEMPLATE/general.md | 11 + .github/PULL_REQUEST_TEMPLATE.md | 9 + CHANGELOG.md | 11 + MANIFEST.in | 5 - README.rst | 14 +- civis_jupyter_notebooks/__init__.py | 8 - civis_jupyter_notebooks/__main__.py | 58 ---- .../assets/civis-git-clone | 15 - .../assets/ipython_config.py | 4 - civis_jupyter_notebooks/log_utils.py | 52 --- .../tests/test_notebook_config.py | 103 ------ .../tests/test_platform_persistence.py | 235 ------------- civis_jupyter_notebooks/tests/test_version.py | 17 - dev-requirements.txt | 2 - pyproject.toml | 64 ++++ requirements.txt | 9 - setup.py | 33 -- src/civis_jupyter_notebooks/__init__.py | 4 + src/civis_jupyter_notebooks/__main__.py | 69 ++++ .../civis_jupyter_notebooks}/assets/.bashrc | 0 .../assets/civis-git-clone | 25 ++ .../assets/civis-jupyter-notebooks-start | 0 .../assets/civis_client_config.py | 6 +- .../assets/custom.css | 0 .../civis_jupyter_notebooks}/assets/custom.js | 0 .../assets/extensions/terminal.js | 0 .../assets/extensions/uncommitted_changes.js | 0 .../assets/fonts/civicons.eot | Bin .../assets/fonts/civicons.svg | 0 .../assets/fonts/civicons.ttf | Bin .../assets/fonts/civicons.woff | Bin .../assets/fonts/config.json | 0 .../assets/initialize-git | 0 .../assets/ipython_config.py | 4 + .../assets/jupyter_notebook_config.py | 10 +- .../extensions/__init__.py | 0 .../extensions/git/__init__.py | 0 .../extensions/git/uncommitted_changes.py | 17 +- .../civis_jupyter_notebooks}/git_utils.py | 12 +- .../notebook_config.py | 28 +- .../platform_persistence.py | 124 ++++--- tests/Dockerfile | 5 +- .../tests => tests}/__init__.py | 0 .../tests => tests}/extensions/__init__.py | 0 .../extensions/git/__init__.py | 0 .../extensions/git/test_uncommited_changes.py | 22 +- .../fixtures/sample_new_notebook.ipynb | 0 .../fixtures/sample_notebook.ipynb | 0 tests/test_ext.py | 2 +- .../tests => tests}/test_git_utils.py | 26 +- tests/test_notebook_config.py | 124 +++++++ tests/test_platform_persistence.py | 327 ++++++++++++++++++ tests/test_version.py | 17 + 55 files changed, 842 insertions(+), 744 deletions(-) create mode 100644 .github/ISSUE_TEMPLATE/general.md create mode 100644 .github/PULL_REQUEST_TEMPLATE.md delete mode 100644 MANIFEST.in delete mode 100644 civis_jupyter_notebooks/__init__.py delete mode 100644 civis_jupyter_notebooks/__main__.py delete mode 100644 civis_jupyter_notebooks/assets/civis-git-clone delete mode 100644 civis_jupyter_notebooks/assets/ipython_config.py delete mode 100644 civis_jupyter_notebooks/log_utils.py delete mode 100644 civis_jupyter_notebooks/tests/test_notebook_config.py delete mode 100644 civis_jupyter_notebooks/tests/test_platform_persistence.py delete mode 100644 civis_jupyter_notebooks/tests/test_version.py delete mode 100644 dev-requirements.txt create mode 100644 pyproject.toml delete mode 100644 requirements.txt delete mode 100644 setup.py create mode 100644 src/civis_jupyter_notebooks/__init__.py create mode 100644 src/civis_jupyter_notebooks/__main__.py rename {civis_jupyter_notebooks => src/civis_jupyter_notebooks}/assets/.bashrc (100%) create mode 100644 src/civis_jupyter_notebooks/assets/civis-git-clone rename {civis_jupyter_notebooks => src/civis_jupyter_notebooks}/assets/civis-jupyter-notebooks-start (100%) rename {civis_jupyter_notebooks => src/civis_jupyter_notebooks}/assets/civis_client_config.py (59%) rename {civis_jupyter_notebooks => src/civis_jupyter_notebooks}/assets/custom.css (100%) rename {civis_jupyter_notebooks => src/civis_jupyter_notebooks}/assets/custom.js (100%) rename {civis_jupyter_notebooks => src/civis_jupyter_notebooks}/assets/extensions/terminal.js (100%) rename {civis_jupyter_notebooks => src/civis_jupyter_notebooks}/assets/extensions/uncommitted_changes.js (100%) rename {civis_jupyter_notebooks => src/civis_jupyter_notebooks}/assets/fonts/civicons.eot (100%) rename {civis_jupyter_notebooks => src/civis_jupyter_notebooks}/assets/fonts/civicons.svg (100%) rename {civis_jupyter_notebooks => src/civis_jupyter_notebooks}/assets/fonts/civicons.ttf (100%) rename {civis_jupyter_notebooks => src/civis_jupyter_notebooks}/assets/fonts/civicons.woff (100%) rename {civis_jupyter_notebooks => src/civis_jupyter_notebooks}/assets/fonts/config.json (100%) rename {civis_jupyter_notebooks => src/civis_jupyter_notebooks}/assets/initialize-git (100%) create mode 100644 src/civis_jupyter_notebooks/assets/ipython_config.py rename {civis_jupyter_notebooks => src/civis_jupyter_notebooks}/assets/jupyter_notebook_config.py (75%) rename {civis_jupyter_notebooks => src/civis_jupyter_notebooks}/extensions/__init__.py (100%) rename {civis_jupyter_notebooks => src/civis_jupyter_notebooks}/extensions/git/__init__.py (100%) rename {civis_jupyter_notebooks => src/civis_jupyter_notebooks}/extensions/git/uncommitted_changes.py (69%) rename {civis_jupyter_notebooks => src/civis_jupyter_notebooks}/git_utils.py (75%) rename {civis_jupyter_notebooks => src/civis_jupyter_notebooks}/notebook_config.py (65%) rename {civis_jupyter_notebooks => src/civis_jupyter_notebooks}/platform_persistence.py (50%) rename {civis_jupyter_notebooks/tests => tests}/__init__.py (100%) rename {civis_jupyter_notebooks/tests => tests}/extensions/__init__.py (100%) rename {civis_jupyter_notebooks/tests => tests}/extensions/git/__init__.py (100%) rename {civis_jupyter_notebooks/tests => tests}/extensions/git/test_uncommited_changes.py (72%) rename {civis_jupyter_notebooks/tests => tests}/fixtures/sample_new_notebook.ipynb (100%) rename {civis_jupyter_notebooks/tests => tests}/fixtures/sample_notebook.ipynb (100%) rename {civis_jupyter_notebooks/tests => tests}/test_git_utils.py (77%) create mode 100644 tests/test_notebook_config.py create mode 100644 tests/test_platform_persistence.py create mode 100644 tests/test_version.py diff --git a/.circleci/config.yml b/.circleci/config.yml index 3d79471..b9ced03 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,98 +1,43 @@ version: 2.1 jobs: - pre-build: - description: A check that doesn't need every supported Python version (e.g., code style checks) - parameters: - command-run: - type: string + build: docker: - # Pick the highest Python 3.x version that this project is known to support - - image: cimg/python:3.11 + # Pick the latest released Python 3.x version from https://circleci.com/developer/images/image/cimg/python + - image: cimg/python:3.12 steps: - checkout - run: - working_directory: ~/project/ - command: << parameters.command-run >> - - build-python: - parameters: - python-version: - type: string - docker: - - image: cimg/python:<< parameters.python-version >> - steps: - - checkout + name: Install the full development requirements + command: | + pip install --progress-bar off -e ".[dev]" && \ + pip list -v - run: - # Test that we can build a source distribution that can correctly - # install from clean slate. - name: Build source distribution and install package from it - working_directory: ~/project/ + name: bandit command: | - pip install --progress-bar off --upgrade pip setuptools build && \ - python -m build && \ - pip install dist/`ls dist/ | grep .whl` + bandit --version && \ + bandit -r src -x tests - run: - name: Install the full development requirements - working_directory: ~/project/ - command: pip install --progress-bar off -r dev-requirements.txt + name: black + command: | + black --check src tests - run: - name: Show installed Python packages - command: pip list -v + name: flake8 + command: | + flake8 src tests - run: - name: Run python tests - working_directory: ~/ - # Avoid being able to import the package by relative import. - # Test code by importing the *installed* package in site-packages. + name: pip-audit command: | - pytest -vv --durations=0 --junitxml=/tmp/testxml/report.xml project/civis_jupyter_notebooks - - store_test_results: - path: /tmp/testxml/ - - setup_remote_docker + pip-audit --version && \ + pip-audit --skip-editable - run: - name: Test Docker image build - working_directory: ~/project/ + name: Build and check the source distribution and wheel command: | - if [ << parameters.python-version >> = 3.11 ] - then - ./tests/run_docker_tests.sh tests/Dockerfile - fi - -workflows: - version: 2 - build-and-test: - jobs: - - pre-build: - name: flake8 - command-run: | - pip install -r dev-requirements.txt && \ - flake8 civis_jupyter_notebooks - - pre-build: - name: twine - command-run: | - pip install --upgrade twine build && \ - python -m build && \ - twine check dist/`ls dist/ | grep .tar.gz` && \ - twine check dist/`ls dist/ | grep .whl` - - pre-build: - name: safety - command-run: | - pip install -e . && \ - pip install --upgrade safety && \ - safety --version && \ - safety check - - pre-build: - name: bandit - command-run: | - pip install --upgrade bandit && \ - bandit --version && \ - bandit -r civis_jupyter_notebooks -x civis_jupyter_notebooks/tests - - build-python: - requires: - - flake8 - - twine - - safety - - bandit - matrix: - parameters: - python-version: ["3.8", "3.9", "3.10", "3.11"] + python -m build && \ + twine check dist/`ls dist/ | grep .tar.gz` && \ + twine check dist/`ls dist/ | grep .whl` + - run: + name: Run python tests + command: pytest --durations=0 --junitxml=/tmp/testxml/report.xml + - store_test_results: + path: /tmp/testxml/ diff --git a/.flake8 b/.flake8 index 113ca5f..8dd399a 100644 --- a/.flake8 +++ b/.flake8 @@ -1,2 +1,3 @@ [flake8] -max-line-length = 125 +max-line-length = 88 +extend-ignore = E203 diff --git a/.github/ISSUE_TEMPLATE/general.md b/.github/ISSUE_TEMPLATE/general.md new file mode 100644 index 0000000..5e6ede5 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/general.md @@ -0,0 +1,11 @@ +--- +name: General +about: Ask a question, report a potential issue, etc. +title: '' +labels: '' +assignees: '' + +--- + +**Note:** Civis employees should _not_ use the GitHub Issues feature at the public "civis-python" codebase +to file a ticket, and should instead use the internal ticketing system. diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 0000000..83b1957 --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,9 @@ + + +--- + +- [ ] (For Civis employees only) Reference to a relevant ticket in the pull request title +- [ ] Changelog entry added to `CHANGELOG.md` at the repo's root level +- [ ] Description of change in the pull request description +- [ ] If applicable, unit tests have been added and/or updated +- [ ] The CircleCI builds have all passed diff --git a/CHANGELOG.md b/CHANGELOG.md index ff7d89c..842a222 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,17 @@ and this project adheres to [Semantic Versioning](http://semver.org/). ## Unreleased +## [2.2.0] - 2024-06-12 + +### Added +- Added support for Python 3.12. + +### Changed +- Updated packaging to use `pyproject.toml`. + +### Removed +- Dropped support for Python 3.8. + ## [2.1.1] - 2023-08-22 ### Changed diff --git a/MANIFEST.in b/MANIFEST.in deleted file mode 100644 index d9907bc..0000000 --- a/MANIFEST.in +++ /dev/null @@ -1,5 +0,0 @@ -include CHANGELOG.md -include LICENSE.txt -include requirements.txt -include README.rst -recursive-include civis_jupyter_notebooks/assets/ * diff --git a/README.rst b/README.rst index 52e2b8a..352786e 100644 --- a/README.rst +++ b/README.rst @@ -35,12 +35,12 @@ See the `example`_ Docker image for more details. .. _example: example -Integration Testing Docker Images with the Civis Platform ---------------------------------------------------------- +Integration Testing Docker Images with Civis Platform +----------------------------------------------------- -If you would like to test your image's integration with the Civis Platform locally follow the steps below: +If you would like to test your image's integration with Civis Platform locally follow the steps below: -1. Create a notebook in your Civis Platform account and grab the id of the notebook. This ID is the number +1. Create a notebook in your Civis Platform account and grab the ID of the notebook. This ID is the number that appears at the end of the URL for the notebook, ``https://platform.civisanalytics.com/#/notebooks/``. 2. Create an environment file called ``my.env`` and add the following to it:: @@ -51,11 +51,11 @@ If you would like to test your image's integration with the Civis Platform local 4. Run the container: ``docker run --rm -p 8888:8888 --env-file my.env test``. 5. Access the notebook at the ip of your Docker host with port 8888 (e.g., ``http://localhost:8888/notebooks/notebook.ipynb``). -Integration Testing Code Changes with the Civis Platform --------------------------------------------------------- +Integration Testing Code Changes with Civis Platform +---------------------------------------------------- The scripts ``tests/build_dev_image.sh`` and ``tests/run_dev_image.sh`` can be used to test the -integration of code changes with the Civis Platform. +integration of code changes with Civis Platform. From the top directory in the repo type:: diff --git a/civis_jupyter_notebooks/__init__.py b/civis_jupyter_notebooks/__init__.py deleted file mode 100644 index 80ee6d0..0000000 --- a/civis_jupyter_notebooks/__init__.py +++ /dev/null @@ -1,8 +0,0 @@ -try: - from importlib.metadata import version -except ModuleNotFoundError: - # For Python 3.7 - from importlib_metadata import version - - -__version__ = version("civis-jupyter-notebook") diff --git a/civis_jupyter_notebooks/__main__.py b/civis_jupyter_notebooks/__main__.py deleted file mode 100644 index a2a7f33..0000000 --- a/civis_jupyter_notebooks/__main__.py +++ /dev/null @@ -1,58 +0,0 @@ -import os -import shutil -import pkg_resources -import subprocess # nosec - -import click - - -@click.command() -def cli(): - """Install configuration files, IPython extensions, Jupyter extensions, - and JavaScript/CSS assets for using a Docker image with Civis Platform - Jupyter notebooks. - """ - - # make home areas and dirs - for dr in [('~', 'work'), - ('~', '.jupyter', 'custom'), - # folder that holds all the JS for notebook frontend extensions - ('~', '.jupyter', 'extensions'), - ('~', '.jupyter', 'custom', 'fonts'), - ('~', '.ipython', 'profile_default')]: - try: - os.makedirs(os.path.expanduser(os.path.join(*dr))) - except OSError: - pass - - # enable civisjupyter extension - for cmd in ['jupyter nbextension install --py civis_jupyter_ext', - 'jupyter nbextension enable --py civis_jupyter_ext']: - subprocess.check_call(cmd, shell=True) # nosec - - # copy code - def _copy(src, dst): - src = pkg_resources.resource_filename(__name__, os.path.join(*src)) - dst = os.path.expanduser(os.path.join(*dst)) - shutil.copy(src, dst) - - _copy(('assets', 'jupyter_notebook_config.py'), ('~', '.jupyter')) - _copy(('assets', 'custom.css'), ('~', '.jupyter', 'custom')) - _copy(('assets', 'custom.js'), ('~', '.jupyter', 'custom')) - for ext in ['eot', 'woff', 'svg', 'ttf']: - _copy(('assets', 'fonts', 'civicons.%s' % ext), ('~', '.jupyter', 'custom', 'fonts')) - _copy(('assets', '.bashrc'), ('~')) - _copy(('assets', 'ipython_config.py'), ('~', '.ipython', 'profile_default')) - _copy(('assets', 'civis_client_config.py'), ('~', '.ipython')) - - # copy frontend extensions - frontend_extensions = pkg_resources.resource_listdir(__name__, os.path.join('assets', 'extensions')) - for fe_ext in frontend_extensions: - _copy(('assets', 'extensions', fe_ext), ('~', '.jupyter', 'extensions')) - - # install and enable nbextensions - subprocess.check_call('jupyter nbextension install ~/.jupyter/extensions', shell=True) # nosec - for extension in frontend_extensions: - ext_name = os.path.splitext(extension)[0] - cmd = 'jupyter nbextension enable extensions/{}'.format(ext_name) - subprocess.check_call(cmd, shell=True) # nosec diff --git a/civis_jupyter_notebooks/assets/civis-git-clone b/civis_jupyter_notebooks/assets/civis-git-clone deleted file mode 100644 index 217313d..0000000 --- a/civis_jupyter_notebooks/assets/civis-git-clone +++ /dev/null @@ -1,15 +0,0 @@ -#!/usr/bin/env python - -import os -from civis_jupyter_notebooks.git_utils import CivisGit, CivisGitError -from civis_jupyter_notebooks import log_utils - -stream_logger = log_utils.setup_stream_logging() - -if os.environ.get('GIT_REPO_URL'): - try: - stream_logger.info('cloning git repository') - CivisGit().clone_repository() - stream_logger.info('clone complete') - except CivisGitError as e: - stream_logger.error('error clonging git repository: {}'.format(str(e))) diff --git a/civis_jupyter_notebooks/assets/ipython_config.py b/civis_jupyter_notebooks/assets/ipython_config.py deleted file mode 100644 index aaa9064..0000000 --- a/civis_jupyter_notebooks/assets/ipython_config.py +++ /dev/null @@ -1,4 +0,0 @@ -c = get_config() # noqa - -c.InteractiveShellApp.extensions.append('civis_jupyter_ext') -c.InteractiveShellApp.exec_files.append('civis_client_config.py') diff --git a/civis_jupyter_notebooks/log_utils.py b/civis_jupyter_notebooks/log_utils.py deleted file mode 100644 index 2105c3a..0000000 --- a/civis_jupyter_notebooks/log_utils.py +++ /dev/null @@ -1,52 +0,0 @@ -import logging -import sys - - -# Adapted from https://stackoverflow.com/a/1383365 -class SingleLevelFilter(logging.Filter): - def __init__(self, passlevel): - self.passlevel = passlevel - - def filter(self, record): - return (record.levelno == self.passlevel) - - -def setup_stream_logging(): - """ - Configure the CIVIS_PLATFORM_BACKEND logger for stream logging. - - DEBUG level logs will be ignored, - INFO level logs will be sent to stdout, - WARNING, ERROR, & CRITICAL level logs will be sent to stderr. - - Returns - ------- - The CIVIS_PLATFORM_BACKEND logger - """ - logger = logging.getLogger('CIVIS_PLATFORM_BACKEND') - - # prevents duplicate log records by preventing messages from being passed to ancestor loggers, e.g. the root logger - logger.propagate = False - - # Sets the lowest level which this logger will pay attention to (i.e. ignore debug level logs) - logger.setLevel(logging.INFO) - - formatter = logging.Formatter( - fmt='[%(levelname).1s %(asctime)s %(name)s] %(message)s', - datefmt="%H:%M:%S") - - # Configures a handler for sending warning level and above logs to stderr - error_handler = logging.StreamHandler(stream=sys.__stderr__) - error_handler.setLevel(logging.WARNING) - error_handler.setFormatter(formatter) - logger.addHandler(error_handler) - - # Configures a handler for sending info level logs to stdout - info_handler = logging.StreamHandler(stream=sys.__stdout__) - info_handler.setLevel(logging.INFO) - # Filter so that the info handler will handle ONLY info level logs - info_handler.addFilter(SingleLevelFilter(logging.INFO)) - info_handler.setFormatter(formatter) - logger.addHandler(info_handler) - - return logger diff --git a/civis_jupyter_notebooks/tests/test_notebook_config.py b/civis_jupyter_notebooks/tests/test_notebook_config.py deleted file mode 100644 index c2fbaad..0000000 --- a/civis_jupyter_notebooks/tests/test_notebook_config.py +++ /dev/null @@ -1,103 +0,0 @@ -import unittest -from unittest.mock import patch, ANY -import os -from traitlets.config.loader import Config - -from civis_jupyter_notebooks import notebook_config, platform_persistence - - -class NotebookConfigTest(unittest.TestCase): - @patch.dict(os.environ, {'DEFAULT_KERNEL': 'kern'}) - def test_config_jupyter_sets_notebook_config(self): - c = Config({}) - notebook_config.config_jupyter(c) - assert(c.NotebookApp.port == 8888) - assert(c.MultiKernelManager.default_kernel_name == 'kern') - - @patch('civis_jupyter_notebooks.platform_persistence.initialize_notebook_from_platform') - @patch('civis_jupyter_notebooks.platform_persistence.post_save') - def test_get_notebook_initializes_and_saves(self, post_save, init_notebook): - notebook_config.get_notebook('path') - init_notebook.assert_called_with('path') - post_save.assert_called_with(ANY, 'path', None) - - @patch('civis_jupyter_notebooks.platform_persistence.initialize_notebook_from_platform') - @patch('civis_jupyter_notebooks.platform_persistence.logger') - @patch('os.kill') - def test_get_notebook_writes_log_and_kills_proc_on_error(self, kill, logger, init_notebook): - init_notebook.side_effect = platform_persistence.NotebookManagementError('err') - notebook_config.get_notebook('path') - logger.error.assert_called_with('err') - logger.warn.assert_called_with(ANY) - kill.assert_called_with(ANY, ANY) - - @patch('civis_jupyter_notebooks.platform_persistence.find_and_install_requirements') - def test_find_and_install_requirements_success(self, persistence_find_and_install_requirements): - c = Config({}) - notebook_config.find_and_install_requirements('path', c) - persistence_find_and_install_requirements.assert_called_with('path') - - @patch('civis_jupyter_notebooks.platform_persistence.find_and_install_requirements') - @patch('civis_jupyter_notebooks.platform_persistence.logger') - def test_find_and_install_requirements_logs_errors(self, logger, persistence_find_and_install_requirements): - persistence_find_and_install_requirements.side_effect = platform_persistence.NotebookManagementError('err') - c = Config({}) - notebook_config.find_and_install_requirements('path', c) - logger.error.assert_called_with("Unable to install requirements.txt:\nerr") - - @patch('civis_jupyter_notebooks.notebook_config.stage_new_notebook') - @patch('civis_jupyter_notebooks.notebook_config.config_jupyter') - @patch('civis_jupyter_notebooks.notebook_config.find_and_install_requirements') - @patch('civis_jupyter_notebooks.notebook_config.get_notebook') - def test_civis_setup_success( - self, - get_notebook, find_and_install_requirements, - config_jupyter, _stage_new_notebook): - c = Config({}) - notebook_config.civis_setup(c) - - assert(c.NotebookApp.default_url == '/notebooks/notebook.ipynb') - config_jupyter.assert_called_with(ANY) - get_notebook.assert_called_with(notebook_config.ROOT_DIR + '/notebook.ipynb') - find_and_install_requirements.assert_called_with(notebook_config.ROOT_DIR, ANY) - - @patch('civis_jupyter_notebooks.notebook_config.stage_new_notebook') - @patch('os.environ.get') - @patch('civis_jupyter_notebooks.notebook_config.config_jupyter') - @patch('civis_jupyter_notebooks.notebook_config.find_and_install_requirements') - @patch('civis_jupyter_notebooks.notebook_config.get_notebook') - def test_civis_setup_uses_environment_setting( - self, - get_notebook, find_and_install_requirements, - config_jupyter, environ_get, _stage_new_notebook): - - c = Config({}) - environ_get.return_value = 'subpath/foo.ipynb' - notebook_config.civis_setup(c) - - assert(c.NotebookApp.default_url == '/notebooks/subpath/foo.ipynb') - config_jupyter.assert_called_with(ANY) - get_notebook.assert_called_with(notebook_config.ROOT_DIR + '/subpath/foo.ipynb') - find_and_install_requirements.assert_called_with(notebook_config.ROOT_DIR + '/subpath', ANY) - - @patch('civis_jupyter_notebooks.notebook_config.CivisGit') - def test_stage_new_notebook_stages_notebook_file(self, civis_git): - civis_git.return_value.is_git_enabled.return_value = True - repo = civis_git.return_value.repo.return_value - repo.untracked_files = ['notebook.ipynb'] - - notebook_config.stage_new_notebook('notebook.ipynb') - repo.index.add.assert_called_with(['notebook.ipynb']) - - @patch('civis_jupyter_notebooks.notebook_config.CivisGit') - def test_stage_new_notebook_git_is_disabled(self, civis_git): - civis_git.return_value.is_git_enabled.return_value = False - notebook_config.stage_new_notebook('notebook.ipynb') - - civis_git.return_value.repo.assert_not_called() - - civis_git.return_value.repo.untracked_files.assert_not_called() - - -if __name__ == '__main__': - unittest.main() diff --git a/civis_jupyter_notebooks/tests/test_platform_persistence.py b/civis_jupyter_notebooks/tests/test_platform_persistence.py deleted file mode 100644 index aece115..0000000 --- a/civis_jupyter_notebooks/tests/test_platform_persistence.py +++ /dev/null @@ -1,235 +0,0 @@ -import os -import subprocess -import nbformat -import requests -import unittest -from unittest.mock import ANY, MagicMock, patch -import logging - -from civis_jupyter_notebooks import platform_persistence -from civis_jupyter_notebooks.platform_persistence import NotebookManagementError - -TEST_NOTEBOOK_PATH = '/path/to/notebook.ipynb' -TEST_PLATFORM_OBJECT_ID = '1914' -SAMPLE_NOTEBOOK = open(os.path.join(os.path.dirname(__file__), 'fixtures/sample_notebook.ipynb')).read() -SAMPLE_NEW_NOTEBOOK = open(os.path.join(os.path.dirname(__file__), 'fixtures/sample_new_notebook.ipynb')).read() - - -class NotebookWithoutNewFlag(nbformat.NotebookNode): - """ Helper that tests if a NotebookNode has the metadata.civis.new_notebook flag set to True """ - def __eq__(self, other): - return not other.get('metadata', {}).get('civis', {}).get('new_notebook', False) - - -class PlatformPersistenceTest(unittest.TestCase): - def setUp(self): - os.environ['CIVIS_API_KEY'] = 'hi mom' - os.environ['PLATFORM_OBJECT_ID'] = TEST_PLATFORM_OBJECT_ID - - logging.disable(logging.INFO) - - @patch('os.makedirs') - @patch('civis_jupyter_notebooks.platform_persistence.open') - @patch('civis.APIClient') - @patch('civis_jupyter_notebooks.platform_persistence.requests.get') - def test_initialize_notebook_will_get_nb_from_platform(self, rg, _client, _op, _makedirs): - rg.return_value = MagicMock(spec=requests.Response, status_code=200, content=SAMPLE_NOTEBOOK) - platform_persistence.initialize_notebook_from_platform(TEST_NOTEBOOK_PATH) - platform_persistence.get_client().notebooks.get.assert_called_with(TEST_PLATFORM_OBJECT_ID) - - @patch('os.makedirs') - @patch('civis_jupyter_notebooks.platform_persistence.open') - @patch('civis_jupyter_notebooks.platform_persistence.__pull_and_load_requirements') - @patch('civis.APIClient') - @patch('civis_jupyter_notebooks.platform_persistence.requests.get') - def test_initialize_notebook_will_pull_nb_from_url(self, rg, _client, requirements, _op, _makedirs): - url = 'http://whatever' - rg.return_value = MagicMock(spec=requests.Response, status_code=200, content=SAMPLE_NOTEBOOK) - platform_persistence.get_client().notebooks.get.return_value.notebook_url = url - platform_persistence.get_client().notebooks.get.return_value.requirements_url = None - platform_persistence.initialize_notebook_from_platform(TEST_NOTEBOOK_PATH) - rg.assert_called_with(url, timeout=60) - requirements.assert_not_called() - - @patch('os.makedirs') - @patch('civis_jupyter_notebooks.platform_persistence.open') - @patch('civis.APIClient') - @patch('civis_jupyter_notebooks.platform_persistence.requests.get') - def test_initialize_notebook_will_throw_error_on_nb_pull(self, rg, _client, _op, _makedirs): - rg.return_value = MagicMock(spec=requests.Response, status_code=500, response={}) - self.assertRaises(NotebookManagementError, - lambda: platform_persistence.initialize_notebook_from_platform(TEST_NOTEBOOK_PATH)) - - @patch('nbformat.write') - @patch('os.makedirs') - @patch('civis_jupyter_notebooks.platform_persistence.open') - @patch('civis.APIClient') - @patch('civis_jupyter_notebooks.platform_persistence.requests.get') - def test_initialize_notebook_will_set_new_notebook_flag_to_false(self, rg, _client, _op, _makedirs, nbwrite): - rg.return_value = MagicMock(spec=requests.Response, status_code=200, content=SAMPLE_NEW_NOTEBOOK) - platform_persistence.get_client().notebooks.get.return_value.notebooks_url = 'something' - platform_persistence.get_client().notebooks.get.return_value.requirements_url = None - platform_persistence.initialize_notebook_from_platform(TEST_NOTEBOOK_PATH) - nbwrite.assert_called_with(NotebookWithoutNewFlag(), ANY) - - @patch('os.path.isfile') - @patch('nbformat.write') - @patch('os.makedirs') - @patch('civis_jupyter_notebooks.platform_persistence.open') - @patch('civis.APIClient') - @patch('civis_jupyter_notebooks.platform_persistence.requests.get') - def test_initialize_notebook_will_use_s3_notebook_if_not_new_and_git_notebook_exists(self, rg, _client, _op, - _makedirs, nbwrite, isfile): - rg.return_value = MagicMock(spec=requests.Response, status_code=200, content=SAMPLE_NOTEBOOK) - platform_persistence.get_client().notebooks.get.return_value.notebooks_url = 'something' - platform_persistence.get_client().notebooks.get.return_value.requirements_url = None - isfile.return_value = True - platform_persistence.initialize_notebook_from_platform(TEST_NOTEBOOK_PATH) - nbwrite.assert_called_with(ANY, ANY) - - @patch('os.path.isfile') - @patch('nbformat.write') - @patch('os.makedirs') - @patch('civis_jupyter_notebooks.platform_persistence.open') - @patch('civis.APIClient') - @patch('civis_jupyter_notebooks.platform_persistence.requests.get') - def test_initialize_notebook_will_discard_s3_notebook_if_new_and_git_notebook_exists(self, rg, _client, _op, - _makedirs, nbwrite, isfile): - rg.return_value = MagicMock(spec=requests.Response, status_code=200, content=SAMPLE_NEW_NOTEBOOK) - platform_persistence.get_client().notebooks.get.return_value.notebooks_url = 'something' - platform_persistence.get_client().notebooks.get.return_value.requirements_url = None - isfile.return_value = True - platform_persistence.initialize_notebook_from_platform(TEST_NOTEBOOK_PATH) - nbwrite.assert_not_called() - - @patch('civis_jupyter_notebooks.platform_persistence.open') - @patch('civis.APIClient') - @patch('os.makedirs') - @patch('civis_jupyter_notebooks.platform_persistence.requests.get') - def test_initialize_notebook_will_create_directories_if_needed(self, rg, makedirs, _client, _op): - rg.return_value = MagicMock(spec=requests.Response, status_code=200, content=SAMPLE_NOTEBOOK) - platform_persistence.initialize_notebook_from_platform(TEST_NOTEBOOK_PATH) - directory = os.path.dirname(TEST_NOTEBOOK_PATH) - makedirs.assert_called_with(directory) - - @patch('os.makedirs') - @patch('civis_jupyter_notebooks.platform_persistence.open') - @patch('civis_jupyter_notebooks.platform_persistence.__pull_and_load_requirements') - @patch('civis.APIClient') - @patch('civis_jupyter_notebooks.platform_persistence.requests.get') - def test_initialize_notebook_will_pull_requirements(self, rg, _client, requirements, _op, _makedirs): - url = 'http://whatever' - rg.return_value = MagicMock(spec=requests.Response, status_code=200, content=SAMPLE_NOTEBOOK) - platform_persistence.get_client().notebooks.get.return_value.requirements_url = url - platform_persistence.initialize_notebook_from_platform(TEST_NOTEBOOK_PATH) - requirements.assert_called_with(url, TEST_NOTEBOOK_PATH) - - @patch('os.makedirs') - @patch('civis_jupyter_notebooks.platform_persistence.open') - @patch('civis_jupyter_notebooks.platform_persistence.__pull_and_load_requirements') - @patch('civis.APIClient') - @patch('civis_jupyter_notebooks.platform_persistence.requests.get') - def test_initialize_notebook_will_error_on_requirements_pull(self, rg, _client, _requirements, _op, _makedirs): - url = 'http://whatever' - rg.return_value = MagicMock(spec=requests.Response, status_code=500) - platform_persistence.get_client().notebooks.get.return_value.requirements_url = url - self.assertRaises(NotebookManagementError, - lambda: platform_persistence.initialize_notebook_from_platform(TEST_NOTEBOOK_PATH)) - - @patch('civis_jupyter_notebooks.platform_persistence.open') - @patch('civis_jupyter_notebooks.platform_persistence.check_call') - @patch('civis.APIClient') - @patch('requests.put') - def test_post_save_fetches_urls_from_api(self, _rput, client, _ccc, _op): - platform_persistence.post_save({'type': 'notebook'}, '', {}) - platform_persistence.get_client().notebooks.list_update_links.assert_called_with(TEST_PLATFORM_OBJECT_ID) - - @patch('civis_jupyter_notebooks.platform_persistence.open') - @patch('civis_jupyter_notebooks.platform_persistence.check_call') - @patch('civis.APIClient') - @patch('requests.put') - @patch('civis_jupyter_notebooks.platform_persistence.save_notebook') - def test_post_save_performs_two_put_operations(self, save, rput, _client, _ccc, _op): - platform_persistence.post_save({'type': 'notebook'}, '', {}) - self.assertTrue(save.called) - - @patch('civis_jupyter_notebooks.platform_persistence.open') - @patch('civis_jupyter_notebooks.platform_persistence.check_call') - @patch('civis.APIClient') - @patch('requests.put') - @patch('civis_jupyter_notebooks.platform_persistence.save_notebook') - @patch('civis_jupyter_notebooks.platform_persistence.get_update_urls') - def test_post_save_skipped_for_non_notebook_types(self, guu, save, _rput, _client, _ccc, _op): - platform_persistence.post_save({'type': 'blargggg'}, '', {}) - self.assertFalse(guu.called) - self.assertFalse(save.called) - - @patch('civis_jupyter_notebooks.platform_persistence.open') - @patch('civis_jupyter_notebooks.platform_persistence.check_call') - @patch('civis.APIClient') - @patch('requests.put') - def test_post_save_generates_preview(self, _rput, _client, check_call, _op): - platform_persistence.post_save({'type': 'notebook'}, 'x/y', {}) - check_call.assert_called_with(['jupyter', 'nbconvert', '--to', 'html', 'y'], cwd='x') - - @patch('civis_jupyter_notebooks.platform_persistence.open') - @patch('civis_jupyter_notebooks.platform_persistence.check_call') - @patch('civis.APIClient') - @patch('requests.put') - def test_generate_preview_throws_error_on_convert(self, _rput, _client, check_call, _op): - check_call.side_effect = subprocess.CalledProcessError('foo', 255) - self.assertRaises(NotebookManagementError, - lambda: platform_persistence.generate_and_save_preview('http://notebook_url_in_s3', 'os/path')) - check_call.assert_called_with(['jupyter', 'nbconvert', '--to', 'html', 'path'], cwd='os') - - @patch('civis.APIClient') - def test_will_regenerate_api_client(self, mock_client): - platform_persistence.get_client() - mock_client.assert_called_with() - - @patch('os.path.isfile') - @patch('os.path.isdir') - @patch('civis_jupyter_notebooks.platform_persistence.pip_install') - def test_find_and_install_requirements_calls_pip_install(self, pip_install, isdir, isfile): - os.path.isdir.return_value = True - os.path.isfile.return_value = True - platform_persistence.find_and_install_requirements('/root/work/foo') - pip_install.assert_called_with('/root/work/foo/requirements.txt') - - @patch('os.path.isfile') - @patch('os.path.isdir') - @patch('civis_jupyter_notebooks.platform_persistence.pip_install') - def test_find_and_install_requirements_searches_tree(self, pip_install, isdir, isfile): - os.path.isdir.return_value = True - os.path.isfile.side_effect = [False, True] - platform_persistence.find_and_install_requirements('/root/work/foo') - pip_install.assert_called_with('/root/work/requirements.txt') - - @patch('os.path.isfile') - @patch('os.path.isdir') - @patch('civis_jupyter_notebooks.platform_persistence.pip_install') - def test_find_and_install_requirements_excludes_root(self, pip_install, isdir, isfile): - os.path.isdir.return_value = True - os.path.isfile.return_value = True - platform_persistence.find_and_install_requirements('/root') - pip_install.assert_not_called() - - @patch('subprocess.check_output') - @patch('sys.executable') - def test_pip_install_calls_subprocess(self, executable, check_output): - platform_persistence.pip_install('/path/requirements.txt') - check_output.assert_called_with( - [executable, '-m', 'pip', 'install', '-r', '/path/requirements.txt'], - stderr=subprocess.STDOUT - ) - - @patch('subprocess.check_output') - @patch('sys.executable') - def test_pip_install_failure_raises_notebookmanagementerror(self, executable, check_output): - check_output.side_effect = subprocess.CalledProcessError(returncode=1, cmd='cmd', output=b'installation error') - with self.assertRaisesRegex(NotebookManagementError, 'installation error'): - platform_persistence.pip_install('/path/requirements.txt') - - -if __name__ == '__main__': - unittest.main() diff --git a/civis_jupyter_notebooks/tests/test_version.py b/civis_jupyter_notebooks/tests/test_version.py deleted file mode 100644 index c218905..0000000 --- a/civis_jupyter_notebooks/tests/test_version.py +++ /dev/null @@ -1,17 +0,0 @@ -import os -import re - -import civis_jupyter_notebooks - - -_REPO_DIR = os.path.dirname(os.path.dirname(os.path.dirname(__file__))) - - -def test_version_number_match_with_changelog(): - """__version__ and CHANGELOG.md match for the latest version number.""" - changelog = open(os.path.join(_REPO_DIR, 'CHANGELOG.md')).read() - version_in_changelog = ( - re.search(r'##\s+\[(\d+\.\d+\.\d+)\]', changelog).groups()[0]) - assert civis_jupyter_notebooks.__version__ == version_in_changelog, ( - 'Make sure both __version__ and CHANGELOG are updated to match the ' - 'latest version number') diff --git a/dev-requirements.txt b/dev-requirements.txt deleted file mode 100644 index 7498475..0000000 --- a/dev-requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -pytest==7.0.1 -flake8==4.0.1 diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..e447177 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,64 @@ +[build-system] +requires = ["setuptools >= 70.0.0", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "civis-jupyter-notebook" +version = "2.2.0" +description = "A tool for building Docker images for Civis Platform Jupyter notebooks" +readme = "README.rst" +requires-python = ">= 3.9" +authors = [ { name = "Civis Analytics", email = "opensource@civisanalytics.com" } ] +license = { text = "BSD-3-Clause" } +dependencies = [ + "civis >= 2.2.0", + "civis-jupyter-extensions >= 1.2.0", + "click >= 6.7", + "GitPython >= 2.1", + "jupyter-core >= 4.6.0", + "notebook >= 6.4.1, < 7.0", + "requests >= 2.18", + "tornado >= 6.1.0", +] +classifiers = [ + "Development Status :: 5 - Production/Stable", + "License :: OSI Approved :: BSD License", + "Programming Language :: Python", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3 :: Only", +] + +[project.urls] +Homepage = "https://www.civisanalytics.com" +Source = "https://github.com/civisanalytics/civis-jupyter-notebook" + +[project.optional-dependencies] +dev = [ + "bandit", # Use the latest version + "black == 24.4.2", + "build == 1.2.1", + "flake8 == 7.0.0", + "pip-audit", # Use the latest version + "pytest == 8.2.0", + "twine == 5.0.0", +] + +[project.scripts] +civis-jupyter-notebooks-install = "civis_jupyter_notebooks.__main__:cli" + +[tool.setuptools] +script-files = [ + "src/civis_jupyter_notebooks/assets/civis-jupyter-notebooks-start", + "src/civis_jupyter_notebooks/assets/initialize-git", + "src/civis_jupyter_notebooks/assets/civis-git-clone", +] + +[tool.setuptools.packages.find] +where = [ "src" ] + +[tool.setuptools.package-data] +civis_jupyter_notebooks = ["assets/**/*", "assets/.*"] + +[tool.pytest.ini_options] +addopts = "--strict-markers --ignore=tests/test_ext.py -vv" +testpaths = ["tests"] diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index d4b7844..0000000 --- a/requirements.txt +++ /dev/null @@ -1,9 +0,0 @@ -civis>=1.9 -requests>=2.18 -click>=6.7 -jupyter-core>=4.6.0 -notebook>=6.4.1,<7.0 -tornado>=6.1.0 -civis-jupyter-extensions>=1.1.0 -GitPython>=2.1 -importlib-metadata >= 1.0 ; python_version < '3.8' diff --git a/setup.py b/setup.py deleted file mode 100644 index 355ba7b..0000000 --- a/setup.py +++ /dev/null @@ -1,33 +0,0 @@ -import os -from setuptools import find_packages, setup - - -def read(fname): - with open(os.path.join(os.path.dirname(__file__), fname)) as _in: - return _in.read() - - -setup( - name="civis-jupyter-notebook", - version="2.1.1", - author="Civis Analytics Inc", - author_email="opensource@civisanalytics.com", - url="https://www.civisanalytics.com", - description=("A tool for building Docker images for Civis " - "Platform Jupyter notebooks."), - packages=find_packages(), - long_description=read('README.rst'), - long_description_content_type="text/x-rst", - include_package_data=True, - license="BSD-3", - install_requires=read('requirements.txt').strip().split('\n'), - scripts=[ - 'civis_jupyter_notebooks/assets/civis-jupyter-notebooks-start', - 'civis_jupyter_notebooks/assets/initialize-git', - 'civis_jupyter_notebooks/assets/civis-git-clone' - ], - entry_points={ - 'console_scripts': [ - 'civis-jupyter-notebooks-install = ' - 'civis_jupyter_notebooks.__main__:cli', - ]}) diff --git a/src/civis_jupyter_notebooks/__init__.py b/src/civis_jupyter_notebooks/__init__.py new file mode 100644 index 0000000..b0c3594 --- /dev/null +++ b/src/civis_jupyter_notebooks/__init__.py @@ -0,0 +1,4 @@ +from importlib.metadata import version + + +__version__ = version("civis-jupyter-notebook") diff --git a/src/civis_jupyter_notebooks/__main__.py b/src/civis_jupyter_notebooks/__main__.py new file mode 100644 index 0000000..4b151db --- /dev/null +++ b/src/civis_jupyter_notebooks/__main__.py @@ -0,0 +1,69 @@ +import os +import shutil +import subprocess # nosec + +import click + + +_THIS_DIR = os.path.abspath(os.path.dirname(__file__)) + + +@click.command() +def cli(): + """Install configuration files, IPython extensions, Jupyter extensions, + and JavaScript/CSS assets for using a Docker image with Civis Platform + Jupyter notebooks. + """ + + # make home areas and dirs + for dr in [ + ("~", "work"), + ("~", ".jupyter", "custom"), + # folder that holds all the JS for notebook frontend extensions + ("~", ".jupyter", "extensions"), + ("~", ".jupyter", "custom", "fonts"), + ("~", ".ipython", "profile_default"), + ]: + try: + os.makedirs(os.path.expanduser(os.path.join(*dr))) + except OSError: + pass + + # enable civisjupyter extension + for cmd in [ + "jupyter nbextension install --py civis_jupyter_ext", + "jupyter nbextension enable --py civis_jupyter_ext", + ]: + subprocess.check_call(cmd, shell=True) # nosec + + # copy code + def _copy(src, dst): + src = os.path.join(_THIS_DIR, *src) + dst = os.path.expanduser(os.path.join(*dst)) + shutil.copy(src, dst) + + _copy(("assets", "jupyter_notebook_config.py"), ("~", ".jupyter")) + _copy(("assets", "custom.css"), ("~", ".jupyter", "custom")) + _copy(("assets", "custom.js"), ("~", ".jupyter", "custom")) + for ext in ["eot", "woff", "svg", "ttf"]: + _copy( + ("assets", "fonts", "civicons.%s" % ext), + ("~", ".jupyter", "custom", "fonts"), + ) + _copy(("assets", ".bashrc"), ("~")) + _copy(("assets", "ipython_config.py"), ("~", ".ipython", "profile_default")) + _copy(("assets", "civis_client_config.py"), ("~", ".ipython")) + + # copy frontend extensions + frontend_extensions = os.listdir(os.path.join(_THIS_DIR, "assets", "extensions")) + for fe_ext in frontend_extensions: + _copy(("assets", "extensions", fe_ext), ("~", ".jupyter", "extensions")) + + # install and enable nbextensions + subprocess.check_call( + "jupyter nbextension install ~/.jupyter/extensions", shell=True + ) # nosec + for extension in frontend_extensions: + ext_name = os.path.splitext(extension)[0] + cmd = "jupyter nbextension enable extensions/{}".format(ext_name) + subprocess.check_call(cmd, shell=True) # nosec diff --git a/civis_jupyter_notebooks/assets/.bashrc b/src/civis_jupyter_notebooks/assets/.bashrc similarity index 100% rename from civis_jupyter_notebooks/assets/.bashrc rename to src/civis_jupyter_notebooks/assets/.bashrc diff --git a/src/civis_jupyter_notebooks/assets/civis-git-clone b/src/civis_jupyter_notebooks/assets/civis-git-clone new file mode 100644 index 0000000..7c6accd --- /dev/null +++ b/src/civis_jupyter_notebooks/assets/civis-git-clone @@ -0,0 +1,25 @@ +#!/usr/bin/env python + +import logging +import os + +import civis +from civis_jupyter_notebooks.git_utils import CivisGit, CivisGitError + + +stream_logger = civis.civis_logger( + name="CIVIS_PLATFORM_BACKEND", + level=logging.INFO, + fmt=logging.Formatter( + fmt="[%(levelname).1s %(asctime)s %(name)s] %(message)s", datefmt="%H:%M:%S" + ), +) + + +if os.environ.get('GIT_REPO_URL'): + try: + stream_logger.info('cloning git repository') + CivisGit().clone_repository() + stream_logger.info('clone complete') + except CivisGitError as e: + stream_logger.error('error cloning git repository: {}'.format(str(e))) diff --git a/civis_jupyter_notebooks/assets/civis-jupyter-notebooks-start b/src/civis_jupyter_notebooks/assets/civis-jupyter-notebooks-start similarity index 100% rename from civis_jupyter_notebooks/assets/civis-jupyter-notebooks-start rename to src/civis_jupyter_notebooks/assets/civis-jupyter-notebooks-start diff --git a/civis_jupyter_notebooks/assets/civis_client_config.py b/src/civis_jupyter_notebooks/assets/civis_client_config.py similarity index 59% rename from civis_jupyter_notebooks/assets/civis_client_config.py rename to src/civis_jupyter_notebooks/assets/civis_client_config.py index 14aa23b..9532fd8 100644 --- a/civis_jupyter_notebooks/assets/civis_client_config.py +++ b/src/civis_jupyter_notebooks/assets/civis_client_config.py @@ -2,10 +2,10 @@ import civis from civis_jupyter_notebooks.platform_persistence import logger as LOGGER -if 'CIVIS_API_KEY' in os.environ: - LOGGER.info('creating civis api client') +if "CIVIS_API_KEY" in os.environ: + LOGGER.info("creating civis api client") client = civis.APIClient() - LOGGER.info('civis api client created') + LOGGER.info("civis api client created") # clean out the namespace for users del os diff --git a/civis_jupyter_notebooks/assets/custom.css b/src/civis_jupyter_notebooks/assets/custom.css similarity index 100% rename from civis_jupyter_notebooks/assets/custom.css rename to src/civis_jupyter_notebooks/assets/custom.css diff --git a/civis_jupyter_notebooks/assets/custom.js b/src/civis_jupyter_notebooks/assets/custom.js similarity index 100% rename from civis_jupyter_notebooks/assets/custom.js rename to src/civis_jupyter_notebooks/assets/custom.js diff --git a/civis_jupyter_notebooks/assets/extensions/terminal.js b/src/civis_jupyter_notebooks/assets/extensions/terminal.js similarity index 100% rename from civis_jupyter_notebooks/assets/extensions/terminal.js rename to src/civis_jupyter_notebooks/assets/extensions/terminal.js diff --git a/civis_jupyter_notebooks/assets/extensions/uncommitted_changes.js b/src/civis_jupyter_notebooks/assets/extensions/uncommitted_changes.js similarity index 100% rename from civis_jupyter_notebooks/assets/extensions/uncommitted_changes.js rename to src/civis_jupyter_notebooks/assets/extensions/uncommitted_changes.js diff --git a/civis_jupyter_notebooks/assets/fonts/civicons.eot b/src/civis_jupyter_notebooks/assets/fonts/civicons.eot similarity index 100% rename from civis_jupyter_notebooks/assets/fonts/civicons.eot rename to src/civis_jupyter_notebooks/assets/fonts/civicons.eot diff --git a/civis_jupyter_notebooks/assets/fonts/civicons.svg b/src/civis_jupyter_notebooks/assets/fonts/civicons.svg similarity index 100% rename from civis_jupyter_notebooks/assets/fonts/civicons.svg rename to src/civis_jupyter_notebooks/assets/fonts/civicons.svg diff --git a/civis_jupyter_notebooks/assets/fonts/civicons.ttf b/src/civis_jupyter_notebooks/assets/fonts/civicons.ttf similarity index 100% rename from civis_jupyter_notebooks/assets/fonts/civicons.ttf rename to src/civis_jupyter_notebooks/assets/fonts/civicons.ttf diff --git a/civis_jupyter_notebooks/assets/fonts/civicons.woff b/src/civis_jupyter_notebooks/assets/fonts/civicons.woff similarity index 100% rename from civis_jupyter_notebooks/assets/fonts/civicons.woff rename to src/civis_jupyter_notebooks/assets/fonts/civicons.woff diff --git a/civis_jupyter_notebooks/assets/fonts/config.json b/src/civis_jupyter_notebooks/assets/fonts/config.json similarity index 100% rename from civis_jupyter_notebooks/assets/fonts/config.json rename to src/civis_jupyter_notebooks/assets/fonts/config.json diff --git a/civis_jupyter_notebooks/assets/initialize-git b/src/civis_jupyter_notebooks/assets/initialize-git similarity index 100% rename from civis_jupyter_notebooks/assets/initialize-git rename to src/civis_jupyter_notebooks/assets/initialize-git diff --git a/src/civis_jupyter_notebooks/assets/ipython_config.py b/src/civis_jupyter_notebooks/assets/ipython_config.py new file mode 100644 index 0000000..102b575 --- /dev/null +++ b/src/civis_jupyter_notebooks/assets/ipython_config.py @@ -0,0 +1,4 @@ +c = get_config() # noqa + +c.InteractiveShellApp.extensions.append("civis_jupyter_ext") +c.InteractiveShellApp.exec_files.append("civis_client_config.py") diff --git a/civis_jupyter_notebooks/assets/jupyter_notebook_config.py b/src/civis_jupyter_notebooks/assets/jupyter_notebook_config.py similarity index 75% rename from civis_jupyter_notebooks/assets/jupyter_notebook_config.py rename to src/civis_jupyter_notebooks/assets/jupyter_notebook_config.py index a8905bf..bd30d0d 100644 --- a/civis_jupyter_notebooks/assets/jupyter_notebook_config.py +++ b/src/civis_jupyter_notebooks/assets/jupyter_notebook_config.py @@ -1,11 +1,13 @@ """Jupyter configuration file Note that the code in this file gets executed in a python process that is independent -of any python process in a kernel run by Jupyter (e.g., an ipython kernel). Thus we can make -changes to the installed packages to use in the ipython kernel here without worrying about -them not being reimported. +of any python process in a kernel run by Jupyter (e.g., an ipython kernel). +Thus we can make +changes to the installed packages to use in the ipython kernel here without worrying +about them not being reimported. """ + from civis_jupyter_notebooks import notebook_config -c = get_config() # noqa +c = get_config() # noqa notebook_config.civis_setup(c) diff --git a/civis_jupyter_notebooks/extensions/__init__.py b/src/civis_jupyter_notebooks/extensions/__init__.py similarity index 100% rename from civis_jupyter_notebooks/extensions/__init__.py rename to src/civis_jupyter_notebooks/extensions/__init__.py diff --git a/civis_jupyter_notebooks/extensions/git/__init__.py b/src/civis_jupyter_notebooks/extensions/git/__init__.py similarity index 100% rename from civis_jupyter_notebooks/extensions/git/__init__.py rename to src/civis_jupyter_notebooks/extensions/git/__init__.py diff --git a/civis_jupyter_notebooks/extensions/git/uncommitted_changes.py b/src/civis_jupyter_notebooks/extensions/git/uncommitted_changes.py similarity index 69% rename from civis_jupyter_notebooks/extensions/git/uncommitted_changes.py rename to src/civis_jupyter_notebooks/extensions/git/uncommitted_changes.py index 829f610..866421c 100644 --- a/civis_jupyter_notebooks/extensions/git/uncommitted_changes.py +++ b/src/civis_jupyter_notebooks/extensions/git/uncommitted_changes.py @@ -13,7 +13,7 @@ def get(self): civis_git = CivisGit() if not civis_git.is_git_enabled(): - self.set_status(404, 'Not a git enabled notebook') + self.set_status(404, "Not a git enabled notebook") else: has_changes = False try: @@ -21,7 +21,7 @@ def get(self): except CivisGitError: pass - response['dirty'] = has_changes + response["dirty"] = has_changes self.set_status(200) self.finish(response) @@ -32,10 +32,13 @@ def log_func(handler): def load_jupyter_server_extension(nbapp): - nbapp.log.info('Uncommitted Changes Ext. Loaded') + nbapp.log.info("Uncommitted Changes Ext. Loaded") webapp = nbapp.web_app - base_url = webapp.settings['base_url'] - webapp.add_handlers(".*$", [ - (ujoin(base_url, r"/git/uncommitted_changes"), UncommittedChangesHandler), - ]) + base_url = webapp.settings["base_url"] + webapp.add_handlers( + ".*$", + [ + (ujoin(base_url, r"/git/uncommitted_changes"), UncommittedChangesHandler), + ], + ) diff --git a/civis_jupyter_notebooks/git_utils.py b/src/civis_jupyter_notebooks/git_utils.py similarity index 75% rename from civis_jupyter_notebooks/git_utils.py rename to src/civis_jupyter_notebooks/git_utils.py index 3e3f0c6..a34225d 100644 --- a/civis_jupyter_notebooks/git_utils.py +++ b/src/civis_jupyter_notebooks/git_utils.py @@ -3,11 +3,15 @@ from git.exc import GitCommandError -class CivisGit(): +class CivisGit: def __init__(self, repo_url=None, repo_mount_path=None, git_repo_ref=None): - self.repo_url = os.environ.get('GIT_REPO_URL', repo_url) - self.git_repo_mount_path = repo_mount_path if repo_mount_path else os.path.expanduser(os.path.join('~', 'work')) - self.git_repo_ref = os.environ.get('GIT_REPO_REF', git_repo_ref) + self.repo_url = os.environ.get("GIT_REPO_URL", repo_url) + self.git_repo_mount_path = ( + repo_mount_path + if repo_mount_path + else os.path.expanduser(os.path.join("~", "work")) + ) + self.git_repo_ref = os.environ.get("GIT_REPO_REF", git_repo_ref) def repo(self): return Repo(self.git_repo_mount_path) diff --git a/civis_jupyter_notebooks/notebook_config.py b/src/civis_jupyter_notebooks/notebook_config.py similarity index 65% rename from civis_jupyter_notebooks/notebook_config.py rename to src/civis_jupyter_notebooks/notebook_config.py index de3775d..95d2420 100644 --- a/civis_jupyter_notebooks/notebook_config.py +++ b/src/civis_jupyter_notebooks/notebook_config.py @@ -5,16 +5,18 @@ from civis_jupyter_notebooks.git_utils import CivisGit -ROOT_DIR = os.path.expanduser(os.path.join('~', 'work')) +ROOT_DIR = os.path.expanduser(os.path.join("~", "work")) def get_notebook(notebook_full_path): try: platform_persistence.initialize_notebook_from_platform(notebook_full_path) - platform_persistence.post_save({'type': 'notebook'}, notebook_full_path, None) + platform_persistence.post_save({"type": "notebook"}, notebook_full_path, None) except platform_persistence.NotebookManagementError as e: platform_persistence.logger.error(str(e)) - platform_persistence.logger.warn('Killing the notebook process b/c of a startup issue') + platform_persistence.logger.warn( + "Killing the notebook process b/c of a startup issue" + ) os.kill(os.getpid(), signal.SIGTERM) @@ -28,20 +30,22 @@ def find_and_install_requirements(requirements_path, c): def config_jupyter(c): # Jupyter Configuration - c.NotebookApp.ip = '0.0.0.0' # nosec - c.NotebookApp.allow_origin = '*' + c.NotebookApp.ip = "0.0.0.0" # nosec + c.NotebookApp.allow_origin = "*" c.NotebookApp.port = 8888 c.NotebookApp.open_browser = False - c.NotebookApp.token = '' # nosec + c.NotebookApp.token = "" # nosec c.NotebookApp.disable_check_xsrf = True - c.NotebookApp.tornado_settings = {'headers': {'Content-Security-Policy': "frame-ancestors *"}} - c.NotebookApp.terminado_settings = {'shell_command': ['bash']} + c.NotebookApp.tornado_settings = { + "headers": {"Content-Security-Policy": "frame-ancestors *"} + } + c.NotebookApp.terminado_settings = {"shell_command": ["bash"]} c.NotebookApp.allow_root = True c.NotebookApp.nbserver_extensions = { - 'civis_jupyter_notebooks.extensions.git.uncommitted_changes': True + "civis_jupyter_notebooks.extensions.git.uncommitted_changes": True } c.FileContentsManager.post_save_hook = platform_persistence.post_save - c.MultiKernelManager.default_kernel_name = os.environ['DEFAULT_KERNEL'] + c.MultiKernelManager.default_kernel_name = os.environ["DEFAULT_KERNEL"] def stage_new_notebook(notebook_file_path): @@ -54,9 +58,9 @@ def stage_new_notebook(notebook_file_path): def civis_setup(c): config_jupyter(c) - nb_file_path = os.environ.get('NOTEBOOK_FILE_PATH', 'notebook.ipynb').strip('/') + nb_file_path = os.environ.get("NOTEBOOK_FILE_PATH", "notebook.ipynb").strip("/") notebook_full_path = os.path.join(ROOT_DIR, nb_file_path) - c.NotebookApp.default_url = '/notebooks/{}'.format(nb_file_path) + c.NotebookApp.default_url = "/notebooks/{}".format(nb_file_path) get_notebook(notebook_full_path) stage_new_notebook(nb_file_path) diff --git a/civis_jupyter_notebooks/platform_persistence.py b/src/civis_jupyter_notebooks/platform_persistence.py similarity index 50% rename from civis_jupyter_notebooks/platform_persistence.py rename to src/civis_jupyter_notebooks/platform_persistence.py index eef5883..b1b307f 100644 --- a/civis_jupyter_notebooks/platform_persistence.py +++ b/src/civis_jupyter_notebooks/platform_persistence.py @@ -2,74 +2,80 @@ This file contains utilities that bind the Jupyter notebook to our platform. It performs two functions: 1. On startup, pull the contents of the notebook from platform to the local disk - 2. As a Jupyter post-save hook, push the contents of the notebook and a HTML preview of the same back to platform. + 2. As a Jupyter post-save hook, push the contents of the notebook and a HTML preview + of the same back to platform. 3. Custom Error class for when a Notebook does not correctly initialize """ -import civis -import nbformat + +import logging import os -import sys import subprocess # nosec -import requests -from io import open +import sys from subprocess import check_call # nosec from subprocess import CalledProcessError # nosec -from civis_jupyter_notebooks import log_utils + +import civis +import nbformat +import requests def initialize_notebook_from_platform(notebook_path): - """ This runs on startup to initialize the notebook """ - logger.info('Retrieving notebook information from Platform') + """This runs on startup to initialize the notebook""" + logger.info("Retrieving notebook information from Platform") client = get_client() - notebook_model = client.notebooks.get(os.environ['PLATFORM_OBJECT_ID']) + notebook_model = client.notebooks.get(os.environ["PLATFORM_OBJECT_ID"]) - logger.info('Pulling contents of notebook file from S3') + logger.info("Pulling contents of notebook file from S3") r = requests.get(notebook_model.notebook_url, timeout=60) if r.status_code != 200: - raise NotebookManagementError('Failed to pull down notebook file from S3') + raise NotebookManagementError("Failed to pull down notebook file from S3") notebook = nbformat.reads(r.content, nbformat.NO_CONVERT) - s3_notebook_new = notebook.get('metadata', {}).get('civis', {}).get('new_notebook', False) + s3_notebook_new = ( + notebook.get("metadata", {}).get("civis", {}).get("new_notebook", False) + ) if s3_notebook_new: - notebook.metadata.pop('civis') + notebook.metadata.pop("civis") # Only overwrite the git version of the notebook with the S3 version if # the S3 version is not the brand new empty template git_notebook_exists = os.path.isfile(notebook_path) if not git_notebook_exists or not s3_notebook_new: - logger.info('Restoring notebook file from S3') + logger.info("Restoring notebook file from S3") directory = os.path.dirname(notebook_path) if not os.path.exists(directory): os.makedirs(directory) - with open(notebook_path, mode='w', encoding='utf-8') as nb_file: + with open(notebook_path, mode="w", encoding="utf-8") as nb_file: nbformat.write(notebook, nb_file) - logger.info('Notebook file ready') + logger.info("Notebook file ready") - if hasattr(notebook_model, 'requirements_url') and notebook_model.requirements_url: + if hasattr(notebook_model, "requirements_url") and notebook_model.requirements_url: __pull_and_load_requirements(notebook_model.requirements_url, notebook_path) def __pull_and_load_requirements(url, notebook_path): - logger.info('Pulling down the requirements file') + logger.info("Pulling down the requirements file") r = requests.get(url, timeout=60) if r.status_code != 200: - raise NotebookManagementError('Failed to pull down requirements.txt file from S3') + raise NotebookManagementError( + "Failed to pull down requirements.txt file from S3" + ) - logger.info('Writing contents of requirements file') - requirements_path = os.path.join(os.path.dirname(notebook_path), 'requirements.txt') - with open(requirements_path, 'wb') as requirements: + logger.info("Writing contents of requirements file") + requirements_path = os.path.join(os.path.dirname(notebook_path), "requirements.txt") + with open(requirements_path, "wb") as requirements: requirements.write(r.content) - logger.info('Requirements file ready') + logger.info("Requirements file ready") def find_and_install_requirements(requirements_path): - while os.path.isdir(requirements_path) and requirements_path != '/root': - requirements_file = os.path.join(requirements_path, 'requirements.txt') - logger.info('Looking for requirements at %s' % requirements_file) + while os.path.isdir(requirements_path) and requirements_path != "/root": + requirements_file = os.path.join(requirements_path, "requirements.txt") + logger.info("Looking for requirements at %s" % requirements_file) if not os.path.isfile(requirements_file): requirements_path = os.path.dirname(requirements_path) continue @@ -79,74 +85,82 @@ def find_and_install_requirements(requirements_path): def pip_install(requirements_file): - logger.info('Installing packages from %s' % requirements_file) + logger.info("Installing packages from %s" % requirements_file) try: subprocess.check_output( # nosec - [sys.executable, '-m', 'pip', 'install', '-r', requirements_file], - stderr=subprocess.STDOUT - ) - logger.info('Installed requirements.txt') + [sys.executable, "-m", "pip", "install", "-r", requirements_file], + stderr=subprocess.STDOUT, + ) + logger.info("Installed requirements.txt") except subprocess.CalledProcessError as e: raise NotebookManagementError(e.output.decode("utf-8")) def post_save(model, os_path, contents_manager): - """ Called from Jupyter post-save hook. Manages save of NB """ + """Called from Jupyter post-save hook. Manages save of NB""" - if model['type'] != 'notebook': + if model["type"] != "notebook": return - logger.info('Getting URLs to update notebook') + logger.info("Getting URLs to update notebook") update_url, update_preview_url = get_update_urls() save_notebook(update_url, os_path) generate_and_save_preview(update_preview_url, os_path) - logger.info('Notebook save complete') + logger.info("Notebook save complete") def get_update_urls(): """ - Get the URLs needed to update the NB. - These URLs expire after a few minutes so do not cache them + Get the URLs needed to update the NB. + These URLs expire after a few minutes so do not cache them """ client = get_client() - urls = client.notebooks.list_update_links(os.environ['PLATFORM_OBJECT_ID']) + urls = client.notebooks.list_update_links(os.environ["PLATFORM_OBJECT_ID"]) return (urls.update_url, urls.update_preview_url) def save_notebook(url, os_path): - """ Push raw notebook to S3 """ - with open(os_path, 'rb') as nb_file: - logger.info('Pushing latest notebook file to S3') + """Push raw notebook to S3""" + with open(os_path, "rb") as nb_file: + logger.info("Pushing latest notebook file to S3") requests.put(url, data=nb_file.read(), timeout=60) - logger.info('Notebook file updated') + logger.info("Notebook file updated") def generate_and_save_preview(url, os_path): - """ Render NB-as-HTML and push that file to S3 """ + """Render NB-as-HTML and push that file to S3""" d, fname = os.path.split(os_path) - logger.info('Rendering notebook to HTML') + logger.info("Rendering notebook to HTML") try: - check_call(['jupyter', 'nbconvert', '--to', 'html', fname], cwd=d) # nosec + check_call(["jupyter", "nbconvert", "--to", "html", fname], cwd=d) # nosec except CalledProcessError as e: - raise NotebookManagementError('nbconvert failed to convert notebook file to html: {}'.format(repr(e))) + raise NotebookManagementError( + "nbconvert failed to convert notebook file to html: {}".format(repr(e)) + ) - preview_path = os.path.splitext(os_path)[0] + '.html' - with open(preview_path, 'rb') as preview_file: - logger.info('Pushing latest notebook preview to S3') + preview_path = os.path.splitext(os_path)[0] + ".html" + with open(preview_path, "rb") as preview_file: + logger.info("Pushing latest notebook preview to S3") requests.put(url, data=preview_file.read(), timeout=60) - logger.info('Notebook preview updated') + logger.info("Notebook preview updated") def get_client(): - """ This gets a client that knows about our notebook endpoints """ + """This gets a client that knows about our notebook endpoints""" return civis.APIClient() class NotebookManagementError(Exception): - ''' + """ raised whenever we hit an error trying to move notebook data between our notebook and platform - ''' + """ -logger = log_utils.setup_stream_logging() +logger = civis.civis_logger( + name="CIVIS_PLATFORM_BACKEND", + level=logging.INFO, + fmt=logging.Formatter( + fmt="[%(levelname).1s %(asctime)s %(name)s] %(message)s", datefmt="%H:%M:%S" + ), +) diff --git a/tests/Dockerfile b/tests/Dockerfile index 58e8647..eb8b38b 100644 --- a/tests/Dockerfile +++ b/tests/Dockerfile @@ -1,13 +1,10 @@ -FROM python:3.10.4-slim-buster +FROM python:3.12.3-slim RUN apt-get update \ && apt-get install -y --no-install-recommends git \ && apt-get purge -y --auto-remove \ && rm -rf /var/lib/apt/lists/* -COPY ./requirements.txt /root/requirements.txt -RUN pip install -r /root/requirements.txt - # Add Tini ENV TINI_VERSION v0.19.0 ADD https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini /tini diff --git a/civis_jupyter_notebooks/tests/__init__.py b/tests/__init__.py similarity index 100% rename from civis_jupyter_notebooks/tests/__init__.py rename to tests/__init__.py diff --git a/civis_jupyter_notebooks/tests/extensions/__init__.py b/tests/extensions/__init__.py similarity index 100% rename from civis_jupyter_notebooks/tests/extensions/__init__.py rename to tests/extensions/__init__.py diff --git a/civis_jupyter_notebooks/tests/extensions/git/__init__.py b/tests/extensions/git/__init__.py similarity index 100% rename from civis_jupyter_notebooks/tests/extensions/git/__init__.py rename to tests/extensions/git/__init__.py diff --git a/civis_jupyter_notebooks/tests/extensions/git/test_uncommited_changes.py b/tests/extensions/git/test_uncommited_changes.py similarity index 72% rename from civis_jupyter_notebooks/tests/extensions/git/test_uncommited_changes.py rename to tests/extensions/git/test_uncommited_changes.py index 3671f97..f36b5df 100644 --- a/civis_jupyter_notebooks/tests/extensions/git/test_uncommited_changes.py +++ b/tests/extensions/git/test_uncommited_changes.py @@ -1,7 +1,9 @@ import unittest from unittest.mock import patch, MagicMock -from civis_jupyter_notebooks.extensions.git.uncommitted_changes import UncommittedChangesHandler +from civis_jupyter_notebooks.extensions.git.uncommitted_changes import ( + UncommittedChangesHandler, +) from civis_jupyter_notebooks.git_utils import CivisGitError @@ -12,36 +14,38 @@ def setUp(self): self.handler.finish = MagicMock() self.handler.set_status = MagicMock() - @patch('civis_jupyter_notebooks.extensions.git.uncommitted_changes.CivisGit') + @patch("civis_jupyter_notebooks.extensions.git.uncommitted_changes.CivisGit") def test_get_will_return_404(self, civis_git): civis_git.return_value.is_git_enabled.return_value = False dummy_response = {} self.handler.get() - self.handler.set_status.assert_called_with(404, 'Not a git enabled notebook') + self.handler.set_status.assert_called_with(404, "Not a git enabled notebook") self.handler.finish.assert_called_with(dummy_response) - @patch('civis_jupyter_notebooks.extensions.git.uncommitted_changes.CivisGit') + @patch("civis_jupyter_notebooks.extensions.git.uncommitted_changes.CivisGit") def test_get_will_return_200(self, civis_git): civis_git.return_value.is_git_enabled.return_value = True civis_git.return_value.has_uncommitted_changes.return_value = True - dummy_response = {'dirty': True} + dummy_response = {"dirty": True} self.handler.get() civis_git.return_value.has_uncommitted_changes.assert_called_with() self.handler.set_status.assert_called_with(200) self.handler.finish.assert_called_with(dummy_response) - @patch('civis_jupyter_notebooks.extensions.git.uncommitted_changes.CivisGit') + @patch("civis_jupyter_notebooks.extensions.git.uncommitted_changes.CivisGit") def test_get_will_return_200_even_with_error(self, civis_git): civis_git.return_value.is_git_enabled.return_value = True - civis_git.return_value.has_uncommitted_changes.side_effect = CivisGitError('dummy error') - dummy_response = {'dirty': False} + civis_git.return_value.has_uncommitted_changes.side_effect = CivisGitError( + "dummy error" + ) + dummy_response = {"dirty": False} self.handler.get() self.handler.set_status.assert_called_with(200) self.handler.finish.assert_called_with(dummy_response) -if __name__ == '__main__': +if __name__ == "__main__": unittest.main() diff --git a/civis_jupyter_notebooks/tests/fixtures/sample_new_notebook.ipynb b/tests/fixtures/sample_new_notebook.ipynb similarity index 100% rename from civis_jupyter_notebooks/tests/fixtures/sample_new_notebook.ipynb rename to tests/fixtures/sample_new_notebook.ipynb diff --git a/civis_jupyter_notebooks/tests/fixtures/sample_notebook.ipynb b/tests/fixtures/sample_notebook.ipynb similarity index 100% rename from civis_jupyter_notebooks/tests/fixtures/sample_notebook.ipynb rename to tests/fixtures/sample_notebook.ipynb diff --git a/tests/test_ext.py b/tests/test_ext.py index fa2a42a..cb2983a 100644 --- a/tests/test_ext.py +++ b/tests/test_ext.py @@ -1,4 +1,4 @@ import IPython ipython = IPython.get_ipython() -ipython.run_line_magic('load_ext', 'civis_jupyter_ext') +ipython.run_line_magic("load_ext", "civis_jupyter_ext") diff --git a/civis_jupyter_notebooks/tests/test_git_utils.py b/tests/test_git_utils.py similarity index 77% rename from civis_jupyter_notebooks/tests/test_git_utils.py rename to tests/test_git_utils.py index 0d9673a..a2e0214 100644 --- a/civis_jupyter_notebooks/tests/test_git_utils.py +++ b/tests/test_git_utils.py @@ -7,24 +7,24 @@ from git import Repo from civis_jupyter_notebooks.git_utils import CivisGit, CivisGitError -REPO_URL = 'http://www.github.com/civisanalytics.foo.git' -REPO_MOUNT_PATH = '/root/work' -GIT_REPO_REF = 'master' +REPO_URL = "http://www.github.com/civisanalytics.foo.git" +REPO_MOUNT_PATH = "/root/work" +GIT_REPO_REF = "master" class GitUtilsTest(unittest.TestCase): def setUp(self): - os.environ['GIT_REPO_URL'] = REPO_URL - os.environ['GIT_REPO_REF'] = GIT_REPO_REF + os.environ["GIT_REPO_URL"] = REPO_URL + os.environ["GIT_REPO_REF"] = GIT_REPO_REF logging.disable(logging.INFO) - @patch('civis_jupyter_notebooks.git_utils.Repo.clone_from') + @patch("civis_jupyter_notebooks.git_utils.Repo.clone_from") def test_clone_repository_throws_error(self, repo_clone): - repo_clone.side_effect = GitCommandError('clone', 'failed') + repo_clone.side_effect = GitCommandError("clone", "failed") self.assertRaises(CivisGitError, lambda: CivisGit().clone_repository()) - @patch('civis_jupyter_notebooks.git_utils.Repo.clone_from') + @patch("civis_jupyter_notebooks.git_utils.Repo.clone_from") def test_clone_repository_succeeds(self, repo_clone): repo_clone.return_value = MagicMock(spec=Repo) CivisGit(repo_mount_path=REPO_MOUNT_PATH).clone_repository() @@ -32,7 +32,7 @@ def test_clone_repository_succeeds(self, repo_clone): repo_clone.assert_called_with(REPO_URL, REPO_MOUNT_PATH) repo_clone.return_value.git.checkout.assert_called_with(GIT_REPO_REF) - @patch('os.environ.get') + @patch("os.environ.get") def test_is_git_enabled_returns_false(self, env): env.return_value = None cg = CivisGit() @@ -43,9 +43,9 @@ def test_is_git_enabled_returns_true(self): def test_has_uncommitted_changes(self): def custom_side_effect(arg): - if arg == 'HEAD': + if arg == "HEAD": return [] - return ['foo.py'] + return ["foo.py"] cg = CivisGit() cg.repo = MagicMock(spec=Repo) @@ -63,9 +63,9 @@ def test_has_no_uncommitted_changes(self): def test_has_uncommitted_changes_throws_error(self): cg = CivisGit() cg.repo = MagicMock(spec=Repo) - cg.repo().index.diff = MagicMock(side_effect=GitCommandError('diff', 'failed')) + cg.repo().index.diff = MagicMock(side_effect=GitCommandError("diff", "failed")) self.assertRaises(CivisGitError, lambda: cg.has_uncommitted_changes()) -if __name__ == '__main__': +if __name__ == "__main__": unittest.main() diff --git a/tests/test_notebook_config.py b/tests/test_notebook_config.py new file mode 100644 index 0000000..698b2b3 --- /dev/null +++ b/tests/test_notebook_config.py @@ -0,0 +1,124 @@ +import unittest +from unittest.mock import patch, ANY +import os +from traitlets.config.loader import Config + +from civis_jupyter_notebooks import notebook_config, platform_persistence + + +class NotebookConfigTest(unittest.TestCase): + @patch.dict(os.environ, {"DEFAULT_KERNEL": "kern"}) + def test_config_jupyter_sets_notebook_config(self): + c = Config({}) + notebook_config.config_jupyter(c) + assert c.NotebookApp.port == 8888 + assert c.MultiKernelManager.default_kernel_name == "kern" + + @patch( + "civis_jupyter_notebooks.platform_persistence.initialize_notebook_from_platform" + ) + @patch("civis_jupyter_notebooks.platform_persistence.post_save") + def test_get_notebook_initializes_and_saves(self, post_save, init_notebook): + notebook_config.get_notebook("path") + init_notebook.assert_called_with("path") + post_save.assert_called_with(ANY, "path", None) + + @patch( + "civis_jupyter_notebooks.platform_persistence.initialize_notebook_from_platform" + ) + @patch("civis_jupyter_notebooks.platform_persistence.logger") + @patch("os.kill") + def test_get_notebook_writes_log_and_kills_proc_on_error( + self, kill, logger, init_notebook + ): + init_notebook.side_effect = platform_persistence.NotebookManagementError("err") + notebook_config.get_notebook("path") + logger.error.assert_called_with("err") + logger.warn.assert_called_with(ANY) + kill.assert_called_with(ANY, ANY) + + @patch("civis_jupyter_notebooks.platform_persistence.find_and_install_requirements") + def test_find_and_install_requirements_success( + self, persistence_find_and_install_requirements + ): + c = Config({}) + notebook_config.find_and_install_requirements("path", c) + persistence_find_and_install_requirements.assert_called_with("path") + + @patch("civis_jupyter_notebooks.platform_persistence.find_and_install_requirements") + @patch("civis_jupyter_notebooks.platform_persistence.logger") + def test_find_and_install_requirements_logs_errors( + self, logger, persistence_find_and_install_requirements + ): + persistence_find_and_install_requirements.side_effect = ( + platform_persistence.NotebookManagementError("err") + ) + c = Config({}) + notebook_config.find_and_install_requirements("path", c) + logger.error.assert_called_with("Unable to install requirements.txt:\nerr") + + @patch("civis_jupyter_notebooks.notebook_config.stage_new_notebook") + @patch("civis_jupyter_notebooks.notebook_config.config_jupyter") + @patch("civis_jupyter_notebooks.notebook_config.find_and_install_requirements") + @patch("civis_jupyter_notebooks.notebook_config.get_notebook") + def test_civis_setup_success( + self, + get_notebook, + find_and_install_requirements, + config_jupyter, + _stage_new_notebook, + ): + c = Config({}) + notebook_config.civis_setup(c) + + assert c.NotebookApp.default_url == "/notebooks/notebook.ipynb" + config_jupyter.assert_called_with(ANY) + get_notebook.assert_called_with(notebook_config.ROOT_DIR + "/notebook.ipynb") + find_and_install_requirements.assert_called_with(notebook_config.ROOT_DIR, ANY) + + @patch("civis_jupyter_notebooks.notebook_config.stage_new_notebook") + @patch("os.environ.get") + @patch("civis_jupyter_notebooks.notebook_config.config_jupyter") + @patch("civis_jupyter_notebooks.notebook_config.find_and_install_requirements") + @patch("civis_jupyter_notebooks.notebook_config.get_notebook") + def test_civis_setup_uses_environment_setting( + self, + get_notebook, + find_and_install_requirements, + config_jupyter, + environ_get, + _stage_new_notebook, + ): + + c = Config({}) + environ_get.return_value = "subpath/foo.ipynb" + notebook_config.civis_setup(c) + + assert c.NotebookApp.default_url == "/notebooks/subpath/foo.ipynb" + config_jupyter.assert_called_with(ANY) + get_notebook.assert_called_with(notebook_config.ROOT_DIR + "/subpath/foo.ipynb") + find_and_install_requirements.assert_called_with( + notebook_config.ROOT_DIR + "/subpath", ANY + ) + + @patch("civis_jupyter_notebooks.notebook_config.CivisGit") + def test_stage_new_notebook_stages_notebook_file(self, civis_git): + civis_git.return_value.is_git_enabled.return_value = True + repo = civis_git.return_value.repo.return_value + repo.untracked_files = ["notebook.ipynb"] + + notebook_config.stage_new_notebook("notebook.ipynb") + repo.index.add.assert_called_with(["notebook.ipynb"]) + + @patch("civis_jupyter_notebooks.notebook_config.CivisGit") + def test_stage_new_notebook_git_is_disabled(self, civis_git): + civis_git.return_value.is_git_enabled.return_value = False + notebook_config.stage_new_notebook("notebook.ipynb") + + civis_git.return_value.repo.assert_not_called() + + civis_git.return_value.repo.untracked_files.assert_not_called() + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_platform_persistence.py b/tests/test_platform_persistence.py new file mode 100644 index 0000000..ec6ef6e --- /dev/null +++ b/tests/test_platform_persistence.py @@ -0,0 +1,327 @@ +import os +import subprocess +import nbformat +import requests +import unittest +from unittest.mock import ANY, MagicMock, patch +import logging + +from civis_jupyter_notebooks import platform_persistence +from civis_jupyter_notebooks.platform_persistence import NotebookManagementError + +TEST_NOTEBOOK_PATH = "/path/to/notebook.ipynb" +TEST_PLATFORM_OBJECT_ID = "1914" +SAMPLE_NOTEBOOK = open( + os.path.join(os.path.dirname(__file__), "fixtures/sample_notebook.ipynb") +).read() +SAMPLE_NEW_NOTEBOOK = open( + os.path.join(os.path.dirname(__file__), "fixtures/sample_new_notebook.ipynb") +).read() + + +class NotebookWithoutNewFlag(nbformat.NotebookNode): + """Helper that tests if a NotebookNode has + the metadata.civis.new_notebook flag set to True""" + + def __eq__(self, other): + return not other.get("metadata", {}).get("civis", {}).get("new_notebook", False) + + +class PlatformPersistenceTest(unittest.TestCase): + def setUp(self): + os.environ["CIVIS_API_KEY"] = "hi mom" + os.environ["PLATFORM_OBJECT_ID"] = TEST_PLATFORM_OBJECT_ID + + logging.disable(logging.INFO) + + @patch("os.makedirs") + @patch("civis_jupyter_notebooks.platform_persistence.open") + @patch("civis.APIClient") + @patch("civis_jupyter_notebooks.platform_persistence.requests.get") + def test_initialize_notebook_will_get_nb_from_platform( + self, rg, _client, _op, _makedirs + ): + rg.return_value = MagicMock( + spec=requests.Response, status_code=200, content=SAMPLE_NOTEBOOK + ) + platform_persistence.initialize_notebook_from_platform(TEST_NOTEBOOK_PATH) + platform_persistence.get_client().notebooks.get.assert_called_with( + TEST_PLATFORM_OBJECT_ID + ) + + @patch("os.makedirs") + @patch("civis_jupyter_notebooks.platform_persistence.open") + @patch("civis_jupyter_notebooks.platform_persistence.__pull_and_load_requirements") + @patch("civis.APIClient") + @patch("civis_jupyter_notebooks.platform_persistence.requests.get") + def test_initialize_notebook_will_pull_nb_from_url( + self, rg, _client, requirements, _op, _makedirs + ): + url = "http://whatever" + rg.return_value = MagicMock( + spec=requests.Response, status_code=200, content=SAMPLE_NOTEBOOK + ) + platform_persistence.get_client().notebooks.get.return_value.notebook_url = url + platform_persistence.get_client().notebooks.get.return_value.requirements_url = ( # noqa: E501 + None + ) + platform_persistence.initialize_notebook_from_platform(TEST_NOTEBOOK_PATH) + rg.assert_called_with(url, timeout=60) + requirements.assert_not_called() + + @patch("os.makedirs") + @patch("civis_jupyter_notebooks.platform_persistence.open") + @patch("civis.APIClient") + @patch("civis_jupyter_notebooks.platform_persistence.requests.get") + def test_initialize_notebook_will_throw_error_on_nb_pull( + self, rg, _client, _op, _makedirs + ): + rg.return_value = MagicMock( + spec=requests.Response, status_code=500, response={} + ) + self.assertRaises( + NotebookManagementError, + lambda: platform_persistence.initialize_notebook_from_platform( + TEST_NOTEBOOK_PATH + ), + ) + + @patch("nbformat.write") + @patch("os.makedirs") + @patch("civis_jupyter_notebooks.platform_persistence.open") + @patch("civis.APIClient") + @patch("civis_jupyter_notebooks.platform_persistence.requests.get") + def test_initialize_notebook_will_set_new_notebook_flag_to_false( + self, rg, _client, _op, _makedirs, nbwrite + ): + rg.return_value = MagicMock( + spec=requests.Response, status_code=200, content=SAMPLE_NEW_NOTEBOOK + ) + platform_persistence.get_client().notebooks.get.return_value.notebooks_url = ( + "something" + ) + platform_persistence.get_client().notebooks.get.return_value.requirements_url = ( # noqa: E501 + None + ) + platform_persistence.initialize_notebook_from_platform(TEST_NOTEBOOK_PATH) + nbwrite.assert_called_with(NotebookWithoutNewFlag(), ANY) + + @patch("os.path.isfile") + @patch("nbformat.write") + @patch("os.makedirs") + @patch("civis_jupyter_notebooks.platform_persistence.open") + @patch("civis.APIClient") + @patch("civis_jupyter_notebooks.platform_persistence.requests.get") + def test_initialize_notebook_will_use_s3_notebook_if_not_new_and_git_notebook_exists( # noqa: E501 + self, rg, _client, _op, _makedirs, nbwrite, isfile + ): + rg.return_value = MagicMock( + spec=requests.Response, status_code=200, content=SAMPLE_NOTEBOOK + ) + platform_persistence.get_client().notebooks.get.return_value.notebooks_url = ( + "something" + ) + platform_persistence.get_client().notebooks.get.return_value.requirements_url = ( # noqa: E501 + None + ) + isfile.return_value = True + platform_persistence.initialize_notebook_from_platform(TEST_NOTEBOOK_PATH) + nbwrite.assert_called_with(ANY, ANY) + + @patch("os.path.isfile") + @patch("nbformat.write") + @patch("os.makedirs") + @patch("civis_jupyter_notebooks.platform_persistence.open") + @patch("civis.APIClient") + @patch("civis_jupyter_notebooks.platform_persistence.requests.get") + def test_initialize_notebook_will_discard_s3_notebook_if_new_and_git_notebook_exists( # noqa: E501 + self, rg, _client, _op, _makedirs, nbwrite, isfile + ): + rg.return_value = MagicMock( + spec=requests.Response, status_code=200, content=SAMPLE_NEW_NOTEBOOK + ) + platform_persistence.get_client().notebooks.get.return_value.notebooks_url = ( + "something" + ) + platform_persistence.get_client().notebooks.get.return_value.requirements_url = ( # noqa: E501 + None + ) + isfile.return_value = True + platform_persistence.initialize_notebook_from_platform(TEST_NOTEBOOK_PATH) + nbwrite.assert_not_called() + + @patch("civis_jupyter_notebooks.platform_persistence.open") + @patch("civis.APIClient") + @patch("os.makedirs") + @patch("civis_jupyter_notebooks.platform_persistence.requests.get") + def test_initialize_notebook_will_create_directories_if_needed( + self, rg, makedirs, _client, _op + ): + rg.return_value = MagicMock( + spec=requests.Response, status_code=200, content=SAMPLE_NOTEBOOK + ) + platform_persistence.initialize_notebook_from_platform(TEST_NOTEBOOK_PATH) + directory = os.path.dirname(TEST_NOTEBOOK_PATH) + makedirs.assert_called_with(directory) + + @patch("os.makedirs") + @patch("civis_jupyter_notebooks.platform_persistence.open") + @patch("civis_jupyter_notebooks.platform_persistence.__pull_and_load_requirements") + @patch("civis.APIClient") + @patch("civis_jupyter_notebooks.platform_persistence.requests.get") + def test_initialize_notebook_will_pull_requirements( + self, rg, _client, requirements, _op, _makedirs + ): + url = "http://whatever" + rg.return_value = MagicMock( + spec=requests.Response, status_code=200, content=SAMPLE_NOTEBOOK + ) + platform_persistence.get_client().notebooks.get.return_value.requirements_url = ( # noqa: E501 + url + ) + platform_persistence.initialize_notebook_from_platform(TEST_NOTEBOOK_PATH) + requirements.assert_called_with(url, TEST_NOTEBOOK_PATH) + + @patch("os.makedirs") + @patch("civis_jupyter_notebooks.platform_persistence.open") + @patch("civis_jupyter_notebooks.platform_persistence.__pull_and_load_requirements") + @patch("civis.APIClient") + @patch("civis_jupyter_notebooks.platform_persistence.requests.get") + def test_initialize_notebook_will_error_on_requirements_pull( + self, rg, _client, _requirements, _op, _makedirs + ): + url = "http://whatever" + rg.return_value = MagicMock(spec=requests.Response, status_code=500) + platform_persistence.get_client().notebooks.get.return_value.requirements_url = ( # noqa: E501 + url + ) + self.assertRaises( + NotebookManagementError, + lambda: platform_persistence.initialize_notebook_from_platform( + TEST_NOTEBOOK_PATH + ), + ) + + @patch("civis_jupyter_notebooks.platform_persistence.open") + @patch("civis_jupyter_notebooks.platform_persistence.check_call") + @patch("civis.APIClient") + @patch("requests.put") + def test_post_save_fetches_urls_from_api(self, _rput, client, _ccc, _op): + platform_persistence.post_save({"type": "notebook"}, "", {}) + platform_persistence.get_client().notebooks.list_update_links.assert_called_with( # noqa: E501 + TEST_PLATFORM_OBJECT_ID + ) + + @patch("civis_jupyter_notebooks.platform_persistence.open") + @patch("civis_jupyter_notebooks.platform_persistence.check_call") + @patch("civis.APIClient") + @patch("requests.put") + @patch("civis_jupyter_notebooks.platform_persistence.save_notebook") + def test_post_save_performs_two_put_operations( + self, save, rput, _client, _ccc, _op + ): + platform_persistence.post_save({"type": "notebook"}, "", {}) + self.assertTrue(save.called) + + @patch("civis_jupyter_notebooks.platform_persistence.open") + @patch("civis_jupyter_notebooks.platform_persistence.check_call") + @patch("civis.APIClient") + @patch("requests.put") + @patch("civis_jupyter_notebooks.platform_persistence.save_notebook") + @patch("civis_jupyter_notebooks.platform_persistence.get_update_urls") + def test_post_save_skipped_for_non_notebook_types( + self, guu, save, _rput, _client, _ccc, _op + ): + platform_persistence.post_save({"type": "blargggg"}, "", {}) + self.assertFalse(guu.called) + self.assertFalse(save.called) + + @patch("civis_jupyter_notebooks.platform_persistence.open") + @patch("civis_jupyter_notebooks.platform_persistence.check_call") + @patch("civis.APIClient") + @patch("requests.put") + def test_post_save_generates_preview(self, _rput, _client, check_call, _op): + platform_persistence.post_save({"type": "notebook"}, "x/y", {}) + check_call.assert_called_with( + ["jupyter", "nbconvert", "--to", "html", "y"], cwd="x" + ) + + @patch("civis_jupyter_notebooks.platform_persistence.open") + @patch("civis_jupyter_notebooks.platform_persistence.check_call") + @patch("civis.APIClient") + @patch("requests.put") + def test_generate_preview_throws_error_on_convert( + self, _rput, _client, check_call, _op + ): + check_call.side_effect = subprocess.CalledProcessError("foo", 255) + self.assertRaises( + NotebookManagementError, + lambda: platform_persistence.generate_and_save_preview( + "http://notebook_url_in_s3", "os/path" + ), + ) + check_call.assert_called_with( + ["jupyter", "nbconvert", "--to", "html", "path"], cwd="os" + ) + + @patch("civis.APIClient") + def test_will_regenerate_api_client(self, mock_client): + platform_persistence.get_client() + mock_client.assert_called_with() + + @patch("os.path.isfile") + @patch("os.path.isdir") + @patch("civis_jupyter_notebooks.platform_persistence.pip_install") + def test_find_and_install_requirements_calls_pip_install( + self, pip_install, isdir, isfile + ): + os.path.isdir.return_value = True + os.path.isfile.return_value = True + platform_persistence.find_and_install_requirements("/root/work/foo") + pip_install.assert_called_with("/root/work/foo/requirements.txt") + + @patch("os.path.isfile") + @patch("os.path.isdir") + @patch("civis_jupyter_notebooks.platform_persistence.pip_install") + def test_find_and_install_requirements_searches_tree( + self, pip_install, isdir, isfile + ): + os.path.isdir.return_value = True + os.path.isfile.side_effect = [False, True] + platform_persistence.find_and_install_requirements("/root/work/foo") + pip_install.assert_called_with("/root/work/requirements.txt") + + @patch("os.path.isfile") + @patch("os.path.isdir") + @patch("civis_jupyter_notebooks.platform_persistence.pip_install") + def test_find_and_install_requirements_excludes_root( + self, pip_install, isdir, isfile + ): + os.path.isdir.return_value = True + os.path.isfile.return_value = True + platform_persistence.find_and_install_requirements("/root") + pip_install.assert_not_called() + + @patch("subprocess.check_output") + @patch("sys.executable") + def test_pip_install_calls_subprocess(self, executable, check_output): + platform_persistence.pip_install("/path/requirements.txt") + check_output.assert_called_with( + [executable, "-m", "pip", "install", "-r", "/path/requirements.txt"], + stderr=subprocess.STDOUT, + ) + + @patch("subprocess.check_output") + @patch("sys.executable") + def test_pip_install_failure_raises_notebookmanagementerror( + self, executable, check_output + ): + check_output.side_effect = subprocess.CalledProcessError( + returncode=1, cmd="cmd", output=b"installation error" + ) + with self.assertRaisesRegex(NotebookManagementError, "installation error"): + platform_persistence.pip_install("/path/requirements.txt") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_version.py b/tests/test_version.py new file mode 100644 index 0000000..b969539 --- /dev/null +++ b/tests/test_version.py @@ -0,0 +1,17 @@ +import os +import re + +import civis_jupyter_notebooks + + +_REPO_DIR = os.path.dirname(os.path.dirname(__file__)) + + +def test_version_number_match_with_changelog(): + """__version__ and CHANGELOG.md match for the latest version number.""" + changelog = open(os.path.join(_REPO_DIR, "CHANGELOG.md")).read() + version_in_changelog = re.search(r"##\s+\[(\d+\.\d+\.\d+)]", changelog).groups()[0] + assert civis_jupyter_notebooks.__version__ == version_in_changelog, ( + "Make sure both __version__ and CHANGELOG are updated to match the " + "latest version number" + )