diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e9b1ef7..400b86d 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -4,7 +4,7 @@ on: push: branches: master pull_request: - branches: "*" + branches: '*' jobs: build: @@ -70,7 +70,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.8", "3.9", "3.10"] + python-version: ['3.8', '3.9', '3.10'] steps: - name: Checkout @@ -91,15 +91,24 @@ jobs: python -m pip install tljh_repo2docker*.whl npm -g install configurable-http-proxy - - name: Run Tests + - name: Run local build backend tests + working-directory: tljh_repo2docker/tests run: | - python -m pytest --cov + python -m pytest local_build --cov + + - name: Run binderhub build backend tests + working-directory: tljh_repo2docker/tests + run: | + python -m pytest binderhub_build --cov integration-tests: name: Integration tests needs: build runs-on: ubuntu-latest - + strategy: + fail-fast: false + matrix: + build-backend: ['local', 'binderhub'] env: PLAYWRIGHT_BROWSERS_PATH: ${{ github.workspace }}/pw-browsers @@ -139,10 +148,10 @@ jobs: run: npx playwright install chromium working-directory: ui-tests - - name: Execute integration tests + - name: Execute integration tests with ${{ matrix.build-backend }} build backend working-directory: ui-tests run: | - npx playwright test + npm run test:${{ matrix.build-backend }} - name: Upload Playwright Test report if: always() @@ -150,5 +159,6 @@ jobs: with: name: tljh-playwright-tests path: | - ui-tests/test-results + ui-tests/local-test-results + ui-tests/binderhub-test-results ui-tests/playwright-report diff --git a/.gitignore b/.gitignore index 0f8645b..41869a8 100644 --- a/.gitignore +++ b/.gitignore @@ -30,3 +30,4 @@ lib/ # Hatch version _version.py +*.sqlite \ No newline at end of file diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 459c00f..59add32 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -46,10 +46,16 @@ docker pull quay.io/jupyterhub/repo2docker:main ## Run -Finally, start `jupyterhub` with the config in `debug` mode: +Finally, start `jupyterhub` with local build backend: ```bash -python -m jupyterhub -f jupyterhub_config.py --debug +python -m jupyterhub -f ui-tests/jupyterhub_config_local.py --debug +``` + +or using `binderhub` build backend + +```bash +python -m jupyterhub -f ui-tests/jupyterhub_config_binderhub.py --debug ``` Open https://localhost:8000 in a web browser. diff --git a/README.md b/README.md index 2df4ef6..6479de2 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ ![Github Actions Status](https://github.com/plasmabio/tljh-repo2docker/workflows/Tests/badge.svg) -TLJH plugin providing a JupyterHub service to build and use Docker images as user environments. The Docker images are built using [`repo2docker`](https://repo2docker.readthedocs.io/en/latest/). +TLJH plugin provides a JupyterHub service to build and use Docker images as user environments. The Docker images can be built locally using [`repo2docker`](https://repo2docker.readthedocs.io/en/latest/) or via the [`binderhub`](https://binderhub.readthedocs.io/en/latest/) service. ## Requirements @@ -46,8 +46,10 @@ The available settings for this service are: - `default_memory_limit`: Default memory limit of a user server; defaults to `None` - `default_cpu_limit`: Default CPU limit of a user server; defaults to `None` - `machine_profiles`: Instead of entering directly the CPU and Memory value, `tljh-repo2docker` can be configured with pre-defined machine profiles and users can only choose from the available option; defaults to `[]` +- `binderhub_url`: The optional URL of the `binderhub` service. If it is available, `tljh-repo2docker` will use this service to build images. +- `db_url`: The connection string of the database. `tljh-repo2docker` needs a database to store the image metadata. By default, it will create a `sqlite` database in the starting directory of the service. To use other databases (`PostgreSQL` or `MySQL`), users need to specify the connection string via this config and install the additional drivers (`asyncpg` or `aiomysql`). -This service requires the following scopes : `read:users`, `admin:servers` and `read:roles:users`. Here is an example of registering `tljh_repo2docker`'s service with JupyterHub +This service requires the following scopes : `read:users`, `admin:servers` and `read:roles:users`. If `binderhub` service is used, ` access:services!service=binder`is also needed. Here is an example of registering `tljh_repo2docker`'s service with JupyterHub ```python # jupyterhub_config.py @@ -78,7 +80,12 @@ c.JupyterHub.load_roles = [ { "description": "Role for tljh_repo2docker service", "name": "tljh-repo2docker-service", - "scopes": ["read:users", "admin:servers", "read:roles:users"], + "scopes": [ + "read:users", + "read:roles:users", + "admin:servers", + "access:services!service=binder", + ], "services": ["tljh_repo2docker"], }, { @@ -147,25 +154,30 @@ c.JupyterHub.load_roles = [ The _Environments_ page shows the list of built environments, as well as the ones currently being built: -![environments](https://raw.githubusercontent.com/plasmabio/tljh-repo2docker/master/ui-tests/tests/ui.test.ts-snapshots/environment-list-linux.png) +![environments](https://raw.githubusercontent.com/plasmabio/tljh-repo2docker/master/ui-tests/local_snapshots/ui.test.ts/environment-list.png) ### Add a new environment Just like on [Binder](https://mybinder.org), new environments can be added by clicking on the _Add New_ button and providing a URL to the repository. Optional names, memory, and CPU limits can also be set for the environment: -![add-new](https://raw.githubusercontent.com/plasmabio/tljh-repo2docker/master/ui-tests/tests/ui.test.ts-snapshots/environment-dialog-linux.png) +![add-new](https://raw.githubusercontent.com/plasmabio/tljh-repo2docker/master/ui-tests/local_snapshots/ui.test.ts/environment-dialog.png) + +> [!NOTE] +> If the build backend is `binderhub` service, users need to select the [repository provider](https://binderhub.readthedocs.io/en/latest/developer/repoproviders.html) and can not specify the custom build arguments + +![add-new-binderhub](https://raw.githubusercontent.com/plasmabio/tljh-repo2docker/master/ui-tests/binderhub_snapshots/ui.test.ts/environment-dialog.png) ### Follow the build logs Clicking on the _Logs_ button will open a new dialog with the build logs: -![logs](https://raw.githubusercontent.com/plasmabio/tljh-repo2docker/master/ui-tests/tests/ui.test.ts-snapshots/environment-console-linux.png) +![logs](https://raw.githubusercontent.com/plasmabio/tljh-repo2docker/master/ui-tests/local_snapshots/ui.test.ts/environment-console.png) ### Select an environment Once ready, the environments can be selected from the JupyterHub spawn page: -![select-env](https://raw.githubusercontent.com/plasmabio/tljh-repo2docker/master/ui-tests/tests/ui.test.ts-snapshots/servers-dialog-linux.png) +![select-env](https://raw.githubusercontent.com/plasmabio/tljh-repo2docker/master/ui-tests/local_snapshots/ui.test.ts/servers-dialog.png) ### Private Repositories @@ -173,12 +185,15 @@ Once ready, the environments can be selected from the JupyterHub spawn page: It is possible to provide the `username` and `password` in the `Credentials` section of the form: -![image](https://raw.githubusercontent.com/plasmabio/tljh-repo2docker/master/ui-tests/tests/ui.test.ts-snapshots/environment-dialog-linux.png) +![image](https://raw.githubusercontent.com/plasmabio/tljh-repo2docker/master/ui-tests/local_snapshots/ui.test.ts/environment-dialog.png) On GitHub and GitLab, a user might have to first create an access token with `read` access to use as the password: ![image](https://user-images.githubusercontent.com/591645/107350843-39c3bf80-6aca-11eb-8b82-6fa95ba4c7e4.png) +> [!NOTE] +> The `binderhub` build backend does not support configuring private repositories credentials from the interface. + ### Machine profiles Instead of entering directly the CPU and Memory value, `tljh-repo2docker` can be configured with pre-defined machine profiles and users can only choose from the available options. The following configuration will add 3 machines with labels Small, Medium and Large to the profile list: diff --git a/dev-requirements.txt b/dev-requirements.txt index 5a9414a..77d4592 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,5 +1,7 @@ git+https://github.com/jupyterhub/the-littlest-jupyterhub@1.0.0 +git+https://github.com/jupyterhub/binderhub.git@main jupyterhub>=4,<5 +alembic>=1.13.0,<1.14 pytest pytest-aiohttp pytest-asyncio diff --git a/pyproject.toml b/pyproject.toml index 8a3f29f..240a096 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,13 +7,21 @@ dependencies = [ "aiodocker~=0.19", "dockerspawner~=12.1", "jupyter_client>=6.1,<8", - "httpx" + "httpx", + "sqlalchemy>=2", + "pydantic>=2,<3", + "alembic>=1.13,<2", + "jupyter-repo2docker>=2024,<2025", + "aiosqlite~=0.19.0" ] dynamic = ["version"] license = {file = "LICENSE"} name = "tljh-repo2docker" readme = "README.md" +[project.scripts] +tljh_repo2docker_upgrade_db = "tljh_repo2docker.dbutil:main" + [project.entry-points.tljh] tljh_repo2docker = "tljh_repo2docker" @@ -56,10 +64,7 @@ source_dir = "src" version_cmd = "hatch version" [tool.jupyter-releaser.hooks] -before-build-npm = [ - "npm install", - "npm run build:prod", -] +before-build-npm = ["npm install", "npm run build:prod"] before-build-python = ["npm run clean"] before-bump-version = ["python -m pip install hatch"] diff --git a/src/environments/App.tsx b/src/environments/App.tsx index 6a3779c..5a301ef 100644 --- a/src/environments/App.tsx +++ b/src/environments/App.tsx @@ -16,6 +16,8 @@ export interface IAppProps { default_cpu_limit: string; default_mem_limit: string; machine_profiles: IMachineProfile[]; + use_binderhub: boolean; + repo_providers?: { label: string; value: string }[]; } export default function App(props: IAppProps) { const jhData = useJupyterhub(); @@ -35,6 +37,8 @@ export default function App(props: IAppProps) { default_cpu_limit={props.default_cpu_limit} default_mem_limit={props.default_mem_limit} machine_profiles={props.machine_profiles} + use_binderhub={props.use_binderhub} + repo_providers={props.repo_providers} /> diff --git a/src/environments/EnvironmentList.tsx b/src/environments/EnvironmentList.tsx index 25aff38..296ad7f 100644 --- a/src/environments/EnvironmentList.tsx +++ b/src/environments/EnvironmentList.tsx @@ -59,7 +59,7 @@ const columns: GridColDef[] = [ ) : params.value === 'building' ? ( ) : null; } @@ -75,7 +75,7 @@ const columns: GridColDef[] = [ return ( ); } diff --git a/src/environments/LogDialog.tsx b/src/environments/LogDialog.tsx index 45e95ca..845acfd 100644 --- a/src/environments/LogDialog.tsx +++ b/src/environments/LogDialog.tsx @@ -65,13 +65,13 @@ function _EnvironmentLogButton(props: IEnvironmentLogButton) { eventSource.onmessage = event => { const data = JSON.parse(event.data); + terminal.write(data.message); + fitAddon.fit(); if (data.phase === 'built') { eventSource.close(); setBuilt(true); return; } - terminal.write(data.message); - fitAddon.fit(); }; } }, [jhData, props.image]); diff --git a/src/environments/NewEnvironmentDialog.tsx b/src/environments/NewEnvironmentDialog.tsx index a36c9e3..335b666 100644 --- a/src/environments/NewEnvironmentDialog.tsx +++ b/src/environments/NewEnvironmentDialog.tsx @@ -13,7 +13,14 @@ import { Select, Typography } from '@mui/material'; -import { Fragment, memo, useCallback, useMemo, useState } from 'react'; +import { + Fragment, + memo, + useCallback, + useEffect, + useMemo, + useState +} from 'react'; import { useAxios } from '../common/AxiosContext'; import { SmallTextField } from '../common/SmallTextField'; @@ -28,9 +35,12 @@ export interface INewEnvironmentDialogProps { default_cpu_limit: string; default_mem_limit: string; machine_profiles: IMachineProfile[]; + use_binderhub: boolean; + repo_providers?: { label: string; value: string }[]; } interface IFormValues { + provider?: string; repo?: string; ref?: string; name?: string; @@ -74,11 +84,53 @@ function _NewEnvironmentDialog(props: INewEnvironmentDialogProps) { [setFormValues] ); const validated = useMemo(() => { - return Boolean(formValues.repo) && Boolean(formValues.ref); - }, [formValues]); + return Boolean(formValues.repo); + }, [formValues.repo]); const [selectedProfile, setSelectedProfile] = useState(0); + const [selectedProvider, setSelectedProvider] = useState(0); + + const onMachineProfileChange = useCallback( + (value?: string | number) => { + if (value !== undefined) { + const index = parseInt(value + ''); + const selected = props.machine_profiles[index]; + if (selected !== undefined) { + updateFormValue('cpu', selected.cpu + ''); + updateFormValue('memory', selected.memory + ''); + setSelectedProfile(index); + } + } + }, + [props.machine_profiles, updateFormValue] + ); + const onRepoProviderChange = useCallback( + (value?: string | number) => { + if (value !== undefined) { + const index = parseInt(value + ''); + const selected = props.repo_providers?.[index]; + if (selected !== undefined) { + updateFormValue('provider', selected.value); + setSelectedProvider(index); + } + } + }, + [props.repo_providers, updateFormValue] + ); + useEffect(() => { + if (props.machine_profiles.length > 0) { + onMachineProfileChange(0); + } + if (props.repo_providers && props.repo_providers.length > 0) { + onRepoProviderChange(0); + } + }, [ + props.machine_profiles, + props.repo_providers, + onMachineProfileChange, + onRepoProviderChange + ]); const MemoryCpuSelector = useMemo(() => { return ( @@ -120,16 +172,7 @@ function _NewEnvironmentDialog(props: INewEnvironmentDialogProps) { value={selectedProfile} label="Machine profile" size="small" - onChange={e => { - const value = e.target.value; - if (value) { - const index = parseInt(value + ''); - const selected = props.machine_profiles[index]; - updateFormValue('cpu', selected.cpu + ''); - updateFormValue('memory', selected.memory + ''); - setSelectedProfile(index); - } - }} + onChange={e => onMachineProfileChange(e.target.value)} > {props.machine_profiles.map((it, idx) => { return ( @@ -141,7 +184,8 @@ function _NewEnvironmentDialog(props: INewEnvironmentDialogProps) { ); - }, [updateFormValue, props.machine_profiles, selectedProfile]); + }, [props.machine_profiles, selectedProfile, onMachineProfileChange]); + return ( @@ -167,6 +211,7 @@ function _NewEnvironmentDialog(props: INewEnvironmentDialogProps) { .replace('https://', '') .replace(/\//g, '-') .replace(/\./g, '-'); + data.ref = data.ref && data.ref.length > 0 ? data.ref : 'HEAD'; data.cpu = data.cpu ?? '2'; data.memory = data.memory ?? '2'; data.username = data.username ?? ''; @@ -186,6 +231,29 @@ function _NewEnvironmentDialog(props: INewEnvironmentDialogProps) { > Create a new environment + {props.use_binderhub && props.repo_providers && ( + + + Repository provider + + + + )} updateFormValue('ref', e.target.value)} value={formValues.ref ?? ''} /> @@ -221,56 +290,64 @@ function _NewEnvironmentDialog(props: INewEnvironmentDialogProps) { {props.machine_profiles.length > 0 ? MachineProfileSelector : MemoryCpuSelector} - - - Advanced - - - updateFormValue('buildargs', e.target.value)} - /> - - - Credentials - - - updateFormValue('username', e.target.value)} - /> - + {!props.use_binderhub && ( + + + + Advanced + + + updateFormValue('buildargs', e.target.value)} + /> + + )} + {!props.use_binderhub && ( + + + + Credentials + + + updateFormValue('username', e.target.value)} + /> + + + )} )} diff --git a/src/servers/ServersList.tsx b/src/servers/ServersList.tsx index 086e605..d65c290 100644 --- a/src/servers/ServersList.tsx +++ b/src/servers/ServersList.tsx @@ -65,7 +65,7 @@ const columns: GridColDef[] = [ ); @@ -86,7 +86,9 @@ function _ServerList(props: IServerListProps) { } const allServers = servers.map((it, id) => { const newItem: any = { ...it, id }; - newItem.image = it.user_options.image ?? ''; + newItem.image = + it.user_options?.display_name ?? it.user_options.image ?? ''; + newItem.uid = it.user_options?.uid ?? null; newItem.last_activity = formatTime(newItem.last_activity); return newItem; }); diff --git a/src/servers/types.ts b/src/servers/types.ts index cf51864..c82723e 100644 --- a/src/servers/types.ts +++ b/src/servers/types.ts @@ -3,6 +3,6 @@ export interface IServerData { name: string; url: string; last_activity: string; - user_options: { image?: string }; + user_options: { image?: string; display_name?: string; uid?: string }; active: boolean; } diff --git a/tljh_repo2docker/__init__.py b/tljh_repo2docker/__init__.py index 8d96e4d..481cc5e 100644 --- a/tljh_repo2docker/__init__.py +++ b/tljh_repo2docker/__init__.py @@ -3,11 +3,14 @@ from jinja2 import BaseLoader, Environment from jupyter_client.localinterfaces import public_ips from jupyterhub.traitlets import ByteSpecification -from tljh.configurer import load_config -from tljh.hooks import hookimpl from traitlets import Unicode from traitlets.config import Configurable +try: + from tljh.hooks import hookimpl +except ModuleNotFoundError: + hookimpl = None + from .docker import list_images # Default CPU period @@ -97,7 +100,10 @@ async def get_options_form(self): """ Override the default form to handle the case when there is only one image. """ - images = await self.list_images() + try: + images = await self.list_images() + except ValueError: + images = [] # make default limits human readable default_mem_limit = self.mem_limit @@ -160,19 +166,24 @@ async def start(self, *args, **kwargs): return await super().start(*args, **kwargs) -@hookimpl -def tljh_custom_jupyterhub_config(c): - # hub - c.JupyterHub.hub_ip = public_ips()[0] - c.JupyterHub.cleanup_servers = False - c.JupyterHub.spawner_class = Repo2DockerSpawner +if hookimpl: + + @hookimpl + def tljh_custom_jupyterhub_config(c): + # hub + c.JupyterHub.hub_ip = public_ips()[0] + c.JupyterHub.cleanup_servers = False + c.JupyterHub.spawner_class = Repo2DockerSpawner - # spawner - c.DockerSpawner.cmd = ["jupyterhub-singleuser"] - c.DockerSpawner.pull_policy = "Never" - c.DockerSpawner.remove = True + # spawner + c.DockerSpawner.cmd = ["jupyterhub-singleuser"] + c.DockerSpawner.pull_policy = "Never" + c.DockerSpawner.remove = True + @hookimpl + def tljh_extra_hub_pip_packages(): + return ["dockerspawner~=0.11", "jupyter_client>=6.1,<8", "aiodocker~=0.19"] -@hookimpl -def tljh_extra_hub_pip_packages(): - return ["dockerspawner~=0.11", "jupyter_client>=6.1,<8", "aiodocker~=0.19"] +else: + tljh_custom_jupyterhub_config = None + tljh_extra_hub_pip_packages = None diff --git a/tljh_repo2docker/alembic/alembic.ini b/tljh_repo2docker/alembic/alembic.ini new file mode 100644 index 0000000..a7354c4 --- /dev/null +++ b/tljh_repo2docker/alembic/alembic.ini @@ -0,0 +1,66 @@ +# A generic, single database configuration. + +[alembic] +script_location = {alembic_dir} +sqlalchemy.url = {db_url} + +# template used to generate migration files +# file_template = %%(rev)s_%%(slug)s + +# max length of characters to apply to the +# "slug" field +#truncate_slug_length = 40 + +# set to 'true' to run the environment during +# the 'revision' command, regardless of autogenerate +# revision_environment = false + +# set to 'true' to allow .pyc and .pyo files without +# a source .py file to be detected as revisions in the +# versions/ directory +# sourceless = false + +# version location specification; this defaults +# to jupyterhub/alembic/versions. When using multiple version +# directories, initial revisions must be specified with --version-path +# version_locations = %(here)s/bar %(here)s/bat jupyterhub/alembic/versions + +# the output encoding used when revision files +# are written from script.py.mako +# output_encoding = utf-8 + + +# Logging configuration +[loggers] +keys = root,sqlalchemy,alembic + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARN +handlers = console +qualname = + +[logger_sqlalchemy] +level = WARN +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S diff --git a/tljh_repo2docker/alembic/env.py b/tljh_repo2docker/alembic/env.py new file mode 100644 index 0000000..6188e7e --- /dev/null +++ b/tljh_repo2docker/alembic/env.py @@ -0,0 +1,65 @@ +import asyncio +import logging +from logging.config import fileConfig + +import alembic +from sqlalchemy import engine_from_config, pool +from sqlalchemy.ext.asyncio import AsyncEngine + +# Alembic Config object, which provides access to values within the .ini file +config = alembic.context.config + +# Interpret the config file for logging +fileConfig(config.config_file_name) +logger = logging.getLogger("alembic.env") + + +def run_migrations_online() -> None: + """ + Run migrations in 'online' mode + """ + connectable = config.attributes.get("connection", None) + + if connectable is None: + connectable = AsyncEngine( + engine_from_config( + config.get_section(config.config_ini_section), + prefix="sqlalchemy.", + poolclass=pool.NullPool, + ) + ) + + if isinstance(connectable, AsyncEngine): + asyncio.run(run_async_migrations(connectable)) + else: + do_run_migrations(connectable) + + +def do_run_migrations(connection): + alembic.context.configure(connection=connection, target_metadata=None) + with alembic.context.begin_transaction(): + alembic.context.run_migrations() + + +async def run_async_migrations(connectable): + async with connectable.connect() as connection: + await connection.run_sync(do_run_migrations) + await connectable.dispose() + + +def run_migrations_offline() -> None: + """ + Run migrations in 'offline' mode. + """ + alembic.context.configure(url=config.get_main_option("sqlalchemy.url")) + + with alembic.context.begin_transaction(): + alembic.context.run_migrations() + + +if alembic.context.is_offline_mode(): + logger.info("Running migrations offline") + run_migrations_offline() +else: + logger.info("Running migrations online") + run_migrations_online() diff --git a/tljh_repo2docker/alembic/script.py.mako b/tljh_repo2docker/alembic/script.py.mako new file mode 100644 index 0000000..43c0940 --- /dev/null +++ b/tljh_repo2docker/alembic/script.py.mako @@ -0,0 +1,24 @@ +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision | comma,n} +Create Date: ${create_date} + +""" + +# revision identifiers, used by Alembic. +revision = ${repr(up_revision)} +down_revision = ${repr(down_revision)} +branch_labels = ${repr(branch_labels)} +depends_on = ${repr(depends_on)} + +from alembic import op +import sqlalchemy as sa +${imports if imports else ""} + +def upgrade(): + ${upgrades if upgrades else "pass"} + + +def downgrade(): + ${downgrades if downgrades else "pass"} diff --git a/tljh_repo2docker/alembic/versions/ac1b4e7e52f3_first_migration.py b/tljh_repo2docker/alembic/versions/ac1b4e7e52f3_first_migration.py new file mode 100644 index 0000000..587ada0 --- /dev/null +++ b/tljh_repo2docker/alembic/versions/ac1b4e7e52f3_first_migration.py @@ -0,0 +1,43 @@ +"""First migration + +Revision ID: ac1b4e7e52f3 +Revises: +Create Date: 2024-04-05 16:25:18.246631 + +""" + +# revision identifiers, used by Alembic. +revision = "ac1b4e7e52f3" +down_revision = None +branch_labels = None +depends_on = None + +import sqlalchemy as sa # noqa +from alembic import op # noqa +from jupyterhub.orm import JSONDict # noqa +from sqlalchemy.dialects import postgresql # noqa + + +def upgrade(): + op.create_table( + "images", + sa.Column("uid", sa.Unicode(36)), + sa.Column("name", sa.Unicode(4096), nullable=False), + sa.Column( + "status", + postgresql.ENUM( + "built", + "building", + "failed", + name="build_status_enum", + create_type=True, + ), + nullable=False, + ), + sa.Column("log", sa.UnicodeText()), + sa.Column("image_meta", JSONDict, nullable=False), + ) + + +def downgrade(): + op.drop_table("images") diff --git a/tljh_repo2docker/app.py b/tljh_repo2docker/app.py index f48fd76..d03b07e 100644 --- a/tljh_repo2docker/app.py +++ b/tljh_repo2docker/app.py @@ -3,6 +3,7 @@ import socket import typing as tp from pathlib import Path +from urllib.parse import urlparse from jinja2 import Environment, PackageLoader from jupyterhub.app import DATA_FILES_PATH @@ -12,7 +13,11 @@ from traitlets import Dict, Int, List, Unicode, default, validate from traitlets.config.application import Application +from .binderhub_builder import BinderHubBuildHandler +from .binderhub_log import BinderHubLogsHandler from .builder import BuildHandler +from .database.manager import ImagesDatabaseManager +from .dbutil import async_session_context_factory, sync_to_async_url, upgrade_if_needed from .environments import EnvironmentsHandler from .logs import LogsHandler from .servers import ServersHandler @@ -118,16 +123,54 @@ def _default_log_level(self): allow_none=True, ) + db_url = Unicode( + "sqlite:///tljh_repo2docker.sqlite", + help="url for the database.", + config=True, + ) + + config_file = Unicode( + "tljh_repo2docker_config.py", + help=""" + Config file to load. + + If a relative path is provided, it is taken relative to current directory + """, + config=True, + ) + + binderhub_url = Unicode( + None, help="URL of the binderhub service.", allow_none=True, config=True + ) + + repo_providers = List( + default_value=[ + {"label": "Git", "value": "git"}, + ], + trait=Dict, + help=""" + Dict of available repo providers in the form of {"label":"value"}. + The references are taken from the binderhub documentation + """, + config=True, + ) + aliases = { "port": "TljhRepo2Docker.port", "ip": "TljhRepo2Docker.ip", + "config": "TljhRepo2Docker.config_file", "default_memory_limit": "TljhRepo2Docker.default_memory_limit", "default_cpu_limit": "TljhRepo2Docker.default_cpu_limit", "machine_profiles": "TljhRepo2Docker.machine_profiles", + "binderhub_url": "TljhRepo2Docker.binderhub_url", + "db_url": "TljhRepo2Docker.db_url", } def init_settings(self) -> tp.Dict: """Initialize settings for the service application.""" + + self.load_config_file(self.config_file) + static_path = DATA_FILES_PATH + "/static/" static_url_prefix = self.service_prefix + "static/" env_opt = {"autoescape": True} @@ -150,7 +193,13 @@ def init_settings(self) -> tp.Dict: default_mem_limit=self.default_memory_limit, default_cpu_limit=self.default_cpu_limit, machine_profiles=self.machine_profiles, + binderhub_url=self.binderhub_url, + repo_providers=self.repo_providers, ) + if hasattr(self, "db_context"): + settings["db_context"] = self.db_context + if hasattr(self, "image_db_manager"): + settings["image_db_manager"] = self.image_db_manager return settings def init_handlers(self) -> tp.List: @@ -184,18 +233,61 @@ def init_handlers(self) -> tp.List: url_path_join(self.service_prefix, r"environments"), EnvironmentsHandler, ), - (url_path_join(self.service_prefix, r"api/environments"), BuildHandler), - ( - url_path_join( - self.service_prefix, r"api/environments/([^/]+)/logs" - ), - LogsHandler, - ), ] ) - + if self.binderhub_url: + handlers.extend( + [ + ( + url_path_join( + self.service_prefix, r"api/environments/([^/]+)/logs" + ), + BinderHubLogsHandler, + ), + ( + url_path_join(self.service_prefix, r"api/environments"), + BinderHubBuildHandler, + ), + ] + ) + else: + handlers.extend( + [ + ( + url_path_join( + self.service_prefix, r"api/environments/([^/]+)/logs" + ), + LogsHandler, + ), + ( + url_path_join(self.service_prefix, r"api/environments"), + BuildHandler, + ), + ] + ) return handlers + def init_db(self): + async_db_url = sync_to_async_url(self.db_url) + urlinfo = urlparse(async_db_url) + if urlinfo.password: + # avoid logging the database password + urlinfo = urlinfo._replace( + netloc=f"{urlinfo.username}:[redacted]@{urlinfo.hostname}:{urlinfo.port}" + ) + db_log_url = urlinfo.geturl() + else: + db_log_url = async_db_url + self.log.info("Connecting to db: %s", db_log_url) + upgrade_if_needed(async_db_url, log=self.log) + try: + self.db_context = async_session_context_factory(async_db_url) + except Exception: + self.log.error("Failed to connect to db: %s", db_log_url) + self.log.debug("Database error was:", exc_info=True) + + self.image_db_manager = ImagesDatabaseManager() + def make_app(self) -> web.Application: """Create the tornado web application. Returns: @@ -208,6 +300,7 @@ def make_app(self) -> web.Application: def start(self): """Start the server.""" + self.init_db() settings = self.init_settings() self.app = web.Application(**settings) diff --git a/tljh_repo2docker/base.py b/tljh_repo2docker/base.py index 060b721..c800531 100644 --- a/tljh_repo2docker/base.py +++ b/tljh_repo2docker/base.py @@ -1,18 +1,30 @@ import functools import json import os +import sys +from contextlib import _AsyncGeneratorContextManager from http.client import responses +from typing import Any, Callable, Dict, List, Optional, Tuple from httpx import AsyncClient from jinja2 import Template from jupyterhub.services.auth import HubOAuthenticated from jupyterhub.utils import url_path_join +from sqlalchemy.ext.asyncio import AsyncSession from tornado import web from tljh_repo2docker import TLJH_R2D_ADMIN_SCOPE +from tljh_repo2docker.database.manager import ImagesDatabaseManager from .model import UserModel +if sys.version_info >= (3, 9): + AsyncSessionContextFactory = Callable[ + [], _AsyncGeneratorContextManager[AsyncSession] + ] +else: + AsyncSessionContextFactory = Any + def require_admin_role(func): """decorator to require admin role to perform an action""" @@ -37,6 +49,9 @@ class BaseHandler(HubOAuthenticated, web.RequestHandler): @property def client(self): + """ + Get the asynchronous HTTP client with valid authorization token. + """ if not BaseHandler._client: api_url = os.environ.get("JUPYTERHUB_API_URL", "") api_token = os.environ.get("JUPYTERHUB_API_TOKEN", None) @@ -157,3 +172,85 @@ def write_error(self, status_code, **kwargs): self.write( json.dumps({"status": status_code, "message": message or status_message}) ) + + @property + def use_binderhub(self) -> bool: + """ + Check if BinderHub is being used by checking for the binderhub url + in the setting. + + Returns: + bool: True if BinderHub is being used, False otherwise. + """ + return self.settings.get("binderhub_url", None) is not None + + def get_db_handlers( + self, + ) -> Tuple[ + Optional[AsyncSessionContextFactory], + Optional[ImagesDatabaseManager], + ]: + """ + Get database handlers. + + Returns the database context and image database manager based on the + configuration and settings. If `use_binderhub` flag is set to True, + returns the configured database context and image database manager; + otherwise, returns None for both. + + Returns: + Tuple[Optional[Callable[[], _AsyncGeneratorContextManager[AsyncSession]]], + Optional[ImagesDatabaseManager]]: A tuple containing: + - The database context, which is a callable returning an + async generator context manager for session management. + - The image database manager, which handles image database + operations. + + """ + if self.use_binderhub: + db_context = self.settings.get("db_context") + image_db_manager = self.settings.get("image_db_manager") + return db_context, image_db_manager + else: + return None, None + + async def get_images_from_db(self) -> List[Dict]: + """ + Retrieve images from the database. + + This method fetches image information from the database, formats it, + and returns a list of dictionaries representing each image. + + Returns: + List[Dict]: A list of dictionaries, each containing information + about an image. Each dictionary has the following keys: + - image_name (str): The name of the docker image. + - uid (str): The unique identifier of the image. + - status (str): The build status of the image. + - display_name (str): The user defined name of the image. + - repo (str): Source repo used to build the image. + - ref (str): Commit reference. + - cpu_limit (str): CPU limit. + - mem_limit (str): Memory limit. + + Note: + If `use_binderhub` flag is set to True and valid database context + and image database manager are available, it retrieves image + information; otherwise, an empty list is returned. + """ + db_context, image_db_manager = self.get_db_handlers() + all_images = [] + if self.use_binderhub and db_context and image_db_manager: + async with db_context() as db: + docker_images = await image_db_manager.read_all(db) + all_images = [ + dict( + image_name=image.name, + uid=str(image.uid), + status=image.status, + **image.image_meta.model_dump(), + ) + for image in docker_images + ] + + return all_images diff --git a/tljh_repo2docker/binderhub_builder.py b/tljh_repo2docker/binderhub_builder.py new file mode 100644 index 0000000..24f7cfa --- /dev/null +++ b/tljh_repo2docker/binderhub_builder.py @@ -0,0 +1,148 @@ +import json +import re +from urllib.parse import quote +from uuid import UUID, uuid4 + +from aiodocker import Docker +from jupyterhub.utils import url_path_join +from tornado import web + +from .base import BaseHandler, require_admin_role +from .database.schemas import ( + BuildStatusType, + DockerImageCreateSchema, + DockerImageUpdateSchema, + ImageMetadataType, +) + +IMAGE_NAME_RE = r"^[a-z0-9-_]+$" + + +class BinderHubBuildHandler(BaseHandler): + """ + Handle requests to build user environments using BinderHub service + """ + + @web.authenticated + @require_admin_role + async def delete(self): + """ + Method to handle the deletion of a specific image. + + Note: + - Only users with admin role or with `TLJH_R2D_ADMIN_SCOPE` scope can access it. + """ + + data = self.get_json_body() + uid = UUID(data["name"]) + + db_context, image_db_manager = self.get_db_handlers() + if not db_context or not image_db_manager: + return + + deleted = False + async with db_context() as db: + image = await image_db_manager.read(db, uid) + if image: + try: + async with Docker() as docker: + await docker.images.delete(image.name) + except Exception: + pass + deleted = await image_db_manager.delete(db, uid) + + self.set_header("content-type", "application/json") + if deleted: + self.set_status(200) + self.finish(json.dumps({"status": "ok"})) + else: + self.set_status(404) + self.finish(json.dumps({"status": "error"})) + + @web.authenticated + @require_admin_role + async def post(self): + """ + Method to handle the creation of a new environment based on provided specifications. + As the build progresses, it updates the build log in the database and checks for completion. + + Note: + - Only users with admin role or with `TLJH_R2D_ADMIN_SCOPE` scope can access it. + """ + data = self.get_json_body() + + repo = data["repo"] + ref = data["ref"] + name = data["name"].lower() + memory = data["memory"] + cpu = data["cpu"] + provider = data["provider"] + if len(repo) == 0: + raise web.HTTPError(400, "Repository is empty") + + if name and not re.match(IMAGE_NAME_RE, name): + raise web.HTTPError( + 400, + f"The name of the environment is restricted to the following characters: {IMAGE_NAME_RE}", + ) + + if memory: + try: + float(memory) + except ValueError: + raise web.HTTPError(400, "Memory Limit must be a number") + + if cpu: + try: + float(cpu) + except ValueError: + raise web.HTTPError(400, "CPU Limit must be a number") + + binder_url = self.settings.get("binderhub_url") + quoted_repo = quote(repo, safe="") + url = url_path_join(binder_url, "build", provider, quoted_repo, ref) + + params = {"build_only": "true"} + + db_context, image_db_manager = self.get_db_handlers() + if not db_context or not image_db_manager: + return + + uid = uuid4() + image_in = DockerImageCreateSchema( + uid=uid, + name=name, + status=BuildStatusType.BUILDING, + log="", + image_meta=ImageMetadataType( + display_name=name, repo=repo, ref=ref, cpu_limit=cpu, mem_limit=memory + ), + ) + self.set_status(200) + self.set_header("content-type", "application/json") + self.finish(json.dumps({"uid": str(uid), "status": "ok"})) + + log = "" + async with db_context() as db: + await image_db_manager.create(db, image_in) + async with self.client.stream("GET", url, params=params, timeout=None) as r: + async for line in r.aiter_lines(): + if line.startswith("data:"): + json_log = json.loads(line.split(":", 1)[1]) + phase = json_log.get("phase", None) + message = json_log.get("message", "") + if phase != "unknown": + log += message + update_data = DockerImageUpdateSchema(uid=uid, log=log) + stop = False + if phase == "ready" or phase == "built": + image_name = json_log.get("imageName", name) + update_data.status = BuildStatusType.BUILT + update_data.name = image_name + stop = True + elif phase == "failed": + update_data.status = BuildStatusType.FAILED + stop = True + await image_db_manager.update(db, update_data) + if stop: + return diff --git a/tljh_repo2docker/binderhub_log.py b/tljh_repo2docker/binderhub_log.py new file mode 100644 index 0000000..faf90ce --- /dev/null +++ b/tljh_repo2docker/binderhub_log.py @@ -0,0 +1,90 @@ +import asyncio +import json +from uuid import UUID + +from tornado import web +from tornado.iostream import StreamClosedError + +from .base import BaseHandler, require_admin_role +from .database.schemas import BuildStatusType + + +class BinderHubLogsHandler(BaseHandler): + """ + Expose a handler to follow the build logs. + """ + + @web.authenticated + @require_admin_role + async def get(self, image_uid: str): + """ + Method to retrieve real-time status updates for a specific image build process. + + This method sets the appropriate headers for Server-Sent Events (SSE) to enable streaming of data over HTTP. + It retrieves the database handlers and the image with the specified UID from the database. + Then, it continuously emits status updates over the stream until the build process is completed or times out. + + Parameters: + - image_uid (str): The UID of the image for which real-time status updates are requested. + + Raises: + - web.HTTPError: If the provided image UID is badly formed or if the requested image is not found. + """ + + self.set_header("Content-Type", "text/event-stream") + self.set_header("Cache-Control", "no-cache") + + db_context, image_db_manager = self.get_db_handlers() + if not db_context or not image_db_manager: + return + + async with db_context() as db: + try: + uuid = UUID(image_uid) + except ValueError: + raise web.HTTPError(400, "Badly formed hexadecimal UUID string") + + image = await image_db_manager.read(db, uuid) + if not image: + raise web.HTTPError(404, "Image not found") + + status = image.status + if status == BuildStatusType.FAILED: + await self._emit({"phase": "error", "message": image.log}) + return + if status == BuildStatusType.BUILT: + await self._emit({"phase": "built", "message": image.log}) + return + + current_log_length = len(image.log) + await self._emit({"phase": "log", "message": image.log}) + time = 0 + TIME_OUT = 3600 + while time < TIME_OUT: + time += 1 + await asyncio.sleep(1) + image = await image_db_manager.read(db, UUID(image_uid)) + if len(image.log) > current_log_length: + await self._emit( + {"phase": "log", "message": image.log[current_log_length:]} + ) + current_log_length = len(image.log) + status = image.status + if status == BuildStatusType.FAILED: + await self._emit({"phase": "error", "message": ""}) + break + if status == BuildStatusType.BUILT: + await self._emit({"phase": "built", "message": ""}) + break + + async def _emit(self, msg): + """ + Asynchronous method to emit a message over a stream. + """ + + try: + self.write(f"data: {json.dumps(msg)}\n\n") + await self.flush() + except StreamClosedError: + self.log.warning("Stream closed while handling %s", self.request.uri) + raise web.Finish() diff --git a/tljh_repo2docker/database/manager.py b/tljh_repo2docker/database/manager.py new file mode 100644 index 0000000..02918d6 --- /dev/null +++ b/tljh_repo2docker/database/manager.py @@ -0,0 +1,192 @@ +import logging +from typing import List, Optional, Type, Union + +import sqlalchemy as sa +from pydantic import UUID4 +from sqlalchemy.exc import IntegrityError, SQLAlchemyError +from sqlalchemy.ext.asyncio import AsyncSession +from tornado.web import HTTPError + +from .model import DockerImageSQL +from .schemas import ( + DockerImageCreateSchema, + DockerImageOutSchema, + DockerImageUpdateSchema, +) + + +class ImagesDatabaseManager: + + @property + def _table(self) -> Type[DockerImageSQL]: + return DockerImageSQL + + @property + def _schema_out(self) -> Type[DockerImageOutSchema]: + return DockerImageOutSchema + + async def create( + self, db: AsyncSession, obj_in: DockerImageCreateSchema + ) -> DockerImageOutSchema: + """ + Create one resource. + + Args: + db: An asyncio version of SQLAlchemy session. + obj_in: An object containing the resource instance to create + + Returns: + The created resource instance on success. + + Raises: + DatabaseError: If `db.commit()` failed. + """ + entry = self._table(**obj_in.model_dump()) + + db.add(entry) + + try: + await db.commit() + # db.refresh(entry) + except IntegrityError as e: + logging.error(f"create: {e}") + raise HTTPError(409, "That resource already exists.") + except SQLAlchemyError as e: + logging.error(f"create: {e}") + raise e + + return self._schema_out.model_validate(entry) + + async def read( + self, db: AsyncSession, uid: UUID4 + ) -> Union[DockerImageOutSchema, None]: + """ + Get one resource by uid. + + Args: + db: An asyncio version of SQLAlchemy session. + uid: The primary key of the resource to retrieve. + + Returns: + The first resource instance found, `None` if no instance retrieved. + """ + if entry := await db.get(self._table, uid): + return self._schema_out.model_validate(entry) + return None + + async def read_many( + self, db: AsyncSession, uids: List[UUID4] + ) -> List[DockerImageOutSchema]: + """ + Get multiple resources. + + Args: + db: An asyncio version of SQLAlchemy session. + uids: The primary keys of the resources to retrieve. + + Returns: + The list of resources retrieved. + """ + resources = ( + await db.execute(sa.select(self._table).where(self._table.uid.in_(uids))) + ).scalars() + return [self._schema_out.model_validate(r) for r in resources] + + async def read_all(self, db: AsyncSession) -> List[DockerImageOutSchema]: + """ + Get all rows. + + Args: + db: An asyncio version of SQLAlchemy session. + + Returns: + The list of resources retrieved. + """ + resources = (await db.execute(sa.select(self._table))).scalars().all() + return [self._schema_out.model_validate(r) for r in resources] + + async def read_by_image_name( + self, db: AsyncSession, image: str + ) -> Optional[DockerImageOutSchema]: + """ + Get image by its name. + + Args: + db: An asyncio version of SQLAlchemy session. + + Returns: + The list of resources retrieved. + """ + statement = sa.select(self._table).where(self._table.name == image) + try: + result = await db.execute(statement) + return self._schema_out.model_validate(result.scalars().first()) + except Exception: + return None + + async def update( + self, db: AsyncSession, obj_in: DockerImageUpdateSchema, optimistic: bool = True + ) -> Union[DockerImageOutSchema, None]: + """ + Update one object. + + Args: + db: An asyncio version of SQLAlchemy session. + obj_in: A model containing values to update + optimistic: If `True`, assert the new model instance to be + `**{**obj_db.dict(), **obj_in.dict()}` + + Returns: + The updated model instance on success, `None` if it does not exist + yet in database. + + Raises: + DatabaseError: If `db.commit()` failed. + """ + if not (obj_db := await self.read(db=db, uid=obj_in.uid)): + await self.create(db, obj_in) + + update_data = obj_in.model_dump(exclude_none=True) + + await db.execute( + sa.update(self._table) + .where(self._table.uid == obj_in.uid) + .values(**update_data) + ) + + try: + await db.commit() + except SQLAlchemyError as e: + logging.error(f"update: {e}") + raise e + + if optimistic: + for field in update_data: + setattr(obj_db, field, update_data[field]) + return self._schema_out.model_validate(obj_db) + + return await self.read(db=db, uid=obj_in.uid) + + async def delete(self, db: AsyncSession, uid: UUID4) -> bool: + """ + Delete one object. + + Args: + db: An asyncio version of SQLAlchemy session. + uid: The primary key of the resource to delete. + + Returns: + bool: `True` if the object has been deleted, `False` otherwise. + + Raises: + DatabaseError: If `db.commit()` failed. + """ + results = await db.execute(sa.delete(self._table).where(self._table.uid == uid)) + + try: + await db.commit() + except SQLAlchemyError as e: + logging.error(f"delete: {e}") + raise e + + return results.rowcount == 1 diff --git a/tljh_repo2docker/database/model.py b/tljh_repo2docker/database/model.py new file mode 100644 index 0000000..166889c --- /dev/null +++ b/tljh_repo2docker/database/model.py @@ -0,0 +1,42 @@ +import uuid + +from jupyterhub.orm import JSONDict +from sqlalchemy import Column, String, Text +from sqlalchemy.dialects.postgresql import ENUM, UUID +from sqlalchemy.orm import DeclarativeMeta, declarative_base + +from .schemas import BuildStatusType + +BaseSQL: DeclarativeMeta = declarative_base() + + +class DockerImageSQL(BaseSQL): + """ + SQLAlchemy image table definition. + """ + + __tablename__ = "images" + + uid = Column( + UUID(as_uuid=True), + primary_key=True, + default=uuid.uuid4, + ) + + name = Column(String(length=4096), unique=False, nullable=False) + + status = Column( + ENUM( + BuildStatusType, + name="build_status_enum", + create_type=False, + values_callable=lambda enum: [e.value for e in enum], + ), + nullable=False, + ) + + log = Column(Text) + + image_meta = Column(JSONDict, default={}) + + __mapper_args__ = {"eager_defaults": True} diff --git a/tljh_repo2docker/database/schemas.py b/tljh_repo2docker/database/schemas.py new file mode 100644 index 0000000..4571c7a --- /dev/null +++ b/tljh_repo2docker/database/schemas.py @@ -0,0 +1,43 @@ +from enum import Enum +from typing import Optional + +from pydantic import UUID4, BaseModel, ConfigDict + + +class BuildStatusType(str, Enum): + BUILT = "built" + BUILDING = "building" + FAILED = "failed" + + +class ImageMetadataType(BaseModel): + display_name: str + repo: str + ref: str + cpu_limit: str + mem_limit: str + + +class DockerImageCreateSchema(BaseModel): + uid: UUID4 + name: str + status: BuildStatusType + log: str + image_meta: ImageMetadataType + + model_config = ConfigDict(use_enum_values=True) + + +class DockerImageUpdateSchema(DockerImageCreateSchema): + uid: UUID4 + name: Optional[str] = None + status: Optional[BuildStatusType] = None + log: Optional[str] = None + image_meta: Optional[ImageMetadataType] = None + + model_config = ConfigDict(use_enum_values=True) + + +class DockerImageOutSchema(DockerImageCreateSchema): + + model_config = ConfigDict(use_enum_values=True, from_attributes=True) diff --git a/tljh_repo2docker/dbutil.py b/tljh_repo2docker/dbutil.py new file mode 100644 index 0000000..b87e20b --- /dev/null +++ b/tljh_repo2docker/dbutil.py @@ -0,0 +1,222 @@ +import os +import shutil +import sys +from contextlib import asynccontextmanager, contextmanager +from datetime import datetime +from pathlib import Path +from subprocess import check_call +from tempfile import TemporaryDirectory +from typing import AsyncGenerator, List +from urllib.parse import urlparse + +import alembic +import alembic.config +from alembic.script import ScriptDirectory +from sqlalchemy import create_engine, inspect, text +from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine + +HERE = Path(__file__).parent.resolve() +ALEMBIC_DIR = HERE / "alembic" +ALEMBIC_INI_TEMPLATE_PATH = ALEMBIC_DIR / "alembic.ini" + + +def write_alembic_ini(alembic_ini: Path, db_url="sqlite:///tljh_repo2docker.sqlite"): + """Write a complete alembic.ini from our template. + + Parameters + ---------- + alembic_ini : str + path to the alembic.ini file that should be written. + db_url : str + The SQLAlchemy database url, e.g. `sqlite:///tljh_repo2docker.sqlite`. + """ + with open(ALEMBIC_INI_TEMPLATE_PATH) as f: + alembic_ini_tpl = f.read() + + with open(alembic_ini, "w") as f: + f.write( + alembic_ini_tpl.format( + alembic_dir=ALEMBIC_DIR, + db_url=str(db_url).replace("%", "%%"), + ) + ) + + +@contextmanager +def _temp_alembic_ini(db_url): + """Context manager for temporary JupyterHub alembic directory + + Temporarily write an alembic.ini file for use with alembic migration scripts. + + Context manager yields alembic.ini path. + + Parameters + ---------- + db_url : str + The SQLAlchemy database url. + + Returns + ------- + alembic_ini: str + The path to the temporary alembic.ini that we have created. + This file will be cleaned up on exit from the context manager. + """ + with TemporaryDirectory() as td: + alembic_ini = Path(td) / "alembic.ini" + write_alembic_ini(alembic_ini, db_url) + yield alembic_ini + + +def upgrade(db_url, revision="head"): + """Upgrade the given database to revision. + + db_url: str + The SQLAlchemy database url. + + revision: str [default: head] + The alembic revision to upgrade to. + """ + with _temp_alembic_ini(db_url) as alembic_ini: + check_call(["alembic", "-c", alembic_ini, "upgrade", revision]) + + +def backup_db_file(db_file, log=None): + """Backup a database file if it exists""" + timestamp = datetime.now().strftime(".%Y-%m-%d-%H%M%S") + backup_db_file = db_file + timestamp + for i in range(1, 10): + if not os.path.exists(backup_db_file): + break + backup_db_file = f"{db_file}.{timestamp}.{i}" + # + if os.path.exists(backup_db_file): + raise OSError("backup db file already exists: %s" % backup_db_file) + if log: + log.info("Backing up %s => %s", db_file, backup_db_file) + shutil.copy(db_file, backup_db_file) + + +def _alembic(db_url: str, alembic_arg: List[str]): + """Run an alembic command with a temporary alembic.ini""" + + with _temp_alembic_ini(db_url) as alembic_ini: + check_call(["alembic", "-c", alembic_ini] + alembic_arg) + + +def check_db_revision(engine): + """Check the database revision""" + # Check database schema version + current_table_names = set(inspect(engine).get_table_names()) + + # alembic needs the password if it's in the URL + engine_url = engine.url.render_as_string(hide_password=False) + + if "alembic_version" not in current_table_names: + return True + + with _temp_alembic_ini(engine_url) as ini: + cfg = alembic.config.Config(ini) + scripts = ScriptDirectory.from_config(cfg) + head = scripts.get_heads()[0] + + # check database schema version + # it should always be defined at this point + with engine.begin() as connection: + alembic_revision = connection.execute( + text("SELECT version_num FROM alembic_version") + ).first()[0] + if alembic_revision == head: + return False + else: + raise Exception( + f"Found database schema version {alembic_revision} != {head}. " + "Backup your database and run `tljh_repo2docker_upgrade_db`" + " to upgrade to the latest schema." + ) + + +def upgrade_if_needed(db_url, log=None): + """Upgrade a database if needed + + If the database is sqlite, a backup file will be created with a timestamp. + Other database systems should perform their own backups prior to calling this. + """ + # run check-db-revision first + engine = create_engine(async_to_sync_url(db_url)) + need_upgrade = check_db_revision(engine=engine) + if not need_upgrade: + if log: + log.info("Database schema is up-to-date") + return + + urlinfo = urlparse(db_url) + if urlinfo.password: + # avoid logging the database password + urlinfo = urlinfo._replace( + netloc=f"{urlinfo.username}:[redacted]@{urlinfo.hostname}:{urlinfo.port}" + ) + db_log_url = urlinfo.geturl() + else: + db_log_url = db_url + if log: + log.info("Upgrading %s", db_log_url) + + upgrade(db_url) + + +def sync_to_async_url(db_url: str) -> str: + """Convert a sync database URL to async one""" + if db_url.startswith("sqlite:"): + return db_url.replace("sqlite:", "sqlite+aiosqlite:") + if db_url.startswith("postgresql:"): + return db_url.replace("postgresql:", "postgresql+asyncpg:") + if db_url.startswith("mysql:"): + return db_url.replace("mysql:", "mysql+aiomysql:") + return db_url + + +def async_to_sync_url(db_url: str) -> str: + """Convert a async database URL to sync one""" + if db_url.startswith("sqlite+aiosqlite:"): + return db_url.replace("sqlite+aiosqlite:", "sqlite:") + if db_url.startswith("postgresql+asyncpg:"): + return db_url.replace("postgresql+asyncpg:", "postgresql:") + if db_url.startswith("mysql+aiomysql:"): + return db_url.replace("mysql+aiomysql:", "mysql:") + return db_url + + +def async_session_context_factory(async_db_url: str): + """ + Factory function to create an asynchronous session context manager. + + Parameters: + - async_db_url (str): The URL for the asynchronous database connection. + + Returns: + - AsyncContextManager[AsyncSession]: An asynchronous context manager that yields + an async session for database interactions within the context. + """ + async_engine = create_async_engine(async_db_url) + async_session_maker = async_sessionmaker( + async_engine, + class_=AsyncSession, + expire_on_commit=False, + autocommit=False, + autoflush=False, + ) + + async def get_async_session() -> AsyncGenerator[AsyncSession, None]: + async with async_session_maker() as session: + yield session + + async_session_context = asynccontextmanager(get_async_session) + return async_session_context + + +def main(): + if len(sys.argv) > 1: + db_url = sys.argv[1] + upgrade(db_url) + else: + print("Missing database URL") diff --git a/tljh_repo2docker/environments.py b/tljh_repo2docker/environments.py index 27044e2..f62b4a2 100644 --- a/tljh_repo2docker/environments.py +++ b/tljh_repo2docker/environments.py @@ -14,14 +14,23 @@ class EnvironmentsHandler(BaseHandler): @web.authenticated @require_admin_role async def get(self): - images = await list_images() - containers = await list_containers() + all_images = [] + + if self.use_binderhub: + all_images = await self.get_images_from_db() + else: + images = await list_images() + containers = await list_containers() + all_images = images + containers + result = self.render_template( "images.html", - images=images + containers, + images=all_images, default_mem_limit=self.settings.get("default_mem_limit"), default_cpu_limit=self.settings.get("default_cpu_limit"), machine_profiles=self.settings.get("machine_profiles", []), + repo_providers=self.settings.get("repo_providers", None), + use_binderhub=self.use_binderhub, ) if isawaitable(result): self.write(await result) diff --git a/tljh_repo2docker/servers.py b/tljh_repo2docker/servers.py index ea4abb2..aa9f1e1 100644 --- a/tljh_repo2docker/servers.py +++ b/tljh_repo2docker/servers.py @@ -1,4 +1,5 @@ from inspect import isawaitable +from typing import Dict, List from tornado import web @@ -13,10 +14,33 @@ class ServersHandler(BaseHandler): @web.authenticated async def get(self): - images = await list_images() + images = [] + if self.use_binderhub: + images = await self.get_images_from_db() + else: + try: + images = await list_images() + except ValueError: + pass + user_data = await self.fetch_user() - server_data = user_data.all_spawners() + server_data: List[Dict] = user_data.all_spawners() + + db_context, image_db_manager = self.get_db_handlers() + if db_context and image_db_manager: + async with db_context() as db: + for data in server_data: + image_name = data.get("user_options", {}).get("image", None) + if image_name: + db_data = await image_db_manager.read_by_image_name( + db, image_name + ) + if db_data: + data["user_options"]["uid"] = str(db_data.uid) + data["user_options"][ + "display_name" + ] = db_data.image_meta.display_name named_server_limit = 0 result = self.render_template( "servers.html", diff --git a/tljh_repo2docker/servers_api.py b/tljh_repo2docker/servers_api.py index 7d30693..db12bf7 100644 --- a/tljh_repo2docker/servers_api.py +++ b/tljh_repo2docker/servers_api.py @@ -1,3 +1,5 @@ +from uuid import UUID + from jupyterhub.utils import url_path_join from tornado import web @@ -12,14 +14,28 @@ class ServersAPIHandler(BaseHandler): @web.authenticated async def post(self): data = self.get_json_body() - image_name = data.get("imageName", None) + + image_name_or_uid = data.get("imageName", None) user_name = data.get("userName", None) server_name = data.get("serverName", "") if user_name != self.current_user["name"]: raise web.HTTPError(403, "Unauthorized") - if not image_name: + if not image_name_or_uid: raise web.HTTPError(400, "Missing image name") + if self.use_binderhub: + db_context, image_db_manager = self.get_db_handlers() + if not db_context or not image_db_manager: + raise web.HTTPError(500, "Server error, missing database") + + async with db_context() as db: + image = await image_db_manager.read(db, UUID(image_name_or_uid)) + if not image: + raise web.HTTPError(404, "Image not found") + image_name = image.name + else: + image_name = image_name_or_uid + post_data = {"image": image_name} path = "" diff --git a/tljh_repo2docker/templates/images.html b/tljh_repo2docker/templates/images.html index 0c54591..87cc8ec 100644 --- a/tljh_repo2docker/templates/images.html +++ b/tljh_repo2docker/templates/images.html @@ -1,7 +1,7 @@ {% extends "page.html" %} {% block main %}
diff --git a/tljh_repo2docker/tests/binderhub_build/__init__.py b/tljh_repo2docker/tests/binderhub_build/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tljh_repo2docker/tests/binderhub_build/conftest.py b/tljh_repo2docker/tests/binderhub_build/conftest.py new file mode 100644 index 0000000..7eee329 --- /dev/null +++ b/tljh_repo2docker/tests/binderhub_build/conftest.py @@ -0,0 +1,102 @@ +import sys +from pathlib import Path + +import pytest +import sqlalchemy as sa +from sqlalchemy.orm import sessionmaker +from traitlets.config import Config + +from tljh_repo2docker import tljh_custom_jupyterhub_config +from tljh_repo2docker.database.model import DockerImageSQL + +ROOT = Path(__file__).parents[3] + +binderhub_service_name = "binder" +binderhub_config = ROOT / "ui-tests" / "binderhub_config.py" +tljh_repo2docker_config = ROOT / "ui-tests" / "tljh_repo2docker_binderhub.py" + +db_url = "sqlite:///tljh_repo2docker.sqlite" + + +@pytest.fixture(scope="module") +def generated_image_name(): + return "https-3a-2f-2fgithub-2ecom-2fplasmabio-2ftljh-2drepo2docker-2dtest-2dbinder-3f035a:06bb545ab3a2888477cbddfed0ea77eae313cfed" + + +@pytest.fixture(scope="module") +def image_name(): + return "tljh-repo2docker-test:HEAD" + + +@pytest.fixture +async def app(hub_app): + config = Config() + tljh_custom_jupyterhub_config(config) + + config.JupyterHub.services.extend( + [ + { + "name": binderhub_service_name, + "admin": True, + "command": [ + sys.executable, + "-m", + "binderhub", + f"--config={binderhub_config}", + ], + "url": "http://localhost:8585", + "oauth_client_id": "service-binderhub", + "oauth_no_confirm": True, + }, + { + "name": "tljh_repo2docker", + "url": "http://127.0.0.1:6789", + "command": [ + sys.executable, + "-m", + "tljh_repo2docker", + "--ip", + "127.0.0.1", + "--port", + "6789", + "--binderhub_url", + "http://localhost:8585/@/space%20word/services/binder/", + ], + "oauth_no_confirm": True, + }, + ] + ) + + config.JupyterHub.load_roles = [ + { + "description": "Role for tljh_repo2docker service", + "name": "tljh-repo2docker-service", + "scopes": [ + "read:users", + "read:roles:users", + "admin:servers", + "access:services!service=binder", + ], + "services": ["tljh_repo2docker"], + } + ] + + app = await hub_app(config=config) + return app + + +@pytest.fixture(scope="session") +def db_session(): + engine = sa.create_engine(db_url) + Session = sessionmaker( + bind=engine, + expire_on_commit=False, + autocommit=False, + autoflush=False, + ) + + session = Session() + yield session + session.query(DockerImageSQL).delete() + session.commit() + session.close() diff --git a/tljh_repo2docker/tests/binderhub_build/test_builder.py b/tljh_repo2docker/tests/binderhub_build/test_builder.py new file mode 100644 index 0000000..42ff6b2 --- /dev/null +++ b/tljh_repo2docker/tests/binderhub_build/test_builder.py @@ -0,0 +1,104 @@ +import asyncio + +import pytest +import sqlalchemy as sa +from aiodocker import Docker, DockerError + +from tljh_repo2docker.database.model import DockerImageSQL + +from ..utils import add_environment, remove_environment, wait_for_image + + +@pytest.mark.asyncio +async def test_add_environment( + app, minimal_repo, image_name, generated_image_name, db_session +): + name, ref = image_name.split(":") + r = await add_environment( + app, repo=minimal_repo, name=name, ref=ref, provider="git" + ) + assert r.status_code == 200 + uid = r.json().get("uid", None) + assert uid is not None + + await wait_for_image(image_name=generated_image_name) + await asyncio.sleep(3) + images_db = db_session.execute(sa.select(DockerImageSQL)).scalars().first() + assert images_db.name == generated_image_name + assert images_db.image_meta["display_name"] == name + assert images_db.image_meta["ref"] == ref + + +@pytest.mark.asyncio +async def test_delete_environment( + app, minimal_repo, image_name, generated_image_name, db_session +): + name, ref = image_name.split(":") + r = await add_environment( + app, repo=minimal_repo, name=name, ref=ref, provider="git" + ) + assert r.status_code == 200 + uid = r.json().get("uid", None) + assert uid is not None + + await wait_for_image(image_name=generated_image_name) + await asyncio.sleep(3) + r = await remove_environment(app, image_name=uid) + assert r.status_code == 200 + + # make sure the image does not exist anymore + docker = Docker() + with pytest.raises(DockerError): + await docker.images.inspect(generated_image_name) + await docker.close() + + +@pytest.mark.asyncio +async def test_delete_unknown_environment(app): + random_uid = "a025d82f-48a7-4d6b-ba31-e7056c3dbca6" + r = await remove_environment(app, image_name=random_uid) + assert r.status_code == 404 + + +@pytest.mark.asyncio +async def test_no_repo(app, image_name): + name, ref = image_name.split(":") + r = await add_environment(app, repo="", name=name, ref=ref, provider="git") + assert r.status_code == 400 + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "memory, cpu", + [ + ("abcded", ""), + ("", "abcde"), + ], +) +async def test_wrong_limits(app, minimal_repo, image_name, memory, cpu): + name, ref = image_name.split(":") + r = await add_environment( + app, + repo=minimal_repo, + name=name, + ref=ref, + memory=memory, + cpu=cpu, + provider="git", + ) + assert r.status_code == 400 + assert "must be a number" in r.text + + +@pytest.mark.asyncio +async def test_wrong_name(app, minimal_repo): + r = await add_environment( + app, repo=minimal_repo, name="#WRONG_NAME#", provider="git" + ) + assert r.status_code == 400 + + +@pytest.mark.asyncio +async def test_missing_provider(app, minimal_repo): + r = await add_environment(app, repo=minimal_repo, name="foobar") + assert r.status_code == 500 diff --git a/tljh_repo2docker/tests/binderhub_build/test_images.py b/tljh_repo2docker/tests/binderhub_build/test_images.py new file mode 100644 index 0000000..f79a2b1 --- /dev/null +++ b/tljh_repo2docker/tests/binderhub_build/test_images.py @@ -0,0 +1,38 @@ +import asyncio + +import pytest + +from ..utils import add_environment, get_service_page, wait_for_image + + +@pytest.mark.asyncio +async def test_images_list_not_admin(app): + cookies = await app.login_user("wash") + r = await get_service_page( + "environments", app, cookies=cookies, allow_redirects=True + ) + assert r.status_code == 403 + + +@pytest.mark.asyncio +async def test_images_list_admin(app, minimal_repo, image_name, generated_image_name): + cookies = await app.login_user("admin") + # add a new envionment + name, ref = image_name.split(":") + r = await add_environment( + app, repo=minimal_repo, name=name, ref=ref, provider="git" + ) + assert r.status_code == 200 + await wait_for_image(image_name=generated_image_name) + await asyncio.sleep(3) + # the environment should be on the page + r = await get_service_page( + "environments", + app, + cookies=cookies, + allow_redirects=True, + ) + r.raise_for_status() + + assert r.status_code == 200 + assert minimal_repo in r.text diff --git a/tljh_repo2docker/tests/binderhub_build/test_logs.py b/tljh_repo2docker/tests/binderhub_build/test_logs.py new file mode 100644 index 0000000..29123dc --- /dev/null +++ b/tljh_repo2docker/tests/binderhub_build/test_logs.py @@ -0,0 +1,42 @@ +import pytest +from jupyterhub.tests.utils import async_requests + +from ..utils import add_environment, api_request, next_event, wait_for_image + + +@pytest.mark.asyncio +async def test_stream_simple(app, minimal_repo, image_name): + name, ref = image_name.split(":") + build_response = await add_environment( + app, repo=minimal_repo, name=name, ref=ref, provider="git" + ) + uid = build_response.json().get("uid", None) + assert uid is not None + r = await api_request(app, "environments", uid, "logs", stream=True) + r.raise_for_status() + + assert r.headers["content-type"] == "text/event-stream" + ex = async_requests.executor + line_iter = iter(r.iter_lines(decode_unicode=True)) + evt = await ex.submit(next_event, line_iter) + evt = await ex.submit(next_event, line_iter) + msg = evt.get("message", "") + assert "Picked Git content provider." in msg + + r.close() + await wait_for_image(image_name=image_name) + + +@pytest.mark.asyncio +async def test_bad_uuid(app, image_name, request): + r = await api_request(app, "environments", "bad-uuid", "logs", stream=True) + request.addfinalizer(r.close) + assert r.status_code == 400 + + +@pytest.mark.asyncio +async def test_image_not_found(app, image_name, request): + random_uid = "a025d82f-48a7-4d6b-ba31-e7056c3dbca6" + r = await api_request(app, "environments", random_uid, "logs", stream=True) + request.addfinalizer(r.close) + assert r.status_code == 404 diff --git a/tljh_repo2docker/tests/conftest.py b/tljh_repo2docker/tests/conftest.py index 602f765..8b52f33 100644 --- a/tljh_repo2docker/tests/conftest.py +++ b/tljh_repo2docker/tests/conftest.py @@ -1,79 +1,19 @@ -import sys - import pytest -from aiodocker import Docker, DockerError -from traitlets.config import Config - -from tljh_repo2docker import tljh_custom_jupyterhub_config - -async def remove_docker_image(image_name): - async with Docker() as docker: - try: - await docker.images.delete(image_name, force=True) - except DockerError: - pass +from .utils import remove_docker_image -@pytest.fixture(scope="module") +@pytest.fixture(scope="session") def minimal_repo(): return "https://github.com/plasmabio/tljh-repo2docker-test-binder" -@pytest.fixture(scope="module") +@pytest.fixture(scope="session") def minimal_repo_uppercase(): return "https://github.com/plasmabio/TLJH-REPO2DOCKER-TEST-BINDER" -@pytest.fixture(scope="module") -def generated_image_name(): - return "plasmabio-tljh-repo2docker-test-binder:HEAD" - - -@pytest.fixture(scope="module") -def image_name(): - return "tljh-repo2docker-test:HEAD" - - -@pytest.fixture -async def app(hub_app): - config = Config() - tljh_custom_jupyterhub_config(config) - - config.JupyterHub.services.extend( - [ - { - "name": "tljh_repo2docker", - "url": "http://127.0.0.1:6789", - "command": [ - sys.executable, - "-m", - "tljh_repo2docker", - "--ip", - "127.0.0.1", - "--port", - "6789", - ], - "oauth_no_confirm": True, - } - ] - ) - - config.JupyterHub.load_roles = [ - { - "description": "Role for tljh_repo2docker service", - "name": "tljh-repo2docker-service", - "scopes": ["read:users", "read:servers", "read:roles:users"], - "services": ["tljh_repo2docker"], - } - ] - - app = await hub_app(config=config) - return app - - @pytest.fixture(autouse=True) -async def remove_all_test_images(image_name, generated_image_name, app): - yield +async def remove_all_test_images(image_name, generated_image_name): await remove_docker_image(image_name) await remove_docker_image(generated_image_name) diff --git a/tljh_repo2docker/tests/local_build/__init__.py b/tljh_repo2docker/tests/local_build/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tljh_repo2docker/tests/local_build/conftest.py b/tljh_repo2docker/tests/local_build/conftest.py new file mode 100644 index 0000000..bcc7069 --- /dev/null +++ b/tljh_repo2docker/tests/local_build/conftest.py @@ -0,0 +1,53 @@ +import sys + +import pytest +from traitlets.config import Config + +from tljh_repo2docker import tljh_custom_jupyterhub_config + + +@pytest.fixture(scope="module") +def generated_image_name(): + return "plasmabio-tljh-repo2docker-test-binder:HEAD" + + +@pytest.fixture(scope="module") +def image_name(): + return "tljh-repo2docker-test:HEAD" + + +@pytest.fixture +async def app(hub_app): + config = Config() + tljh_custom_jupyterhub_config(config) + + config.JupyterHub.services.extend( + [ + { + "name": "tljh_repo2docker", + "url": "http://127.0.0.1:6789", + "command": [ + sys.executable, + "-m", + "tljh_repo2docker", + "--ip", + "127.0.0.1", + "--port", + "6789", + ], + "oauth_no_confirm": True, + } + ] + ) + + config.JupyterHub.load_roles = [ + { + "description": "Role for tljh_repo2docker service", + "name": "tljh-repo2docker-service", + "scopes": ["read:users", "read:roles:users", "admin:servers"], + "services": ["tljh_repo2docker"], + } + ] + + app = await hub_app(config=config) + return app diff --git a/tljh_repo2docker/tests/test_builder.py b/tljh_repo2docker/tests/local_build/test_builder.py similarity index 97% rename from tljh_repo2docker/tests/test_builder.py rename to tljh_repo2docker/tests/local_build/test_builder.py index 8bd0114..5a7cf13 100644 --- a/tljh_repo2docker/tests/test_builder.py +++ b/tljh_repo2docker/tests/local_build/test_builder.py @@ -1,7 +1,7 @@ import pytest from aiodocker import Docker, DockerError -from .utils import add_environment, remove_environment, wait_for_image +from ..utils import add_environment, remove_environment, wait_for_image @pytest.mark.asyncio diff --git a/tljh_repo2docker/tests/test_images.py b/tljh_repo2docker/tests/local_build/test_images.py similarity index 83% rename from tljh_repo2docker/tests/test_images.py rename to tljh_repo2docker/tests/local_build/test_images.py index e90aff0..21ce841 100644 --- a/tljh_repo2docker/tests/test_images.py +++ b/tljh_repo2docker/tests/local_build/test_images.py @@ -1,7 +1,7 @@ import pytest from jupyterhub.tests.utils import get_page -from .utils import add_environment, get_service_page, wait_for_image +from ..utils import add_environment, get_service_page, wait_for_image @pytest.mark.asyncio @@ -15,7 +15,7 @@ async def test_images_list_admin(app): ) r.raise_for_status() assert ( - '{"images": [], "default_mem_limit": "None", "default_cpu_limit":"None", "machine_profiles": []}' + '{"repo_providers": [{"label": "Git", "value": "git"}], "use_binderhub": false, "images": [], "default_mem_limit": "None", "default_cpu_limit":"None", "machine_profiles": []}' in r.text ) diff --git a/tljh_repo2docker/tests/test_logs.py b/tljh_repo2docker/tests/local_build/test_logs.py similarity index 66% rename from tljh_repo2docker/tests/test_logs.py rename to tljh_repo2docker/tests/local_build/test_logs.py index 8e3ab3b..077e75a 100644 --- a/tljh_repo2docker/tests/test_logs.py +++ b/tljh_repo2docker/tests/local_build/test_logs.py @@ -1,22 +1,7 @@ -import json - import pytest from jupyterhub.tests.utils import async_requests -from .utils import add_environment, api_request, wait_for_image - - -def next_event(it): - """read an event from an eventstream - From: https://github.com/jupyterhub/jupyterhub/blob/81d423d6c674765400a6fe88064c1366b7070f94/jupyterhub/tests/test_api.py#L692-L700 - """ - while True: - try: - line = next(it) - except StopIteration: - return - if line.startswith("data:"): - return json.loads(line.split(":", 1)[1]) +from ..utils import add_environment, api_request, next_event, wait_for_image @pytest.mark.asyncio diff --git a/tljh_repo2docker/tests/utils.py b/tljh_repo2docker/tests/utils.py index 14d9d3e..9387967 100644 --- a/tljh_repo2docker/tests/utils.py +++ b/tljh_repo2docker/tests/utils.py @@ -2,8 +2,13 @@ import json from aiodocker import Docker, DockerError -from jupyterhub.tests.utils import (async_requests, auth_header, - check_db_locks, public_host, public_url) +from jupyterhub.tests.utils import ( + async_requests, + auth_header, + check_db_locks, + public_host, + public_url, +) from jupyterhub.utils import url_path_join as ujoin from tornado.httputil import url_concat @@ -44,21 +49,24 @@ def get_service_page(path, app, **kw): return async_requests.get(url, **kw) -async def add_environment(app, *, repo, ref="HEAD", name="", memory="", cpu=""): +async def add_environment( + app, *, repo, ref="HEAD", name="", memory="", cpu="", provider=None +): """Use the POST endpoint to add a new environment""" + data = { + "repo": repo, + "ref": ref, + "name": name, + "memory": memory, + "cpu": cpu, + } + if provider: + data["provider"] = provider r = await api_request( app, "environments", method="post", - data=json.dumps( - { - "repo": repo, - "ref": ref, - "name": name, - "memory": memory, - "cpu": cpu, - } - ), + data=json.dumps(data), ) return r @@ -93,3 +101,24 @@ async def remove_environment(app, *, image_name): ), ) return r + + +async def remove_docker_image(image_name): + async with Docker() as docker: + try: + await docker.images.delete(image_name, force=True) + except DockerError: + pass + + +def next_event(it): + """read an event from an eventstream + From: https://github.com/jupyterhub/jupyterhub/blob/81d423d6c674765400a6fe88064c1366b7070f94/jupyterhub/tests/test_api.py#L692-L700 + """ + while True: + try: + line = next(it) + except StopIteration: + return + if line.startswith("data:"): + return json.loads(line.split(":", 1)[1]) diff --git a/ui-tests/binderhub_config.py b/ui-tests/binderhub_config.py new file mode 100644 index 0000000..febcd0d --- /dev/null +++ b/ui-tests/binderhub_config.py @@ -0,0 +1,46 @@ +""" +A development config to test BinderHub locally. + +If you are running BinderHub manually (not via JupyterHub) run +`python -m binderhub -f binderhub_config.py` + +Override the external access URL for JupyterHub by setting the +environment variable JUPYTERHUB_EXTERNAL_URL +Host IP is needed in a few places +""" + +import os + +from binderhub.build_local import LocalRepo2dockerBuild +from binderhub.quota import LaunchQuota + + +c.BinderHub.debug = True +c.BinderHub.auth_enabled = True +c.BinderHub.enable_api_only_mode = True + +use_registry = bool(os.getenv("BINDERHUB_USE_REGISTRY", False)) +c.BinderHub.use_registry = use_registry + +if use_registry: + c.BinderHub.image_prefix = os.getenv( + "BINDERHUB_IMAGE_PREFIX", "" + ) # https://binderhub.readthedocs.io/en/latest/zero-to-binderhub/setup-binderhub.html#id2 + c.DockerRegistry.auth_config_url = "https://index.docker.io/v1/" + + c.BuildExecutor.push_secret = "*" # + +c.BinderHub.builder_required = False + +c.BinderHub.build_class = LocalRepo2dockerBuild +c.BinderHub.launch_quota_class = LaunchQuota + +c.BinderHub.hub_url_local = "http://localhost:8000" +# c.BinderHub.enable_api_only_mode = True + +# Assert that we're running as a managed JupyterHub service +# (otherwise c.BinderHub.hub_api_token is needed) +assert os.getenv("JUPYTERHUB_API_TOKEN") + +c.BinderHub.base_url = os.getenv("JUPYTERHUB_SERVICE_PREFIX") +c.BinderHub.hub_url = os.getenv("JUPYTERHUB_BASE_URL") diff --git a/ui-tests/tests/ui.test.ts-snapshots/admin-linux.png b/ui-tests/binderhub_snapshots/ui.test.ts/admin.png similarity index 100% rename from ui-tests/tests/ui.test.ts-snapshots/admin-linux.png rename to ui-tests/binderhub_snapshots/ui.test.ts/admin.png diff --git a/ui-tests/tests/ui.test.ts-snapshots/environment-console-linux.png b/ui-tests/binderhub_snapshots/ui.test.ts/environment-console.png similarity index 100% rename from ui-tests/tests/ui.test.ts-snapshots/environment-console-linux.png rename to ui-tests/binderhub_snapshots/ui.test.ts/environment-console.png diff --git a/ui-tests/binderhub_snapshots/ui.test.ts/environment-dialog.png b/ui-tests/binderhub_snapshots/ui.test.ts/environment-dialog.png new file mode 100644 index 0000000..7bffcb0 Binary files /dev/null and b/ui-tests/binderhub_snapshots/ui.test.ts/environment-dialog.png differ diff --git a/ui-tests/tests/ui.test.ts-snapshots/environment-list-linux.png b/ui-tests/binderhub_snapshots/ui.test.ts/environment-list.png similarity index 100% rename from ui-tests/tests/ui.test.ts-snapshots/environment-list-linux.png rename to ui-tests/binderhub_snapshots/ui.test.ts/environment-list.png diff --git a/ui-tests/tests/ui.test.ts-snapshots/environment-remove-confirm-linux.png b/ui-tests/binderhub_snapshots/ui.test.ts/environment-remove-confirm.png similarity index 100% rename from ui-tests/tests/ui.test.ts-snapshots/environment-remove-confirm-linux.png rename to ui-tests/binderhub_snapshots/ui.test.ts/environment-remove-confirm.png diff --git a/ui-tests/tests/ui.test.ts-snapshots/environment-removed-linux.png b/ui-tests/binderhub_snapshots/ui.test.ts/environment-removed.png similarity index 100% rename from ui-tests/tests/ui.test.ts-snapshots/environment-removed-linux.png rename to ui-tests/binderhub_snapshots/ui.test.ts/environment-removed.png diff --git a/ui-tests/tests/ui.test.ts-snapshots/environments-page-linux.png b/ui-tests/binderhub_snapshots/ui.test.ts/environments-page.png similarity index 100% rename from ui-tests/tests/ui.test.ts-snapshots/environments-page-linux.png rename to ui-tests/binderhub_snapshots/ui.test.ts/environments-page.png diff --git a/ui-tests/tests/ui.test.ts-snapshots/login-page-linux.png b/ui-tests/binderhub_snapshots/ui.test.ts/login-page.png similarity index 100% rename from ui-tests/tests/ui.test.ts-snapshots/login-page-linux.png rename to ui-tests/binderhub_snapshots/ui.test.ts/login-page.png diff --git a/ui-tests/binderhub_snapshots/ui.test.ts/running-servers.png b/ui-tests/binderhub_snapshots/ui.test.ts/running-servers.png new file mode 100644 index 0000000..1276c50 Binary files /dev/null and b/ui-tests/binderhub_snapshots/ui.test.ts/running-servers.png differ diff --git a/ui-tests/binderhub_snapshots/ui.test.ts/server-remove-confirm.png b/ui-tests/binderhub_snapshots/ui.test.ts/server-remove-confirm.png new file mode 100644 index 0000000..09f970b Binary files /dev/null and b/ui-tests/binderhub_snapshots/ui.test.ts/server-remove-confirm.png differ diff --git a/ui-tests/tests/ui.test.ts-snapshots/server-removed-linux.png b/ui-tests/binderhub_snapshots/ui.test.ts/server-removed.png similarity index 100% rename from ui-tests/tests/ui.test.ts-snapshots/server-removed-linux.png rename to ui-tests/binderhub_snapshots/ui.test.ts/server-removed.png diff --git a/ui-tests/tests/ui.test.ts-snapshots/servers-dialog-linux.png b/ui-tests/binderhub_snapshots/ui.test.ts/servers-dialog.png similarity index 100% rename from ui-tests/tests/ui.test.ts-snapshots/servers-dialog-linux.png rename to ui-tests/binderhub_snapshots/ui.test.ts/servers-dialog.png diff --git a/ui-tests/tests/ui.test.ts-snapshots/servers-page-linux.png b/ui-tests/binderhub_snapshots/ui.test.ts/servers-page.png similarity index 100% rename from ui-tests/tests/ui.test.ts-snapshots/servers-page-linux.png rename to ui-tests/binderhub_snapshots/ui.test.ts/servers-page.png diff --git a/ui-tests/tests/ui.test.ts-snapshots/user-linux.png b/ui-tests/binderhub_snapshots/ui.test.ts/user.png similarity index 100% rename from ui-tests/tests/ui.test.ts-snapshots/user-linux.png rename to ui-tests/binderhub_snapshots/ui.test.ts/user.png diff --git a/ui-tests/jupyterhub_config_binderhub.py b/ui-tests/jupyterhub_config_binderhub.py new file mode 100644 index 0000000..10b251f --- /dev/null +++ b/ui-tests/jupyterhub_config_binderhub.py @@ -0,0 +1,106 @@ +""" +This file is only used for local development +and overrides some of the default values from the plugin. +""" + +import os +from pathlib import Path +from jupyterhub.auth import DummyAuthenticator +from tljh.configurer import apply_config, load_config +from tljh_repo2docker import tljh_custom_jupyterhub_config, TLJH_R2D_ADMIN_SCOPE +import sys + + +HERE = Path(__file__).parent +tljh_config = load_config() +tljh_config["services"]["cull"]["enabled"] = False +apply_config(tljh_config, c) + +tljh_custom_jupyterhub_config(c) +tljh_repo2docker_config = HERE / "tljh_repo2docker_binderhub.py" + +c.JupyterHub.authenticator_class = DummyAuthenticator + +c.JupyterHub.allow_named_servers = True +c.JupyterHub.ip = "0.0.0.0" + + +binderhub_service_name = "binder" +binderhub_config = HERE / "binderhub_config.py" + + +binderhub_environment = {} +for env_var in ["JUPYTERHUB_EXTERNAL_URL", "GITHUB_ACCESS_TOKEN"]: + if os.getenv(env_var) is not None: + binderhub_environment[env_var] = os.getenv(env_var) + +c.JupyterHub.services.extend( + [ + { + "name": binderhub_service_name, + "admin": True, + "command": [ + sys.executable, + "-m", + "binderhub", + f"--config={binderhub_config}", + ], + "url": "http://localhost:8585", + "environment": binderhub_environment, + "oauth_client_id": "service-binderhub", + "oauth_no_confirm": True, + }, + { + "name": "tljh_repo2docker", + "url": "http://127.0.0.1:6789", + "command": [ + sys.executable, + "-m", + "tljh_repo2docker", + "--ip", + "0.0.0.0", + "--port", + "6789", + "--config", + f"{tljh_repo2docker_config}", + ], + "oauth_no_confirm": True, + "oauth_client_allowed_scopes": [ + TLJH_R2D_ADMIN_SCOPE, + ], + }, + ] +) + +c.JupyterHub.custom_scopes = { + TLJH_R2D_ADMIN_SCOPE: { + "description": "Admin access to myservice", + }, +} + +c.JupyterHub.load_roles = [ + { + "description": "Role for tljh_repo2docker service", + "name": "tljh-repo2docker-service", + "scopes": [ + "read:users", + "read:roles:users", + "admin:servers", + "access:services!service=binder", + ], + "services": ["tljh_repo2docker"], + }, + { + "name": "tljh-repo2docker-service-admin", + "users": ["alice"], + "scopes": [TLJH_R2D_ADMIN_SCOPE], + }, + { + "name": "user", + "scopes": [ + "self", + # access to the env page + "access:services!service=tljh_repo2docker", + ], + }, +] diff --git a/jupyterhub_config.py b/ui-tests/jupyterhub_config_local.py similarity index 100% rename from jupyterhub_config.py rename to ui-tests/jupyterhub_config_local.py diff --git a/ui-tests/local_snapshots/ui.test.ts/admin.png b/ui-tests/local_snapshots/ui.test.ts/admin.png new file mode 100644 index 0000000..f1bd625 Binary files /dev/null and b/ui-tests/local_snapshots/ui.test.ts/admin.png differ diff --git a/ui-tests/local_snapshots/ui.test.ts/environment-console.png b/ui-tests/local_snapshots/ui.test.ts/environment-console.png new file mode 100644 index 0000000..d517fcf Binary files /dev/null and b/ui-tests/local_snapshots/ui.test.ts/environment-console.png differ diff --git a/ui-tests/tests/ui.test.ts-snapshots/environment-dialog-linux.png b/ui-tests/local_snapshots/ui.test.ts/environment-dialog.png similarity index 100% rename from ui-tests/tests/ui.test.ts-snapshots/environment-dialog-linux.png rename to ui-tests/local_snapshots/ui.test.ts/environment-dialog.png diff --git a/ui-tests/local_snapshots/ui.test.ts/environment-list.png b/ui-tests/local_snapshots/ui.test.ts/environment-list.png new file mode 100644 index 0000000..966517a Binary files /dev/null and b/ui-tests/local_snapshots/ui.test.ts/environment-list.png differ diff --git a/ui-tests/local_snapshots/ui.test.ts/environment-remove-confirm.png b/ui-tests/local_snapshots/ui.test.ts/environment-remove-confirm.png new file mode 100644 index 0000000..2d9a6f5 Binary files /dev/null and b/ui-tests/local_snapshots/ui.test.ts/environment-remove-confirm.png differ diff --git a/ui-tests/local_snapshots/ui.test.ts/environment-removed.png b/ui-tests/local_snapshots/ui.test.ts/environment-removed.png new file mode 100644 index 0000000..185a854 Binary files /dev/null and b/ui-tests/local_snapshots/ui.test.ts/environment-removed.png differ diff --git a/ui-tests/local_snapshots/ui.test.ts/environments-page.png b/ui-tests/local_snapshots/ui.test.ts/environments-page.png new file mode 100644 index 0000000..185a854 Binary files /dev/null and b/ui-tests/local_snapshots/ui.test.ts/environments-page.png differ diff --git a/ui-tests/local_snapshots/ui.test.ts/login-page.png b/ui-tests/local_snapshots/ui.test.ts/login-page.png new file mode 100644 index 0000000..17033c5 Binary files /dev/null and b/ui-tests/local_snapshots/ui.test.ts/login-page.png differ diff --git a/ui-tests/tests/ui.test.ts-snapshots/running-servers-linux.png b/ui-tests/local_snapshots/ui.test.ts/running-servers.png similarity index 100% rename from ui-tests/tests/ui.test.ts-snapshots/running-servers-linux.png rename to ui-tests/local_snapshots/ui.test.ts/running-servers.png diff --git a/ui-tests/tests/ui.test.ts-snapshots/server-remove-confirm-linux.png b/ui-tests/local_snapshots/ui.test.ts/server-remove-confirm.png similarity index 100% rename from ui-tests/tests/ui.test.ts-snapshots/server-remove-confirm-linux.png rename to ui-tests/local_snapshots/ui.test.ts/server-remove-confirm.png diff --git a/ui-tests/local_snapshots/ui.test.ts/server-removed.png b/ui-tests/local_snapshots/ui.test.ts/server-removed.png new file mode 100644 index 0000000..a4ea831 Binary files /dev/null and b/ui-tests/local_snapshots/ui.test.ts/server-removed.png differ diff --git a/ui-tests/local_snapshots/ui.test.ts/servers-dialog.png b/ui-tests/local_snapshots/ui.test.ts/servers-dialog.png new file mode 100644 index 0000000..474517f Binary files /dev/null and b/ui-tests/local_snapshots/ui.test.ts/servers-dialog.png differ diff --git a/ui-tests/local_snapshots/ui.test.ts/servers-page.png b/ui-tests/local_snapshots/ui.test.ts/servers-page.png new file mode 100644 index 0000000..f1bd625 Binary files /dev/null and b/ui-tests/local_snapshots/ui.test.ts/servers-page.png differ diff --git a/ui-tests/local_snapshots/ui.test.ts/user.png b/ui-tests/local_snapshots/ui.test.ts/user.png new file mode 100644 index 0000000..0ba2fe8 Binary files /dev/null and b/ui-tests/local_snapshots/ui.test.ts/user.png differ diff --git a/ui-tests/package.json b/ui-tests/package.json index ed8db4c..ef3e8d4 100644 --- a/ui-tests/package.json +++ b/ui-tests/package.json @@ -4,7 +4,8 @@ "description": "tljh_repo2docker UI Tests", "private": true, "scripts": { - "test": "npx playwright test --workers 1", + "test:local": "CONFIG_FILE=local npx playwright test --workers 1", + "test:binderhub": "CONFIG_FILE=binderhub npx playwright test --workers 1", "test:update": "npx playwright test --update-snapshots", "test:debug": "PWDEBUG=1 npx playwright test --workers 1" }, diff --git a/ui-tests/playwright.config.js b/ui-tests/playwright.config.js index e0d50c6..05942d8 100644 --- a/ui-tests/playwright.config.js +++ b/ui-tests/playwright.config.js @@ -1,19 +1,25 @@ +const snapshotDir = `${process.env.CONFIG_FILE}_snapshots`; + +const configFile = `jupyterhub_config_${process.env.CONFIG_FILE}.py`; + module.exports = { timeout: 600000, reporter: [[process.env.CI ? 'dot' : 'list'], ['html']], + outputDir: `${process.env.CONFIG_FILE}-test-results`, use: { baseURL: 'http://localhost:8000', video: 'retain-on-failure', - trace: 'on-first-retry' + trace: 'retain-on-failure' }, - retries: 1, + retries: 0, expect: { toMatchSnapshot: { maxDiffPixelRatio: 0.001 } }, + snapshotPathTemplate: `{testDir}/${snapshotDir}/{testFileName}/{arg}{ext}`, webServer: { - command: 'python -m jupyterhub -f ../jupyterhub_config.py', + command: `python -m jupyterhub -f ./${configFile}`, url: 'http://localhost:8000', timeout: 120 * 1000, reuseExistingServer: true diff --git a/ui-tests/tests/ui.test.ts b/ui-tests/tests/ui.test.ts index d4fb46a..23ec77b 100644 --- a/ui-tests/tests/ui.test.ts +++ b/ui-tests/tests/ui.test.ts @@ -98,10 +98,9 @@ test.describe('tljh_repo2docker UI Tests', () => { .getByRole('button') .first() .click(); - await page.waitForSelector( - 'span:has-text("Successfully tagged python-env:HEAD")', - { timeout: 600000 } - ); + await page.waitForSelector('span:has-text("Successfully tagged")', { + timeout: 600000 + }); expect(await page.screenshot()).toMatchSnapshot('environment-console.png', { maxDiffPixelRatio: 0.05 }); @@ -143,7 +142,7 @@ test.describe('tljh_repo2docker UI Tests', () => { name: 'Create Server' }); await createServer.click(); - await expect(createServer).toHaveCount(0); + await expect(createServer).toHaveCount(0, { timeout: 20000 }); await page.waitForURL('**/servers'); await page.waitForTimeout(1000); diff --git a/ui-tests/tljh_repo2docker_binderhub.py b/ui-tests/tljh_repo2docker_binderhub.py new file mode 100644 index 0000000..e00e7ba --- /dev/null +++ b/ui-tests/tljh_repo2docker_binderhub.py @@ -0,0 +1,9 @@ +c.TljhRepo2Docker.db_url = "sqlite:///tljh_repo2docker.sqlite" + +c.TljhRepo2Docker.machine_profiles = [ + {"label": "Small", "cpu": 2, "memory": 2}, + {"label": "Medium", "cpu": 4, "memory": 4}, + {"label": "Large", "cpu": 8, "memory": 8}, +] + +c.TljhRepo2Docker.binderhub_url = "http://localhost:8585/services/binder/"