diff --git a/README.adoc b/README.adoc new file mode 100644 index 00000000..779602e8 --- /dev/null +++ b/README.adoc @@ -0,0 +1,321 @@ +:toc: +:!toc-title: +:toclevels: 3 +:sectnums: 3 +:doctype: book + += OpenBikeSensor Portal + +This repository contains the source code required to run the +https://openbikesensor.org[OpenBikeSensor] data collection portal. It is +separated into the components: + +* **api**: The backend service, written in Python 3 with https://sanicframework.org[Sanic], + https://www.sqlalchemy.org[SQLAlchemy], and a https://www.postgresql.org[PostgreSQL] + / https://postgis.net/[PostGIS] database for storage. It also highly depends on + https://openmaptiles.org[OpenMapTiles] to generate vector tiles of the data. +* **frontend**: A https://react.dev[React] single-page application which allows + access to the data, provides summaries and visualizations, and lets users + adjust settings and manage and publish their tracks. + +In the link:docs/architecture.md[Architecture Documentation] you will find further +details on the components that make up the entire application. + +This project follows https://semver.org[semantic versioning]. Refer to +https://github.com/openbikesensor/portal/issues/44[issue #44] for a description +of what it means for our project and what is considered the public interface. + +== Clone the Project + +First of all, you must clone this project. This project uses submodules, +thus ensure that they are cloned as well: + +[source,shell] +---- +git clone --recursive https://github.com/openbikesensor/portal + +# ... or if you forgot the --recursive argument, you can run this in the +# repository's directory later: +git submodule update --init --recursive +---- + +== Production setup + +There is a guide for a deployment, based on Docker in the +link:deployment[deployment] directory. Lots of non-Docker deployment strategies +are possible, but they are not officially supported, so please do not expect +the authors of the software to assist in troubleshooting. + +This is a rather complex application, and it is expected that you know the +basics of deploying a modern web application securely onto a production server. +We are sorry that we cannot guide you through all the details of that, as we +just don't have the capacities to do so. Please research the respective topics +first. If you struggle with application-specific issues, please let us know, we +might be able to assist with those. + +Please note that you will always need to install your own reverse proxy that +terminates TLS for you and handles certificates. We do not support TLS directly +in the application; instead, please use this preferred method. + +=== Migrating (Production) + +Migrations are done with +https://alembic.sqlalchemy.org/en/latest/index.html[Alembic], please refer to +its documentation for help. Most of the time, running this command will do all +the migrations you need: + +[source,shell] +---- +docker-compose run --rm api tools/upgrade.py +---- + +This command is equivalent to running migrations through *alembic*, then +regenerating the SQL functions that compute vector tiles directly in the +database: + +[source,shell] +---- +# equivalent to the above command, you don't usually run these +docker-compose run --rm api alembic upgrade head +docker-compose run --rm api tools/prepare_sql_tiles +---- + +=== Upgrading from v0.2 to v0.3 + +After v0.2 we switched the underlying technology of the API and the database. +We now have no more MongoDB; everything has moved to the PostgreSQL installation. +For development setups, it is advised to just reset the whole state (remove the +`local` folder) and start fresh. For production upgrades, please follow the +relevant section in link:UPGRADING.md[UPGRADING.md]. + + +== Development setup + +We've moved the whole development setup into Docker to make it easy for +everyone to get involved. + +=== Install docker + +Please install https://docs.docker.com/engine/install[Docker Engine] as well as +https://docs.docker.com/compose/install[Docker Compose] onto your machine. + +NOTE: Depending on the Docker Engine version, you might be able to run + `docker compose` instead of `docker-compose`. In that case you don't need to + install the standalone Docker Compose. + + + + Following the + https://docs.docker.com/compose/install/#scenario-three-install-the-compose-standalone[Docker documentation] + the use of the standalone `docker-compose` is not supported any longer by the + latest Docker Engine. In this documentation the commands use the new form + `docker compose`. + +Clone the repository as described before. + +=== Prepare database + +Start the PostgreSQL database: + +[source,shell] +---- +docker compose up --detach postgres +---- + +The first time you start the `postgres` container, a lot of extensions will be +installed. This might take a while, so check the logs of the docker container +(e.g. `docker compose logs --follow postgres`) until you see: + +---- +PostgreSQL init process complete; ready for start up. +---- + +If you don't wait long enough, the following commands might fail. In this case, +you can always stop the container, remove the data directory (`local/postgres/`) +and restart the process. + +Run the upgrade command to generate the database schema: + +[source,shell] +---- +docker compose run --rm api tools/upgrade.py +---- + +You will need to re-run this command after updates, to migrate the database and +(re-)create the functions in the SQL database that are used when generating +vector tiles. + +You should now also import OpenStreetMap data, see +link:#_import_openstreetmap_data[below for instructions]. + +=== Configure Keycloak + +The Login will not be possible until you configure the KeyCloak realm correctly. +Start your keycloak instance: + +[source,shell] +---- +docker compose up --detach keycloak +---- + +Navigate to http://localhost:3003/auth and follow these steps: + +- Click on *Administration Console* and sign-in with username: `admin` / password: `admin`. +- Click on the drop-down in the upper left corner (it shows _master_), click on *Create Realm*. +- Enter `obs-dev` (spelling matters) as *Realm name*, click on *Create*. +- In the sidebar, navigate to *Manage* → *Clients*, click on *Create client*. +- _(1) General Settings_: Enter `portal` as *Client ID*, click on *Next*. +- _(2) Capability config_: Switch *Client authentication* on, leave all other + settings as they are, click on *Next*. +- _(3) Login settings_: As *Valid Redirect URIs* enter + `\http://localhost:3000/login/redirect`, click on *Save*. +- Click on tab *Credentials*, copy the *Client secret* to the clipboard. +- Create a file `api/config.overrides.py` and store the secret in it as: + + KEYCLOAK_CLIENT_SECRET = "your client secret" + + +NOTE: You can use the file `api/config.overrides.py` in development mode to change + settings without editing the git-tracked default file `api/config.dev.py`. + Options in the file `api/config.overrides.py` take precedence. + +- In the sidebar, navigate to *Manage* → *Users*, click on *Create new user*. +- Give the user a name, e.g. `test`, set the switch *Email verified* to `yes`, + leave all other settings as they are, click on *Create*. +- On tab *Credentials*, click *Set password*, enter a password and set + *Temporary* to `off`, click on *Save*. + +We are going to automate this process. For now, you will have to repeat it +every time you reset your Keycloak settings, which are stored inside the +PostgreSQL as well. Luckily, the script `api/tools/reset_database.py` does +*not* affect the state of the Keycloak database, so this should be rather rare. + +=== Boot the application + +Now you can run the remaining parts of the application: + +[source,shell] +---- +docker compose up --detach --build api worker frontend +---- + +Your frontend should be running at http://localhost:3001 and the API at +http://localhost:3000 - but you probably only need to access the frontend for +testing. + +=== Start and stop the application + +To graceful stop all application containers run + +[source,shell] +---- +docker compose stop +---- + +To restart the stopped containers run + +[source,shell] +---- +docker compose start +---- + +=== Migrating (Development) + +Migrations are done with https://alembic.sqlalchemy.org/en/latest/index.html[Alembic], +please refer to its documentation for help. Most of the time, running this +command will do all the migrations you need: + +[source,shell] +---- +docker compose run --rm api alembic upgrade head +---- + +== Import OpenStreetMap data + +NOTE: This step may be skipped if you are using link:docs/lean-mode.md[Lean mode]. + +You need to import road information from OpenStreetMap for the portal to work. +This information is stored in your PostgreSQL database and used when processing +tracks (instead of querying the Overpass API), as well as for vector tile +generation. The process applies to both development and production setups. For +development, you should choose a small area for testing, such as your local +county or city, to keep the amount of data small. For production use you have +to import the whole region you are serving. + +* Install https://osm2pgsql.org/doc/install.html[osm2pgsql]. +* Download the area(s) you would like to import from https://download.geofabrik.de[GeoFabrik]. +* Import each file like this: + + osm2pgsql --create --hstore --style roads_import.lua --output=flex \ + --host=localhost --database=postgres --username=keycloak --password \ + path/to/downloaded/myarea-latest.osm.pbf + +You might need to adjust the host, database and username to your setup, and also +provide the correct password when queried. For the development-setup the password +is `password`. For production, you might need to expose the containers port and/or +create a TCP tunnel, for example with SSH, such that you can run the import from +your local host and write to the remote database. + +The import process should take a few seconds to minutes, depending on the area +size. A whole country might even take one or more hours. You should probably +not try to import `planet.osm.pbf`. + +You can run the process multiple times, with the same or different area files, +to import or update the data. However, for this to work, the actual +https://osm2pgsql.org/doc/manual.html#running-osm2pgsql[command +line arguments] are a bit different each time, including when first importing, +and the disk space required is much higher. + +Refer to the documentation of `osm2pgsql` for assistance. We are using the "flex +mode", the provided script `roads_import.lua` describes the transformations +and extractions to perform on the original data. + +== Troubleshooting + +If any step of the instructions does not work for you, please open a GitHub +issue and describe your problem, as it is important to us that onboarding is +super easy :) + +=== Connecting to the PostgreSQL database + +If you need to connect to your development PostgreSQL database, you should +install `psql` locally. The port `5432` is already forwarded, so you can connect +with: + +[source,shell] +---- +psql --host=localhost --username=keycloak --dbname=postgres +---- + +The password is `password`. + +== License + +---- +Copyright (C) 2020-2023 OpenBikeSensor Contributors +Contact: https://openbikesensor.org + +The OpenBikeSensor Portal is free software: you can redistribute it +and/or modify it under the terms of the GNU Lesser General Public License +as published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. + +The OpenBikeSensor Portal is distributed in the hope that it will be +useful, but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +General Public License for more details. + +You should have received a copy of the GNU Lesser General Public License +along with the OpenBikeSensor Portal. If not, see https://www.gnu.org/licenses. +---- + +See also link:COPYING[COPYING] and link:COPYING.LESSER[COPYING.LESSER]. + +The above does not apply to the files listed below, their respective licenses +are included in a file next to each of them, named accordingly: + +* `frontend/src/mapstyles/bright.json` +* `frontend/src/mapstyles/positron.json` + +There are lots of other licenses to consider when using this software, +especially in conjunction with imported data and other tools. Check out the +link:docs/licenses.md[Licenses Documentation] for an (unofficial) overview of the +license landscape surrounding this project. diff --git a/README.md b/README.md deleted file mode 100644 index 1fbbd7f6..00000000 --- a/README.md +++ /dev/null @@ -1,283 +0,0 @@ -# OpenBikeSensor Portal - -This repository contains the source code required to run the -[OpenBikeSensor](https://openbikesensor.org) data collection portal. It is -separated into components: - -* **api**: The backend service, written in Python 3 with - [Sanic](https://sanicframework.org/), - [SQLAlchemy](https://www.sqlalchemy.org/), and a PostgreSQL/PostGIS database - for storage. It also depends highly on - [OpenMapTiles](https://openmaptiles.org) to generate vector tiles of the - data. -* **frontend**: A React single-page application that allows access to the data, - provides summaries and visualizations, and lets users adjust settings and - manage and publish their tracks. - -Check out the [Architecture Documentation](docs/architecture.md) for more -details on what parts the whole application is made of. - -This project follows [semantic versioning](https://semver.org). Refer to [issue -#44](https://github.com/openbikesensor/portal/issues/44) for a description of -what that means for our project and what is considered the public interface. - -## Clone the Project - -First of all, you must clone this project. This project uses submodules, -thus ensure, that they are cloned as well: - -```bash -git clone --recursive https://github.com/openbikesensor/portal - -# ... or if you forgot the --recursive argument, you can run this in the -# repository's directory later: -git submodule update --init --recursive -``` - -## Production setup - -There is a guide for a deployment based on docker in the -[deployment](deployment) folder. Lots of non-docker deployment strategy are -possible, but they are not "officially" supported, so please do not expect the -authors of the software to assist in troubleshooting. - -This is a rather complex application, and it is expected that you know the -basics of deploying a modern web application securely onto a production server. -We are sorry that we cannot guide you through all the details of that, as we -just don't have the capacities to do so. Please research the respective topics -first. If you struggle with application-specific issues, please let us know, we -might be able to assist with those. - -Please note that you will always need to install your own reverse proxy that -terminates TLS for you and handles certificates. We do not support TLS directly -in the application, instead, please use this prefered method. - -Upgrading and migrating is descrube - -### Migrating (Production) - -Migrations are done with -[Alembic](https://alembic.sqlalchemy.org/en/latest/index.html), please refer to -its documentation for help. Most of the time, running this command will do all -the migrations you need: - -```bash -docker-compose run --rm api tools/upgrade.py -``` - -This command is equivalent to running migrations through *alembic*, then -regenerating the SQL functions that compute vector tiles directly in the -database: - -```bash -# equivalent to the above command, you don't usually run these -docker-compose run --rm api alembic upgrade head -docker-compose run --rm api tools/prepare_sql_tiles -``` - - -docker-compose run --rm api alembic upgrade head - -### Upgrading from v0.2 to v0.3 - -After v0.2 we switched the underlying technology of the API and the database. -We now have no more MongoDB, instead, everything has moved to the PostgreSQL -installation. For development setups, it is advised to just reset the whole -state (remove the `local` folder) and start fresh. For production upgrades, -please follow the relevant section in [`UPGRADING.md`](./UPGRADING.md). - - -## Development setup - -We've moved the whole development setup into Docker to make it easy for -everyone to get involved. - -### Install docker - -Please [install Docker Engine](https://docs.docker.com/engine/install/) as well as -[Docker Compose](https://docs.docker.com/compose/install/) onto your machine. - -Then clone the repository as described above. - -### Configure Keycloak - - -Login will not be possible until you configure the keycloak realm correctly. Boot your keycloak instance: - -```bash -docker-compose up -d keycloak -``` - -Now navigate to http://localhost:3003/ and follow these steps: - -- Click *Administration Console* and log in with `admin` / `admin`. -- Hover over the realm name on the top left and click *Add realm*. -- Name the Realm `obs-dev` (spelling matters) and create it. -- In the sidebar, navigate to *Configure* → *Clients*, and click *Create* on the top right. -- *Client ID* should be `portal`. Click *Save*. -- In the Tab *Settings*, edit the new client's *Access Type* to *confidential* - and enter as *Valid Redirect URIs*: `http://localhost:3000/login/redirect`, - then *Save* -- Under *Credentials*, copy the *Secret*. Create a file at `api/config.overrides.py` with the secret in it: - - ```python - KEYCLOAK_CLIENT_SECRET="your secret here" - ``` - - You can use this file in development mode to change settings without editing - the git-controlled default file at `api/config.dev.py`. Options in this file - take precendence. -- In the sidebar, navigate to *Manage* → *Users*, and click *Add user* on the top right. -- Give the user a name (e.g. `test`), leave the rest as-is. -- Under the tab *Credentials*, choose a new password, and make it - non-temporary. Click *Set Password*. - -We are going to automate this process. For now, you will have to repeat it -every time you reset your keycloak settings, which are stored inside the -PostgreSQL as well. Luckily, the script `api/tools/reset_database.py` does -*not* affect the state of the keycloak database, so this should be rather rare. - -### Prepare database - -Start the PostgreSQL database: - -```bash -docker-compose up -d postgres -``` - -The first time you start postgres, a lot of extensions will be installed. This -takes a while, so check the logs of the docker container until you see: - -> PostgreSQL init process complete; ready for start up. - -If you don't wait long enough, the following commands might fail. In this case, -you can always stop the container, remove the data directory (`local/postgres`) -and restart the process. - -Next, run the upgrade command to generate the database schema: - -```bash -docker-compose run --rm api tools/upgrade.py -``` - -You will need to re-run this command after updates, to migrate the database and -(re-)create the functions in the SQL database that are used when generating -vector tiles. - -You should also import OpenStreetMap data now, see below for instructions. - -### Boot the application - -Now you can run the remaining parts of the application: - -```bash -docker-compose up -d --build api worker frontend -``` - -Your frontend should be running at http://localhost:3001 and the API at -http://localhost:3000 -- but you probably only need to access the frontend for -testing. - -### Migrating (Development) - -Migrations are done with -[Alembic](https://alembic.sqlalchemy.org/en/latest/index.html), please refer to -its documentation for help. Most of the time, running this command will do all -the migrations you need: - -```bash -docker-compose run --rm api alembic upgrade head -``` - - -## Import OpenStreetMap data - -**Hint:** This step may be skipped if you are using [Lean mode](./docs/lean-mode.md). - -You need to import road information from OpenStreetMap for the portal to work. -This information is stored in your PostgreSQL database and used when processing -tracks (instead of querying the Overpass API), as well as for vector tile -generation. The process applies to both development and production setups. For -development, you should choose a small area for testing, such as your local -county or city, to keep the amount of data small. For production use you have -to import the whole region you are serving. - -* Install `osm2pgsql`. -* Download the area(s) you would like to import from [GeoFabrik](https://download.geofabrik.de). -* Import each file like this: - - ```bash - osm2pgsql --create --hstore --style roads_import.lua -O flex \ - -H localhost -d obs -U obs -W \ - path/to/downloaded/myarea-latest.osm.pbf - ``` - -You might need to adjust the host, database and username (`-H`, `-d`, `-U`) to -your setup, and also provide the correct password when queried. For the -development setup the password is `obs`. For production, you might need to -expose the containers port and/or create a TCP tunnel, for example with SSH, -such that you can run the import from your local host and write to the remote -database. - -The import process should take a few seconds to minutes, depending on the area -size. A whole country might even take one or more hours. You should probably -not try to import `planet.osm.pbf`. - -You can run the process multiple times, with the same or different area files, -to import or update the data. However, for this to work, the actual [command -line arguments](https://osm2pgsql.org/doc/manual.html#running-osm2pgsql) are a -bit different each time, including when first importing, and the disk space -required is much higher. - -Refer to the documentation of `osm2pgsql` for assistance. We are using "flex -mode", the provided script `roads_import.lua` describes the transformations -and extractions to perform on the original data. - -## Troubleshooting - -If any step of the instructions does not work for you, please open an issue and -describe the problem you're having, as it is important to us that onboarding is -super easy :) - -### Connecting to the PostgreSQL database - -If you need to connect to your development PostgreSQL database, you should -install `psql` locally. The port 5432 is already forwarded, so you can connect with: - -``` -psql -h localhost -U obs -d obs -``` - -The password is `obs` as well. - -## License - - Copyright (C) 2020-2021 OpenBikeSensor Contributors - Contact: https://openbikesensor.org - - The OpenBikeSensor Portal is free software: you can redistribute it - and/or modify it under the terms of the GNU Lesser General Public License - as published by the Free Software Foundation, either version 3 of the - License, or (at your option) any later version. - - The OpenBikeSensor Portal is distributed in the hope that it will be - useful, but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser - General Public License for more details. - - You should have received a copy of the GNU Lesser General Public License - along with the OpenBikeSensor Portal. If not, see - . - -See also [`COPYING`](./COPYING) and [`COPYING.LESSER`](./COPYING.LESSER). - -The above does not apply to the files listed below, their respective licenses -are included in a file next to each of them, named accordingly: - -* `frontend/src/mapstyles/bright.json` -* `frontend/src/mapstyles/positron.json` - -There are lots of other licenses to consider when using this software, -especially in conjunction with imported data and other tools. Check out the -[Licenses Documentation](docs/licenses.md) for an (unofficial) overview of the -license landscape surrounding this project. diff --git a/api/.gitignore b/api/.gitignore index 0baee91c..691c06cf 100644 --- a/api/.gitignore +++ b/api/.gitignore @@ -34,6 +34,10 @@ node_modules # Optional REPL history .node_repl_history +# Python bytecode +__pycache__/ + +# JetBrains IDE .idea # Storage place for local files, such as developer database etc. diff --git a/api/config.dev.py b/api/config.dev.py index e146973a..9b568008 100644 --- a/api/config.dev.py +++ b/api/config.dev.py @@ -5,7 +5,7 @@ AUTO_RESTART = True SECRET = "!!!!!!!!!!!!CHANGE ME!!!!!!!!!!!!" LEAN_MODE = False -POSTGRES_URL = "postgresql+asyncpg://obs:obs@postgres/obs" +POSTGRES_URL = "postgresql+asyncpg://keycloak:password@postgres/postgres" POSTGRES_POOL_SIZE = 20 POSTGRES_MAX_OVERFLOW = 2 * POSTGRES_POOL_SIZE KEYCLOAK_URL = "http://keycloak:8080/auth/realms/obs-dev/" diff --git a/docker-compose.yaml b/docker-compose.yaml index ac68236d..91458065 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -10,13 +10,18 @@ services: postgres: image: "openmaptiles/postgis:6.0" environment: - POSTGRES_USER: obs - POSTGRES_PASSWORD: obs - POSTGRES_DB: obs + POSTGRES_USER: keycloak + POSTGRES_PASSWORD: password + POSTGRES_DB: postgres ports: - '5432:5432' volumes: - - ./local/postgres/data:/var/lib/postgresql/data + - ./local/postgres/data:/var/lib/postgresql/data + healthcheck: + test: /usr/bin/pg_isready --username=keycloak --dbname=postgres --quiet --timeout=0 + interval: 5s + timeout: 10s + retries: 120 api: image: openbikesensor-api @@ -36,8 +41,10 @@ services: - ./api/migrations:/opt/obs/api/migrations - ./api/alembic.ini:/opt/obs/api/alembic.ini depends_on: - - postgres - - keycloak + postgres: + condition: service_healthy + keycloak: + condition: service_started ports: - '3000:3000' restart: on-failure @@ -57,8 +64,10 @@ services: - ./api/config.overrides.py:/opt/obs/api/config.overrides.py - ./local/api-data:/data depends_on: - - postgres - - keycloak + postgres: + condition: service_healthy + keycloak: + condition: service_started restart: on-failure entrypoint: - python @@ -87,17 +96,21 @@ services: - start keycloak: - image: jboss/keycloak + image: quay.io/keycloak/keycloak:21.0 ports: - - 3003:8080 + - '3003:8080' depends_on: - - postgres + postgres: + condition: service_healthy environment: - KEYCLOAK_USER: admin - KEYCLOAK_PASSWORD: admin - KEYCLOAK_FRONTEND_URL: http://localhost:3003/auth/ - DB_VENDOR: postgres - DB_ADDR: postgres - DB_DATABASE: obs - DB_USER: obs - DB_PASSWORD: obs + KC_HOSTNAME: localhost + KC_DB_URL: jdbc:postgresql://postgres:5432/postgres + KC_DB: postgres + KC_DB_USERNAME: keycloak + KC_DB_PASSWORD: password + KEYCLOAK_ADMIN: admin + KEYCLOAK_ADMIN_PASSWORD: admin + command: + - start-dev + - --http-relative-path + - /auth