From 06ace521e94cb65aab2ea6a459ed94418dcb5a0a Mon Sep 17 00:00:00 2001 From: Erik Taubeneck Date: Sat, 16 Mar 2024 20:07:57 -0700 Subject: [PATCH] Add HTTPS to Sidecar for deployment (#8) * add traefik config and update cli * cannot use env variables in traefik.yaml, use static config env variables instead * must call traefik with sudo to run on these ports * use traefik cli args instead of env for key/cert * move key/cert into dynamic_conf * use different env formatting * move tls config into a file created in cli command * fix bug in dynamic_config * fix bug in dynamic_config * use different env format * move dynamic config into cli * make sure to wrap single quotes around double quotes when needed * remove single/double quotes, add backticks * fix ports * use adjacent subdomains, not nested * add draft-mpc.vercel.app to CORS domains * make test data directory before generating it * adjust ports, not inferred from network.toml * use https not ws for checking status * use http not https for checking status * turn off verify for status check temporarily * use https for status check * turn off verification for terminate posts * use https for terminate posts * fix traefik bug * remove tls from helper traefik config * readd tls from helper traefik config * try a different approach to not using tls for helpers * local traefik working. helpers still not working with domains * server updates, use localhost for ipa connections * remove unneeded helper_domain from cli * Update README.md formating * use sidecar0 instead of sidecar-coordinator * removed signed call to /stop. needs to be handled differently * add multi-threading to compile features for IPA * add a step to generate the MPC steps file * typo * fix script path * add env option to command * add -m flag to collect_steps * add cwd to subclasses of Command * fix pylint errors * use mkcert CA with httpx * update github.tsx to warn if OCTOKIT_GITHUB_API_KEY isn't present * avoid race condition with getting a query that may be being created * add -f to git checkout command, as producing steps.txt causes an overwrite * remove verify=False from httpx requests * update readme for first use of mkcert * refresh IPA self signed local_dev keys * fix bug with test directory not existing, wrap query run in exception handling to call crash on exception * fix pylint and grammer error * update TODO in readme --- .gitignore | 4 + README.md | 133 ++++++++++++++++++++- local_dev/config/coordinator.key | 5 - local_dev/config/h1.key | 6 +- local_dev/config/h1_mk.key | 2 +- local_dev/config/h2.key | 6 +- local_dev/config/h2_mk.key | 2 +- local_dev/config/h3.key | 6 +- local_dev/config/network.toml | 66 +++++----- local_dev/config/pub/h1.pem | 14 +-- local_dev/config/pub/h1_mk.pub | 2 +- local_dev/config/pub/h2.pem | 14 +-- local_dev/config/pub/h2_mk.pub | 2 +- local_dev/config/pub/h3.pem | 16 +-- local_dev/config/pub/h3_mk.pub | 2 +- server/app/auth/callback/route.ts | 9 +- server/app/login/GitHubOAuthComponent.tsx | 6 +- server/app/query/github.tsx | 19 ++- server/app/query/servers.tsx | 21 ++-- server/supabase/config.toml | 10 +- sidecar/app/helpers.py | 39 ++---- sidecar/app/main.py | 4 +- sidecar/app/query/base.py | 38 ++++-- sidecar/app/query/command.py | 12 +- sidecar/app/query/ipa.py | 79 +++++++----- sidecar/app/query/step.py | 2 + sidecar/app/routes/start.py | 2 +- sidecar/app/routes/stop.py | 38 ------ sidecar/app/settings.py | 13 +- sidecar/cli/cli.py | 122 ++++++++++++++++--- sidecar/traefik/dynamic-local/dynamic.yaml | 53 ++++++++ sidecar/traefik/dynamic-local/tls.yaml | 4 + sidecar/traefik/dynamic/dynamic.yaml | 14 +++ sidecar/traefik/dynamic/tls.yaml | 4 + sidecar/traefik/traefik-local.yaml | 9 ++ sidecar/traefik/traefik.yaml | 9 ++ 36 files changed, 544 insertions(+), 243 deletions(-) delete mode 100644 local_dev/config/coordinator.key create mode 100644 sidecar/traefik/dynamic-local/dynamic.yaml create mode 100644 sidecar/traefik/dynamic-local/tls.yaml create mode 100644 sidecar/traefik/dynamic/dynamic.yaml create mode 100644 sidecar/traefik/dynamic/tls.yaml create mode 100644 sidecar/traefik/traefik-local.yaml create mode 100644 sidecar/traefik/traefik.yaml diff --git a/.gitignore b/.gitignore index 29c14c9..80a13a0 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,7 @@ IGNORE-ME* # local env files .env* + +# local certs +local_dev/config/cert.pem +local_dev/config/key.pem diff --git a/README.md b/README.md index 54ca41e..6ef0c07 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ draft is a project designed to help test [IPA](https://github.com/private-attribution/ipa) at scale. It contains 2 components: 1. draft-server: a web front end and service that starts queries an displays logs from the MPC helper servers -2. draft-sidecar: a sidecar back end API that runs next to the IPA binary on helper servers. this include a CLI for setup and running. +2. draft-sidecar: a sidecar back end API that runs next to the IPA binary on helper servers. This includes a CLI for setup and running. # Get started @@ -94,11 +94,50 @@ In the output, you'll find an `ANON_KEY`. Update the `server/.env` file one more ``` NEXT_PUBLIC_SUPABASE_URL="http://localhost:54321" NEXT_PUBLIC_SUPABASE_ANON_KEY="" -NEXT_PUBLIC_SITE_URL=http://localhost:3000 +NEXT_PUBLIC_SITE_URL="https://draft.test" SUPABASE_AUTH_GITHUB_CLIENT_ID="" SUPABASE_AUTH_GITHUB_SECRET="" ``` +**Traefik** + +install traefik + +``` +brew install traefik +``` + +update /etc/hosts with (requires sudo) + +``` +127.0.0.1 draft.test +127.0.0.1 sidecar0.draft.test +127.0.0.1 sidecar1.draft.test +127.0.0.1 sidecar2.draft.test +127.0.0.1 sidecar3.draft.test +``` + +make local certs + +install mkcert with + +``` +brew install mkcert +``` + +make the cert with + +``` +mkcert -cert-file "local_dev/config/cert.pem" -key-file "local_dev/config/key.pem" "draft.test" "*.draft.test" +``` + +If you get a warning about the cert not being installed (i.e., it's the first time you've used mkcert), also run: +``` +mkcert -install +``` + +**Run local dev** + You're now ready to install, run, and develop on `draft`! To start the local development environment: @@ -113,7 +152,7 @@ draft start-local-dev If needed, clone this repo: ``` -git clone https://github.com/eriktaubeneck/draft.git +git clone https://github.com/private-attribution/draft.git cd draft ``` @@ -124,6 +163,92 @@ source .venv/bin/activate pip install --editable . ``` +### IPA specific certs + +We check in self signed certs that are only for local development (and are not secure! They are in a public repo!) + +They will periodically expire. You can regenerate them with a compiled helper binary: + +``` +target/release/helper keygen --name localhost --tls-key local_dev/config/h1.key --tls-cert local_dev/config/pub/h1.pem --mk-public-key local_dev/config/pub/h1_mk.pub --mk-private-key local_dev/config/h1_mk.key +target/release/helper keygen --name localhost --tls-key local_dev/config/h2.key --tls-cert local_dev/config/pub/h2.pem --mk-public-key local_dev/config/pub/h2_mk.pub --mk-private-key local_dev/config/h2_mk.key +target/release/helper keygen --name localhost --tls-key local_dev/config/h3.key --tls-cert local_dev/config/pub/h3.pem --mk-public-key local_dev/config/pub/h3_mk.pub --mk-private-key local_dev/config/h3_mk.key +``` + +The public content will also need to be pasted into `local_dev/config/network.toml` for each helper. + +## Deployment + +### Requirements + +*Instructions for AWS Linux 2023* + +1. **Python3.11**: Install with `sudo yum install python3.11` +2. **git**: Install with `sudo yum install git` +3. **draft** (this package): + 1. Clone with `git clone https://github.com/private-attribution/draft.git` + 2. Enter directory `cd draft`. + 3. Create virtualenv: `python3.11 -m venv .venv` + 4. Use virtualeenv: `source .venv/bin/activate` + 5. Upgrade pip: `pip install --upgrade pip` + 6. Install: `pip install --editable .` +4. **traefik**: + 1. Download version 2.11: `wget https://github.com/traefik/traefik/releases/download/v2.11.0/traefik_v2.11.0_linux_amd64.tar.gz` + 2. Validate checksum: `sha256sum traefik_v2.11.0_linux_amd64.tar.gz` should print `7f31f1cc566bd094f038579fc36e354fd545cf899523eb507c3cfcbbdb8b9552 traefik_v2.11.0_linux_amd64.tar.gz` + 3. Extract the binary: `tar -zxvf traefik_v2.11.0_linux_amd64.tar.gz` +5. **tmux**: `sudo yum install tmux` + + +### Generating TLS certs with Let's Encrypt + +You will need a domain name and TLS certificates for the sidecar to properly run over HTTPS. The following instructions assume your domain is `example.com`, please replace with the domain you'd like to use. You will need to create two sub-domains, `sidecar.example.com` and `helper.example.com`. (Note, you could also use a sub-domain as your base domain, e.g., `test.example.com` with two sub-domains of that: `sidecar.test.example.com` and `helper.test.example.com`.) + +1. Set up DNS records for `sidecar.example.com` and `helper.example.com` pointing to a server you control. +2. Make sure you've installed the requirements above, and are using the virtual environment. +3. Install `certbot`: `pip install certbot` +4. `sudo .venv/bin/certbot certonly --standalone -m cert-renewal@example.com -d "sidecar.example.com,helper.example.com"` + 1. Note that you must point directly to `.venv/bin/certbot` as `sudo` does not operate in the virtualenv. +5. Accept the [Let's Encrypt terms](https://letsencrypt.org/documents/LE-SA-v1.3-September-21-2022.pdf). + + +### Make Configuration + +For this stage, you'll need to know a few things about the other parties involved: +1. Their root domain +2. Their public keys +3. Everyone's *identity* (e.g., 0, 1, 2, 3) + + +One you know these: +1. Make a config directory `mkdir config` +2. Copy the default network config: `cp local_dev/config/network.toml config/.` +3. Update that file. + 1. Replace `helper0.draft.test` and `sidecar0.draft.test` with the respective domains for party with identity=0. + 2. Repeat for identity= 1, 2, and 3. + 3. Replace respective certificates with their public keys. +4. Move your Let's Encrypt key and cert into place: `sudo ln -s /etc/letsencrypt/live/sidecar.example.com/fullchain.pem config/cert.pem` and `sudo ln -s /etc/letsencrypt/live/sidecar.example.com/privkey.pem key.pem` +5. Generate IPA specific keys: + 1. Compile `ipa` with `cargo build --bin helper --features="web-app real-world-infra compact-gate stall-detection multi-threading" --no-default-features --release` + 2. Make the keys with `target/release/helper keygen --name localhost --tls-key h1.key --tls-cert h1.pem --mk-public-key h1_mk.pub --mk-private-key h1_mk.key` (replace h1 with for each helper) + 3. Add the public keys content into `network.toml` + 4. Add the public keys to `config/pub` (all helpers need all helper public keys). + 4. For each helper, put their private keys in `config`. + + +### Run draft + +You'll want this to continue to run, even if you disconnect from the host, so it's a good idea to start a tmux session: + +``` +tmux new -s draft-session +``` + +``` +draft start-helper-sidecar --identity --root_domain example.com --config_path config +``` + + + -## Credit +# Credit [Beer tap icons created by wanicon - Flaticon]("https://www.flaticon.com/free-icons/beer-tap") diff --git a/local_dev/config/coordinator.key b/local_dev/config/coordinator.key deleted file mode 100644 index d1763a6..0000000 --- a/local_dev/config/coordinator.key +++ /dev/null @@ -1,5 +0,0 @@ ------BEGIN PRIVATE KEY----- -MIGHAgEAMBMGByqGSM49AgEGCCqGSM49AwEHBG0wawIBAQQgV3QSqe+FY9sGFVlW -gwgMlyvQfNhOBfAqkjlW0HBkA4+hRANCAASP4CHXZ0iCuuWu0Cl3cczy8kpztvc2 -thbitqVR4o7G1rbifsTu+Iva9FD7wTWodq3pzvMAsTI82QrGoB6cbELH ------END PRIVATE KEY----- diff --git a/local_dev/config/h1.key b/local_dev/config/h1.key index 0bacad4..f5f85a9 100644 --- a/local_dev/config/h1.key +++ b/local_dev/config/h1.key @@ -1,5 +1,5 @@ -----BEGIN PRIVATE KEY----- -MIGHAgEAMBMGByqGSM49AgEGCCqGSM49AwEHBG0wawIBAQQgnHorOkrEJyf6LDPr -0ILtQgWgXL49FX7ceoAouLg3wRuhRANCAAT61P9K+vLXu+dWdjoqKGatzasipb0g -gLqOHg5OwazEaneNExmv0xLmg25xuwL7eD+EYfq9AXgixs6vODgSPihr +MIGHAgEAMBMGByqGSM49AgEGCCqGSM49AwEHBG0wawIBAQQgNXRbeh+/oz5xv2yY +uSR0EPFuRratsNNVf9BzoBthCZyhRANCAASa4rehLdFG8wIcRyHg04c8Sj7XGHx9 +hwa65bmXgEEsoNph/7uFVdZIgKswWXX/IQU7UTznqWD8WpXGGnbkj+Eo -----END PRIVATE KEY----- diff --git a/local_dev/config/h1_mk.key b/local_dev/config/h1_mk.key index f2f1b03..10d9928 100644 --- a/local_dev/config/h1_mk.key +++ b/local_dev/config/h1_mk.key @@ -1 +1 @@ -e07c9bb08f1a70c8a65c82fd70d8cc6421eb52a400690643a45c9d793803d288 \ No newline at end of file +9e98e12742ca6a1b6f7543b6fbe1e40f6ed946bcfaf94eabd8701b2c21c92773 \ No newline at end of file diff --git a/local_dev/config/h2.key b/local_dev/config/h2.key index 83eb5e6..b4f0d21 100644 --- a/local_dev/config/h2.key +++ b/local_dev/config/h2.key @@ -1,5 +1,5 @@ -----BEGIN PRIVATE KEY----- -MIGHAgEAMBMGByqGSM49AgEGCCqGSM49AwEHBG0wawIBAQQgqzg9oloWvOdlsnLP -tNtCTbsiN3/9VC3BIjw5jNlXNn6hRANCAATiidzqYon1ecMzoy+gW1ZflyljEVfh -h0wANWdGQQXJQ8mJqo6RQGgZ95JGPO5cHRIimFZFqS51T5m55VRACVkt +MIGHAgEAMBMGByqGSM49AgEGCCqGSM49AwEHBG0wawIBAQQgpj/IkcF3+K5LvYO3 +rFBGXRLQcIDTITGMwrzH48IwdFqhRANCAARE1EvqrvduIUxdaPPxYgVi68qA8uWH +vZVNkjywpBcfQ4vevAov5KWzYkR0aCDI82IziNSv1T5PQipvGHmGBwu5 -----END PRIVATE KEY----- diff --git a/local_dev/config/h2_mk.key b/local_dev/config/h2_mk.key index 2001ddc..3ce735c 100644 --- a/local_dev/config/h2_mk.key +++ b/local_dev/config/h2_mk.key @@ -1 +1 @@ -17f3ee5d2f28aea3f6ad8c9a4ab448315c0447f85815d6d57599ed559315f353 \ No newline at end of file +baa3cc11f2cfe092eb86acabc028889438735fc667d0c88214185f8802b316a7 \ No newline at end of file diff --git a/local_dev/config/h3.key b/local_dev/config/h3.key index 8abbfcb..c765c43 100644 --- a/local_dev/config/h3.key +++ b/local_dev/config/h3.key @@ -1,5 +1,5 @@ -----BEGIN PRIVATE KEY----- -MIGHAgEAMBMGByqGSM49AgEGCCqGSM49AwEHBG0wawIBAQQgjmstRdmbAtf3JqVq -0u37RmTGbMn9+wrh0gijKuOKN/uhRANCAATpFEd+whG8LYmTVDpGsFQ5dy8wjIdL -WyCrCjcKYiJIHih+boD4NeDJB96e51M3nUt9/akdgXvXr5S5qZhA3GwK +MIGHAgEAMBMGByqGSM49AgEGCCqGSM49AwEHBG0wawIBAQQgPEB0qkkpZ1sp44NS +Ogbiz1MLgvyO/N3uvXgEhEGtKGahRANCAATLqehLL42VKHNmfZtY2BVehHsQNyGq +fmOrs6V+DXrQ0eKgGF4ad1lrqXxJFMpVg2i1vOm/kq6GlvM6AqqcjuVc -----END PRIVATE KEY----- diff --git a/local_dev/config/network.toml b/local_dev/config/network.toml index 1f7e730..d9e5a08 100644 --- a/local_dev/config/network.toml +++ b/local_dev/config/network.toml @@ -1,18 +1,18 @@ [[peers]] certificate = """ -----BEGIN CERTIFICATE----- -MIIBZTCCAQugAwIBAgIIHerS5sIdRy4wCgYIKoZIzj0EAwIwFDESMBAGA1UEAwwJ -bG9jYWxob3N0MB4XDTIzMTIxNTE5NTE1MFoXDTI0MDMxNTE5NTE1MFowFDESMBAG -A1UEAwwJbG9jYWxob3N0MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAE+tT/Svry -17vnVnY6Kihmrc2rIqW9IIC6jh4OTsGsxGp3jRMZr9MS5oNucbsC+3g/hGH6vQF4 -IsbOrzg4Ej4oa6NHMEUwFAYDVR0RBA0wC4IJbG9jYWxob3N0MA4GA1UdDwEB/wQE +MIIBZTCCAQugAwIBAgIIRxb0DaIIjkkwCgYIKoZIzj0EAwIwFDESMBAGA1UEAwwJ +bG9jYWxob3N0MB4XDTI0MDMxNTAxMTI0M1oXDTI0MDYxNDAxMTI0M1owFDESMBAG +A1UEAwwJbG9jYWxob3N0MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEmuK3oS3R +RvMCHEch4NOHPEo+1xh8fYcGuuW5l4BBLKDaYf+7hVXWSICrMFl1/yEFO1E856lg +/FqVxhp25I/hKKNHMEUwFAYDVR0RBA0wC4IJbG9jYWxob3N0MA4GA1UdDwEB/wQE AwICpDAdBgNVHSUEFjAUBggrBgEFBQcDAQYIKwYBBQUHAwIwCgYIKoZIzj0EAwID -SAAwRQIgZyp9ReXpjC+ZVx/rZ8lk5kGgWsvNQhvidpE6EPD8wP4CIQD0hZSIXPEC -N0Gz2XisE0JNL5f0tEyrJf/PwSlnazeMxw== +SAAwRQIgYgv5V5unp9q0WSnuPttA5fNASFLKrvslL+T0BKfLjRoCIQC4B+fmHpqX +GVYq2Y0sGz79X+evTPmyJo7X3ye5DlSDeg== -----END CERTIFICATE----- """ url = "localhost:7431" -sidecar_port = "17431" +sidecar_url = "sidecar1.draft.test" [peers.hpke] public_key = "fde0d0c958db9f49d3f1b49cb6830b867cc810bff9e7d0cbf17c777969f3c23e" @@ -20,18 +20,18 @@ public_key = "fde0d0c958db9f49d3f1b49cb6830b867cc810bff9e7d0cbf17c777969f3c23e" [[peers]] certificate = """ -----BEGIN CERTIFICATE----- -MIIBZDCCAQugAwIBAgIIVcv1NVaCs0swCgYIKoZIzj0EAwIwFDESMBAGA1UEAwwJ -bG9jYWxob3N0MB4XDTIzMTIxNTE5NTE1MFoXDTI0MDMxNTE5NTE1MFowFDESMBAG -A1UEAwwJbG9jYWxob3N0MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAE4onc6mKJ -9XnDM6MvoFtWX5cpYxFX4YdMADVnRkEFyUPJiaqOkUBoGfeSRjzuXB0SIphWRaku -dU+ZueVUQAlZLaNHMEUwFAYDVR0RBA0wC4IJbG9jYWxob3N0MA4GA1UdDwEB/wQE +MIIBZDCCAQugAwIBAgIIIHqS6JxF2+AwCgYIKoZIzj0EAwIwFDESMBAGA1UEAwwJ +bG9jYWxob3N0MB4XDTI0MDMxNTAxMTMyMVoXDTI0MDYxNDAxMTMyMVowFDESMBAG +A1UEAwwJbG9jYWxob3N0MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAERNRL6q73 +biFMXWjz8WIFYuvKgPLlh72VTZI8sKQXH0OL3rwKL+Sls2JEdGggyPNiM4jUr9U+ +T0Iqbxh5hgcLuaNHMEUwFAYDVR0RBA0wC4IJbG9jYWxob3N0MA4GA1UdDwEB/wQE AwICpDAdBgNVHSUEFjAUBggrBgEFBQcDAQYIKwYBBQUHAwIwCgYIKoZIzj0EAwID -RwAwRAIgaX95X9bgeZHgbTCl73N2j61AnljyS8DXQ7mWb6fsQXECIFgvumh8TASD -9ylYODUrZag+pK4GCiq2UdjUVMuq/8l1 +RwAwRAIgUBVQLsrbhfoLfg6a2ATU+ulhYmFNvweQ/Xj1M9QgXaECIEbsLs0h4TRG +loU+/Eo4LOm5CkEd8fPOuSdZTp1s8IGT -----END CERTIFICATE----- """ url = "localhost:7432" -sidecar_port = "17432" +sidecar_url = "sidecar2.draft.test" [peers.hpke] public_key = "4e8f1cd4114a8ee8adc58a33050782e2f8ded3336a9c65725f35998e765c4e2d" @@ -39,18 +39,18 @@ public_key = "4e8f1cd4114a8ee8adc58a33050782e2f8ded3336a9c65725f35998e765c4e2d" [[peers]] certificate = """ -----BEGIN CERTIFICATE----- -MIIBZTCCAQugAwIBAgIITHy0LezBdSAwCgYIKoZIzj0EAwIwFDESMBAGA1UEAwwJ -bG9jYWxob3N0MB4XDTIzMTIxNTE5NTE1MFoXDTI0MDMxNTE5NTE1MFowFDESMBAG -A1UEAwwJbG9jYWxob3N0MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAE6RRHfsIR -vC2Jk1Q6RrBUOXcvMIyHS1sgqwo3CmIiSB4ofm6A+DXgyQfenudTN51Lff2pHYF7 -16+UuamYQNxsCqNHMEUwFAYDVR0RBA0wC4IJbG9jYWxob3N0MA4GA1UdDwEB/wQE -AwICpDAdBgNVHSUEFjAUBggrBgEFBQcDAQYIKwYBBQUHAwIwCgYIKoZIzj0EAwID -SAAwRQIgDKTyh8M5hbF1k0o5tAFMFd3NeSekm1P4fb6u+jH9LxcCIQDTIPObVtwc -B6Bgc2gw5JC/G6ahPglwIkjO2ew02/ax6g== +MIIBYzCCAQqgAwIBAgIHYwBqW8VtbjAKBggqhkjOPQQDAjAUMRIwEAYDVQQDDAls +b2NhbGhvc3QwHhcNMjQwMzE1MDExMzUyWhcNMjQwNjE0MDExMzUyWjAUMRIwEAYD +VQQDDAlsb2NhbGhvc3QwWTATBgcqhkjOPQIBBggqhkjOPQMBBwNCAATLqehLL42V +KHNmfZtY2BVehHsQNyGqfmOrs6V+DXrQ0eKgGF4ad1lrqXxJFMpVg2i1vOm/kq6G +lvM6AqqcjuVco0cwRTAUBgNVHREEDTALgglsb2NhbGhvc3QwDgYDVR0PAQH/BAQD +AgKkMB0GA1UdJQQWMBQGCCsGAQUFBwMBBggrBgEFBQcDAjAKBggqhkjOPQQDAgNH +ADBEAiAfszb6imTolbufxqBhMd5gmCRmdxLWVDYCCF3wpa0bLQIgVDzc0X3eqN5U +Ghgnqau5gaGAljARRWQNo8WVu6juWjs= -----END CERTIFICATE----- """ url = "localhost:7433" -sidecar_port = "17433" +sidecar_url = "sidecar3.draft.test" [peers.hpke] public_key = "ebedcfa02354a1d17aed80b0ed55028d0616152d5f8971291e030231dc92063d" @@ -61,14 +61,16 @@ version = "http2" [coordinator] url = "localhost:7430" -sidecar_port = "17430" +sidecar_url = "sidecar0.draft.test" certificate = """ -----BEGIN CERTIFICATE----- -MIIBHDCBwqADAgECAghMfLQt7MF1IDAKBggqhkjOPQQDAjAUMRIwEAYDVQQDDAls -b2NhbGhvc3QwHhcNMjMxMjE1MTk1MTUwWhcNMjQwMzE1MTk1MTUwWjAUMRIwEAYD -VQQDDAlsb2NhbGhvc3QwWTATBgcqhkjOPQIBBggqhkjOPQMBBwNCAASP4CHXZ0iC -uuWu0Cl3cczy8kpztvc2thbitqVR4o7G1rbifsTu+Iva9FD7wTWodq3pzvMAsTI8 -2QrGoB6cbELHMAoGCCqGSM49BAMCA0kAMEYCIQDavRFEtYwIR0lFZ0aZz0Pw4ZuJ -3AOJm90MaoL/Qwd0TAIhAM975+pAXYOZaXJNG3nhPKnXZRtcWLnNO3gXaMg9k6h0 +MIIBZDCCAQugAwIBAgIIechkxwTdoxUwCgYIKoZIzj0EAwIwFDESMBAGA1UEAwwJ +bG9jYWxob3N0MB4XDTI0MDMxNTAxMTQzNloXDTI0MDYxNDAxMTQzNlowFDESMBAG +A1UEAwwJbG9jYWxob3N0MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAECDd9DJ5Y +9+8CGmIxSmFqhL3geQrGBoNwzgz9ohidaFVdh9tzG1X4PdqegHp4KsyIZPjPEewG +OnIeuQGl0FllcaNHMEUwFAYDVR0RBA0wC4IJbG9jYWxob3N0MA4GA1UdDwEB/wQE +AwICpDAdBgNVHSUEFjAUBggrBgEFBQcDAQYIKwYBBQUHAwIwCgYIKoZIzj0EAwID +RwAwRAIgcwd2+uARHJA6GCp5FbUAVJDweRx6cC8QSjjY+aqxr2oCIEhSf5o5lbvj +01Um1sTpQaLMd0qRgsYs5mINziRFZBap -----END CERTIFICATE----- """ diff --git a/local_dev/config/pub/h1.pem b/local_dev/config/pub/h1.pem index 4a315ee..34e7c18 100644 --- a/local_dev/config/pub/h1.pem +++ b/local_dev/config/pub/h1.pem @@ -1,10 +1,10 @@ -----BEGIN CERTIFICATE----- -MIIBZTCCAQugAwIBAgIIHerS5sIdRy4wCgYIKoZIzj0EAwIwFDESMBAGA1UEAwwJ -bG9jYWxob3N0MB4XDTIzMTIxNTE5NTE1MFoXDTI0MDMxNTE5NTE1MFowFDESMBAG -A1UEAwwJbG9jYWxob3N0MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAE+tT/Svry -17vnVnY6Kihmrc2rIqW9IIC6jh4OTsGsxGp3jRMZr9MS5oNucbsC+3g/hGH6vQF4 -IsbOrzg4Ej4oa6NHMEUwFAYDVR0RBA0wC4IJbG9jYWxob3N0MA4GA1UdDwEB/wQE +MIIBZTCCAQugAwIBAgIIRxb0DaIIjkkwCgYIKoZIzj0EAwIwFDESMBAGA1UEAwwJ +bG9jYWxob3N0MB4XDTI0MDMxNTAxMTI0M1oXDTI0MDYxNDAxMTI0M1owFDESMBAG +A1UEAwwJbG9jYWxob3N0MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEmuK3oS3R +RvMCHEch4NOHPEo+1xh8fYcGuuW5l4BBLKDaYf+7hVXWSICrMFl1/yEFO1E856lg +/FqVxhp25I/hKKNHMEUwFAYDVR0RBA0wC4IJbG9jYWxob3N0MA4GA1UdDwEB/wQE AwICpDAdBgNVHSUEFjAUBggrBgEFBQcDAQYIKwYBBQUHAwIwCgYIKoZIzj0EAwID -SAAwRQIgZyp9ReXpjC+ZVx/rZ8lk5kGgWsvNQhvidpE6EPD8wP4CIQD0hZSIXPEC -N0Gz2XisE0JNL5f0tEyrJf/PwSlnazeMxw== +SAAwRQIgYgv5V5unp9q0WSnuPttA5fNASFLKrvslL+T0BKfLjRoCIQC4B+fmHpqX +GVYq2Y0sGz79X+evTPmyJo7X3ye5DlSDeg== -----END CERTIFICATE----- diff --git a/local_dev/config/pub/h1_mk.pub b/local_dev/config/pub/h1_mk.pub index 9b00684..23ed86d 100644 --- a/local_dev/config/pub/h1_mk.pub +++ b/local_dev/config/pub/h1_mk.pub @@ -1 +1 @@ -fde0d0c958db9f49d3f1b49cb6830b867cc810bff9e7d0cbf17c777969f3c23e \ No newline at end of file +008eb82d82def11d250243bc06d96637e9fa73e362de92ae729b6a599cc15b5c \ No newline at end of file diff --git a/local_dev/config/pub/h2.pem b/local_dev/config/pub/h2.pem index ce6e3c2..0cdc58c 100644 --- a/local_dev/config/pub/h2.pem +++ b/local_dev/config/pub/h2.pem @@ -1,10 +1,10 @@ -----BEGIN CERTIFICATE----- -MIIBZDCCAQugAwIBAgIIVcv1NVaCs0swCgYIKoZIzj0EAwIwFDESMBAGA1UEAwwJ -bG9jYWxob3N0MB4XDTIzMTIxNTE5NTE1MFoXDTI0MDMxNTE5NTE1MFowFDESMBAG -A1UEAwwJbG9jYWxob3N0MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAE4onc6mKJ -9XnDM6MvoFtWX5cpYxFX4YdMADVnRkEFyUPJiaqOkUBoGfeSRjzuXB0SIphWRaku -dU+ZueVUQAlZLaNHMEUwFAYDVR0RBA0wC4IJbG9jYWxob3N0MA4GA1UdDwEB/wQE +MIIBZDCCAQugAwIBAgIIIHqS6JxF2+AwCgYIKoZIzj0EAwIwFDESMBAGA1UEAwwJ +bG9jYWxob3N0MB4XDTI0MDMxNTAxMTMyMVoXDTI0MDYxNDAxMTMyMVowFDESMBAG +A1UEAwwJbG9jYWxob3N0MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAERNRL6q73 +biFMXWjz8WIFYuvKgPLlh72VTZI8sKQXH0OL3rwKL+Sls2JEdGggyPNiM4jUr9U+ +T0Iqbxh5hgcLuaNHMEUwFAYDVR0RBA0wC4IJbG9jYWxob3N0MA4GA1UdDwEB/wQE AwICpDAdBgNVHSUEFjAUBggrBgEFBQcDAQYIKwYBBQUHAwIwCgYIKoZIzj0EAwID -RwAwRAIgaX95X9bgeZHgbTCl73N2j61AnljyS8DXQ7mWb6fsQXECIFgvumh8TASD -9ylYODUrZag+pK4GCiq2UdjUVMuq/8l1 +RwAwRAIgUBVQLsrbhfoLfg6a2ATU+ulhYmFNvweQ/Xj1M9QgXaECIEbsLs0h4TRG +loU+/Eo4LOm5CkEd8fPOuSdZTp1s8IGT -----END CERTIFICATE----- diff --git a/local_dev/config/pub/h2_mk.pub b/local_dev/config/pub/h2_mk.pub index 48fe500..259093d 100644 --- a/local_dev/config/pub/h2_mk.pub +++ b/local_dev/config/pub/h2_mk.pub @@ -1 +1 @@ -4e8f1cd4114a8ee8adc58a33050782e2f8ded3336a9c65725f35998e765c4e2d \ No newline at end of file +d7cdae88176fd5ee2bef524b776a15fc52e4b9c3f986d34fe815c7463e7a425b \ No newline at end of file diff --git a/local_dev/config/pub/h3.pem b/local_dev/config/pub/h3.pem index 13fe584..d25c5d7 100644 --- a/local_dev/config/pub/h3.pem +++ b/local_dev/config/pub/h3.pem @@ -1,10 +1,10 @@ -----BEGIN CERTIFICATE----- -MIIBZTCCAQugAwIBAgIITHy0LezBdSAwCgYIKoZIzj0EAwIwFDESMBAGA1UEAwwJ -bG9jYWxob3N0MB4XDTIzMTIxNTE5NTE1MFoXDTI0MDMxNTE5NTE1MFowFDESMBAG -A1UEAwwJbG9jYWxob3N0MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAE6RRHfsIR -vC2Jk1Q6RrBUOXcvMIyHS1sgqwo3CmIiSB4ofm6A+DXgyQfenudTN51Lff2pHYF7 -16+UuamYQNxsCqNHMEUwFAYDVR0RBA0wC4IJbG9jYWxob3N0MA4GA1UdDwEB/wQE -AwICpDAdBgNVHSUEFjAUBggrBgEFBQcDAQYIKwYBBQUHAwIwCgYIKoZIzj0EAwID -SAAwRQIgDKTyh8M5hbF1k0o5tAFMFd3NeSekm1P4fb6u+jH9LxcCIQDTIPObVtwc -B6Bgc2gw5JC/G6ahPglwIkjO2ew02/ax6g== +MIIBYzCCAQqgAwIBAgIHYwBqW8VtbjAKBggqhkjOPQQDAjAUMRIwEAYDVQQDDAls +b2NhbGhvc3QwHhcNMjQwMzE1MDExMzUyWhcNMjQwNjE0MDExMzUyWjAUMRIwEAYD +VQQDDAlsb2NhbGhvc3QwWTATBgcqhkjOPQIBBggqhkjOPQMBBwNCAATLqehLL42V +KHNmfZtY2BVehHsQNyGqfmOrs6V+DXrQ0eKgGF4ad1lrqXxJFMpVg2i1vOm/kq6G +lvM6AqqcjuVco0cwRTAUBgNVHREEDTALgglsb2NhbGhvc3QwDgYDVR0PAQH/BAQD +AgKkMB0GA1UdJQQWMBQGCCsGAQUFBwMBBggrBgEFBQcDAjAKBggqhkjOPQQDAgNH +ADBEAiAfszb6imTolbufxqBhMd5gmCRmdxLWVDYCCF3wpa0bLQIgVDzc0X3eqN5U +Ghgnqau5gaGAljARRWQNo8WVu6juWjs= -----END CERTIFICATE----- diff --git a/local_dev/config/pub/h3_mk.pub b/local_dev/config/pub/h3_mk.pub index 4f594b5..b5e90a0 100644 --- a/local_dev/config/pub/h3_mk.pub +++ b/local_dev/config/pub/h3_mk.pub @@ -1 +1 @@ -ebedcfa02354a1d17aed80b0ed55028d0616152d5f8971291e030231dc92063d \ No newline at end of file +db0edf0d4148340a36a286c5dfcc99fe42fcbfb3a4d491fd961730adc4ca5545 \ No newline at end of file diff --git a/server/app/auth/callback/route.ts b/server/app/auth/callback/route.ts index 641d2b2..9dd5daa 100644 --- a/server/app/auth/callback/route.ts +++ b/server/app/auth/callback/route.ts @@ -3,11 +3,18 @@ import { NextResponse } from "next/server"; import { createServerClient, type CookieOptions } from "@supabase/ssr"; export async function GET(request: Request) { - const { searchParams, origin } = new URL(request.url); + const { searchParams } = new URL(request.url); const code = searchParams.get("code"); // if "next" is in param, use it as the redirect URL const next = searchParams.get("next") ?? "/"; + let origin = + process?.env?.NEXT_PUBLIC_SITE_URL ?? // Set this to your site URL in production env. + process?.env?.NEXT_PUBLIC_VERCEL_URL ?? // Automatically set by Vercel. + "https://draft.test/"; + // Make sure to include `https://` + origin = origin.includes("https://") ? origin : `https://${origin}`; + if (code) { const cookieStore = cookies(); const supabase = createServerClient( diff --git a/server/app/login/GitHubOAuthComponent.tsx b/server/app/login/GitHubOAuthComponent.tsx index 3dde4c0..7edc30c 100644 --- a/server/app/login/GitHubOAuthComponent.tsx +++ b/server/app/login/GitHubOAuthComponent.tsx @@ -12,9 +12,9 @@ export default function GitHubOAuthComponent() { let url = process?.env?.NEXT_PUBLIC_SITE_URL ?? // Set this to your site URL in production env. process?.env?.NEXT_PUBLIC_VERCEL_URL ?? // Automatically set by Vercel. - "http://localhost:3000/"; - // Make sure to include `https://` when not localhost. - url = url.includes("http") ? url : `https://${url}`; + "https://draft.test/"; + // Make sure to include `https://` + url = url.includes("https://") ? url : `https://${url}`; // Make sure to include a trailing `/`. url = url.charAt(url.length - 1) === "/" ? url : `${url}/`; diff --git a/server/app/query/github.tsx b/server/app/query/github.tsx index d48fbed..d55de63 100644 --- a/server/app/query/github.tsx +++ b/server/app/query/github.tsx @@ -9,6 +9,13 @@ export interface Branch { commitHash: string; } +// TODO: raise error if api key is expired +if (process.env.OCTOKIT_GITHUB_API_KEY === undefined) { + console.warn( + "WARNING: Octokit requires a personal access token to function properly. Please add OCTOKIT_GITHUB_API_KEY to .env. It does not require any permissions.", + ); +} + export async function Branches(owner: string, repo: string): Promise { const branchesIter = octokit.paginate.iterator( octokit.rest.repos.listBranches, @@ -16,7 +23,7 @@ export async function Branches(owner: string, repo: string): Promise { owner: owner, repo: repo, per_page: 100, - auth: process.env.GITHUB_API_KEY, + auth: process.env.OCTOKIT_GITHUB_API_KEY, }, ); @@ -30,9 +37,11 @@ export async function Branches(owner: string, repo: string): Promise { } } - const mainBranch = branchesArray.find((branch) => branch.name === "main"); - if (mainBranch) { - branchesArray.unshift(mainBranch); + const mainBranchIndex = branchesArray.findIndex( + (branch) => branch.name === "main", + ); + if (mainBranchIndex != -1) { + branchesArray.unshift(branchesArray.splice(mainBranchIndex, 1)[0]); } branchesArray.unshift({ name: "N/A", commitHash: "" }); return branchesArray; @@ -45,7 +54,7 @@ export async function Commits(owner: string, repo: string): Promise { owner: owner, repo: repo, per_page: 100, - auth: process.env.GITHUB_API_KEY, + auth: process.env.OCTOKIT_GITHUB_API_KEY, }, ); diff --git a/server/app/query/servers.tsx b/server/app/query/servers.tsx index ebf3847..d23a043 100644 --- a/server/app/query/servers.tsx +++ b/server/app/query/servers.tsx @@ -87,19 +87,19 @@ export class RemoteServer { logsWebSocketURL(id: string): URL { const webSocketURL = new URL(`/ws/logs/${id}`, this.baseURL); - webSocketURL.protocol = "ws"; + webSocketURL.protocol = "wss"; return webSocketURL; } statusWebSocketURL(id: string): URL { const webSocketURL = new URL(`/ws/status/${id}`, this.baseURL); - webSocketURL.protocol = "ws"; + webSocketURL.protocol = "wss"; return webSocketURL; } statsWebSocketURL(id: string): URL { const webSocketURL = new URL(`/ws/stats/${id}`, this.baseURL); - webSocketURL.protocol = "ws"; + webSocketURL.protocol = "wss"; return webSocketURL; } @@ -271,19 +271,26 @@ export const IPARemoteServers: RemoteServersType = { [RemoteServerNames.Coordinator]: new IPACoordinatorRemoteServer( RemoteServerNames.Coordinator, new URL( - process?.env?.NEXT_PUBLIC_COORDINATOR_URL ?? "http://localhost:17430", + process?.env?.NEXT_PUBLIC_COORDINATOR_URL ?? + "https://sidecar0.draft.test", ), ), [RemoteServerNames.Helper1]: new IPAHelperRemoteServer( RemoteServerNames.Helper1, - new URL(process?.env?.NEXT_PUBLIC_HELPER1_URL ?? "http://localhost:17431"), + new URL( + process?.env?.NEXT_PUBLIC_HELPER1_URL ?? "https://sidecar1.draft.test", + ), ), [RemoteServerNames.Helper2]: new IPAHelperRemoteServer( RemoteServerNames.Helper2, - new URL(process?.env?.NEXT_PUBLIC_HELPER2_URL ?? "http://localhost:17432"), + new URL( + process?.env?.NEXT_PUBLIC_HELPER2_URL ?? "https://sidecar2.draft.test", + ), ), [RemoteServerNames.Helper3]: new IPAHelperRemoteServer( RemoteServerNames.Helper3, - new URL(process?.env?.NEXT_PUBLIC_HELPER3_URL ?? "http://localhost:17433"), + new URL( + process?.env?.NEXT_PUBLIC_HELPER3_URL ?? "https://sidecar3.draft.test", + ), ), }; diff --git a/server/supabase/config.toml b/server/supabase/config.toml index bb13569..9157ecd 100644 --- a/server/supabase/config.toml +++ b/server/supabase/config.toml @@ -40,9 +40,9 @@ file_size_limit = "50MiB" [auth] # The base URL of your website. Used as an allow-list for redirects and for constructing URLs used # in emails. -site_url = "http://localhost:3000" +site_url = "https://draft.test" # A list of *exact* URLs that auth providers are permitted to redirect to post authentication. -additional_redirect_urls = ["https://localhost:3000/auth/callback"] +additional_redirect_urls = ["https://draft.test/auth/callback"] # How long tokens are valid for, in seconds. Defaults to 3600 (1 hour), maximum 604,800 seconds (one # week). jwt_expiry = 3600 @@ -59,11 +59,11 @@ secret = "env(SUPABASE_AUTH_GITHUB_SECRET)" redirect_uri = "http://localhost:54321/auth/v1/callback" [analytics] -enabled = true +enabled = false port = 54327 vector_port = 54328 # Setup BigQuery project to enable log viewer on local development stack. # See: https://supabase.com/docs/guides/getting-started/local-development#enabling-local-logging -gcp_project_id = "" -gcp_project_number = "" +gcp_project_id = "null" +gcp_project_number = "null" gcp_jwt_path = "supabase/gcloud.json" diff --git a/sidecar/app/helpers.py b/sidecar/app/helpers.py index 41cdf94..0686aef 100644 --- a/sidecar/app/helpers.py +++ b/sidecar/app/helpers.py @@ -18,19 +18,10 @@ class Role(IntEnum): @dataclass class Helper: role: Role - hostname: str - sidecar_port: int - helper_port: int + helper_url: ParseResult + sidecar_url: ParseResult public_key: EllipticCurvePublicKey - @property - def sidecar_url(self) -> ParseResult: - return urlparse(f"http://{self.hostname}:{self.sidecar_port}") - - @property - def helper_url(self) -> ParseResult: - return urlparse(f"http://{self.hostname}:{self.helper_port}") - def load_helpers_from_network_config(network_config_path: Path) -> dict[Role, Helper]: with network_config_path.open("rb") as f: @@ -39,30 +30,21 @@ def load_helpers_from_network_config(network_config_path: Path) -> dict[Role, He helper_roles = list(r for r in Role if r != Role.COORDINATOR) helpers = {} for helper_config, role in zip(helper_configs, helper_roles): - url = urlparse(f"http://{helper_config['url']}") - hostname = str(url.hostname) - helper_port = int(url.port or 0) - sidecar_port = int(helper_config.get("sidecar_port", 0)) - if not hostname or not helper_port or not sidecar_port: - raise Exception(f"{network_data=} missing data.") + helper_url = urlparse(f"http://{helper_config['url']}") + sidecar_url = urlparse(f"http://{helper_config['sidecar_url']}") public_key_pem_data = helper_config.get("certificate") cert = load_pem_x509_certificate(public_key_pem_data.encode("utf8")) public_key = cert.public_key() assert isinstance(public_key, EllipticCurvePublicKey) helpers[role] = Helper( role=role, - hostname=hostname, - helper_port=helper_port, - sidecar_port=sidecar_port, + helper_url=helper_url, + sidecar_url=sidecar_url, public_key=public_key, ) - url = urlparse(f"http://{network_data['coordinator']['url']}") - hostname = str(url.hostname) - helper_port = int(url.port or 0) - sidecar_port = int(network_data["coordinator"].get("sidecar_port", 0)) - if not hostname or not helper_port or not sidecar_port: - raise Exception(f"{network_data=} missing data.") + helper_url = urlparse(f"http://{network_data['coordinator']['url']}") + sidecar_url = urlparse(f"http://{network_data['coordinator']['sidecar_url']}") public_key_pem_data = network_data["coordinator"].get("certificate") cert = load_pem_x509_certificate(public_key_pem_data.encode("utf8")) public_key = cert.public_key() @@ -70,9 +52,8 @@ def load_helpers_from_network_config(network_config_path: Path) -> dict[Role, He helpers[Role.COORDINATOR] = Helper( role=Role.COORDINATOR, - hostname=hostname, - helper_port=helper_port, - sidecar_port=sidecar_port, + helper_url=helper_url, + sidecar_url=sidecar_url, public_key=public_key, ) return helpers diff --git a/sidecar/app/main.py b/sidecar/app/main.py index 037f587..7c283ca 100644 --- a/sidecar/app/main.py +++ b/sidecar/app/main.py @@ -8,9 +8,7 @@ app.include_router(start.router) app.include_router(stop.router) -origins = [ - "http://localhost:3000", -] +origins = ["https://draft.test", "https://draft-mpc.vercel.app"] app.add_middleware( CORSMiddleware, diff --git a/sidecar/app/query/base.py b/sidecar/app/query/base.py index 3b3f12d..41d5b5e 100644 --- a/sidecar/app/query/base.py +++ b/sidecar/app/query/base.py @@ -18,6 +18,10 @@ queries: dict[str, "Query"] = {} +class QueryExistsError(Exception): + pass + + @dataclass class Query: # pylint: disable=too-many-instance-attributes @@ -45,7 +49,7 @@ def __post_init__(self): ) self.logger.debug(f"adding new Query {self}.") if queries.get(self.query_id) is not None: - raise Exception(f"{self.query_id} already exists") + raise QueryExistsError(f"{self.query_id} already exists") queries[self.query_id] = self @property @@ -65,7 +69,14 @@ def get_from_query_id(cls, query_id) -> Optional["Query"]: query = queries.get(query_id) if query: return query - query = cls(query_id) + try: + query = cls(query_id) + except QueryExistsError as e: + # avoid race condition on queries + query = queries.get(query_id) + if query: + return query + raise e if query.status_file_path.exists(): with query.status_file_path.open("r") as f: status_str = f.readline() @@ -104,13 +115,22 @@ def steps(self) -> Iterable[Step]: def start(self): self.start_time = time.time() - for step in self.steps: - self.logger.info(f"Starting: {step}") - self.status = step.status - self.current_step = step - step.start() - if not step.success: - self.crash() + try: + for step in self.steps: + if self.finished: + break + self.logger.info(f"Starting: {step}") + self.status = step.status + self.current_step = step + step.start() + if not step.success: + self.crash() + # pylint: disable=broad-exception-caught + except Exception as e: + # intentially crash on any python exception + # as well as command failure + self.logger.error(e) + self.crash() if not self.finished: self.finish() diff --git a/sidecar/app/query/command.py b/sidecar/app/query/command.py index 51cca81..137b23f 100644 --- a/sidecar/app/query/command.py +++ b/sidecar/app/query/command.py @@ -16,6 +16,7 @@ class Command: cmd: str env: Optional[dict] = field(default_factory=lambda: {**os.environ}, repr=False) + cwd: Optional[Path] = field(default=None, repr=True) process: Optional[subprocess.Popen] = field(init=False, default=None, repr=True) @property @@ -65,6 +66,7 @@ def build_process(self): return subprocess.Popen( shlex.split(self.cmd), env=self.env, + cwd=self.cwd, ) def start(self): @@ -89,19 +91,18 @@ class FileOutputCommand(Command): output_file_path: Path output_file: Optional[TextIO] = field(repr=False, init=False) - def __post_init__(self): - # need to manually close in start method - # pylint: disable=consider-using-with - self.output_file = self.output_file_path.open("wb") - def build_process(self): return subprocess.Popen( shlex.split(self.cmd), stdout=self.output_file, env=self.env, + cwd=self.cwd, ) def start(self): + # build_process needs to return, so this needs to be manually closed + # pylint: disable=consider-using-with + self.output_file = self.output_file_path.open("wb") super().start() self.output_file.close() @@ -114,6 +115,7 @@ def build_process(self): return subprocess.Popen( shlex.split(self.cmd), env=self.env, + cwd=self.cwd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, diff --git a/sidecar/app/query/ipa.py b/sidecar/app/query/ipa.py index d81246f..8e3e43a 100644 --- a/sidecar/app/query/ipa.py +++ b/sidecar/app/query/ipa.py @@ -1,16 +1,13 @@ from __future__ import annotations -import base64 import time from dataclasses import dataclass, field from pathlib import Path from typing import ClassVar -from urllib.parse import urljoin, urlunparse +from urllib.parse import urlunparse import httpx import loguru -from cryptography.hazmat.primitives import hashes -from cryptography.hazmat.primitives.asymmetric import ec from ..helpers import Role from ..local_paths import Paths @@ -29,12 +26,13 @@ def send_kill_signals(self): for helper in settings.helpers.values(): if helper.role == self.role: continue - finish_url = urljoin( - helper.sidecar_url.geturl(), f"/stop/kill/{self.query_id}" - ) - r = httpx.post( - finish_url, + finish_url = urlunparse( + helper.sidecar_url._replace( + scheme="https", path=f"/stop/kill/{self.query_id}" + ), ) + + r = httpx.post(finish_url) self.logger.info(f"sent post request: {r.text}") def crash(self): @@ -101,7 +99,7 @@ def build_from_query(cls, query: IPAQuery): def build_command(self) -> LoggerOutputCommand: return LoggerOutputCommand( - cmd=f"git -C {self.repo_path} checkout {self.commit_hash}", + cmd=f"git -C {self.repo_path} checkout -f {self.commit_hash}", logger=self.logger, ) @@ -153,12 +151,36 @@ def build_from_query(cls, query: IPAQuery): def build_command(self) -> LoggerOutputCommand: return LoggerOutputCommand( cmd=f"cargo build --bin helper --manifest-path={self.manifest_path} " - f'--features="web-app real-world-infra compact-gate stall-detection" ' - f"--no-default-features --target-dir={self.target_path} --release", + f'--features="web-app real-world-infra compact-gate stall-detection ' + f'multi-threading" --no-default-features --target-dir={self.target_path} ' + f"--release", logger=self.logger, ) +@dataclass(kw_only=True) +class IPAHelperCollectStepsStep(CommandStep): + repo_path: Path + logger: loguru.Logger = field(repr=False) + status: ClassVar[Status] = Status.COMPILING + + @classmethod + def build_from_query(cls, query: IPAQuery): + repo_path = query.paths.repo_path + return cls( + repo_path=repo_path, + logger=query.logger, + ) + + def build_command(self) -> FileOutputCommand: + output_file_path = self.repo_path / Path("ipa-core/src/protocol/step/steps.txt") + return FileOutputCommand( + cmd="python3 scripts/collect_steps.py -m", + cwd=self.repo_path, + output_file_path=output_file_path, + ) + + @dataclass(kw_only=True) class IPACoordinatorGenerateTestDataStep(CommandStep): output_file_path: Path @@ -168,6 +190,9 @@ class IPACoordinatorGenerateTestDataStep(CommandStep): max_trigger_value: int status: ClassVar[Status] = Status.COMPILING + def pre_run(self): + self.output_file_path.parent.mkdir(parents=True, exist_ok=True) + @classmethod def build_from_query(cls, query: IPACoordinatorQuery): return cls( @@ -199,20 +224,19 @@ def build_from_query(cls, query: IPAQuery): ) def run(self): - helper_urls = [ + sidecar_urls = [ helper.sidecar_url for helper in settings.helpers.values() if helper.role != Role.COORDINATOR ] - for helper_url in helper_urls: + for sidecar_url in sidecar_urls: url = urlunparse( - helper_url._replace( - scheme="ws", path=f"/start/ipa-helper/{self.query_id}/status" + sidecar_url._replace( + scheme="https", path=f"/start/ipa-helper/{self.query_id}/status" ), ) while True: r = httpx.get(url).json() - print(r) status = r.get("status") match status: case Status.IN_PROGRESS.name: @@ -295,26 +319,18 @@ class IPACoordinatorQuery(IPAQuery): IPACoordinatorStartStep, ] - def sign_query_id(self): - return base64.b64encode( - settings.private_key.sign( - self.query_id.encode("utf8"), ec.ECDSA(hashes.SHA256()) - ) - ).decode("utf8") - def send_terminate_signals(self): - signature = self.sign_query_id() self.logger.info("sending terminate signals") for helper in settings.helpers.values(): if helper.role == self.role: continue - finish_url = urljoin( - helper.sidecar_url.geturl(), f"/stop/finish/{self.query_id}" - ) - r = httpx.post( - finish_url, - json={"identity": str(self.role.value), "signature": signature}, + finish_url = urlunparse( + helper.sidecar_url._replace( + scheme="https", path=f"/stop/finish/{self.query_id}" + ), ) + + r = httpx.post(finish_url) self.logger.info(f"sent post request: {finish_url}: {r.text}") def finish(self): @@ -375,5 +391,6 @@ class IPAHelperQuery(IPAQuery): IPAFetchUpstreamStep, IPACheckoutCommitStep, IPAHelperCompileStep, + IPAHelperCollectStepsStep, IPAStartHelperStep, ] diff --git a/sidecar/app/query/step.py b/sidecar/app/query/step.py index 6a949ea..3ff58b9 100644 --- a/sidecar/app/query/step.py +++ b/sidecar/app/query/step.py @@ -80,6 +80,8 @@ def memory_rss_usage(self) -> int: @dataclass(kw_only=True) class CommandStep(Step, ABC): + # pylint: disable=fixme + # TODO : maybe delete env from here # [fixme] env: Optional[dict] = field(default_factory=lambda: {**os.environ}, repr=False) command: Command = field(init=False, repr=True) diff --git a/sidecar/app/routes/start.py b/sidecar/app/routes/start.py index b3ff611..2b42304 100644 --- a/sidecar/app/routes/start.py +++ b/sidecar/app/routes/start.py @@ -53,7 +53,7 @@ def start_ipa_helper( query = IPAHelperQuery( paths=paths, query_id=query_id, - port=settings.helper.helper_port, + port=settings.helper_port, ) background_tasks.add_task(query.start) diff --git a/sidecar/app/routes/stop.py b/sidecar/app/routes/stop.py index 8bffb89..4dde4d8 100644 --- a/sidecar/app/routes/stop.py +++ b/sidecar/app/routes/stop.py @@ -1,16 +1,8 @@ -import base64 - -from cryptography.exceptions import InvalidSignature -from cryptography.hazmat.primitives import hashes -from cryptography.hazmat.primitives.asymmetric import ec from fastapi import APIRouter -from pydantic import BaseModel -from ..helpers import Role from ..logger import logger from ..query.base import Query from ..query.step import Status -from ..settings import settings router = APIRouter( prefix="/stop", @@ -20,40 +12,10 @@ ) -def validate_query_signature( - query_id: str, - identity: int, - signature: bytes, -) -> bool: - helpers = settings.helpers - role = Role(identity) - helper = helpers[role] - try: - helper.public_key.verify( - signature, query_id.encode("utf8"), ec.ECDSA(hashes.SHA256()) - ) - return True - except InvalidSignature: - return False - - -# pyre-ignore: https://pyre-check.org/docs/errors/#dataclass-like-classes -class SignedRequestModel(BaseModel): - identity: int - signature: bytes - - @router.post("/finish/{query_id}") def finish( query_id: str, - data: SignedRequestModel, ): - identity = data.identity - signature = base64.b64decode(data.signature) - logger.info(f"finish called for {query_id=}") - if not validate_query_signature(query_id, identity, signature): - logger.warning("signature invalid") - return {"message": "Invalid signature"} query = Query.get_from_query_id(query_id) if query is None: return {"message": "Query not found", "query_id": query_id} diff --git a/sidecar/app/settings.py b/sidecar/app/settings.py index c5acf8f..0035412 100644 --- a/sidecar/app/settings.py +++ b/sidecar/app/settings.py @@ -1,8 +1,6 @@ from pathlib import Path from typing import Annotated, Any -from cryptography.hazmat.primitives.asymmetric.ec import EllipticCurvePrivateKey -from cryptography.hazmat.primitives.serialization import load_pem_private_key from pydantic.functional_validators import BeforeValidator from pydantic_settings import BaseSettings @@ -18,17 +16,12 @@ class Settings(BaseSettings): root_path: Annotated[Path, BeforeValidator(gen_path)] config_path: Annotated[Path, BeforeValidator(gen_path)] network_config_path: Annotated[Path, BeforeValidator(gen_path)] - private_key_pem_path: Annotated[Path, BeforeValidator(gen_path)] role: Role + helper_port: int _helpers: dict[Role, Helper] - _private_key: EllipticCurvePrivateKey def model_post_init(self, __context) -> None: self._helpers = load_helpers_from_network_config(self.network_config_path) - with self.private_key_pem_path.open("rb") as f: - _private_key = load_pem_private_key(f.read(), None) - assert isinstance(_private_key, EllipticCurvePrivateKey) - self._private_key = _private_key @property def helper(self): @@ -38,9 +31,5 @@ def helper(self): def helpers(self): return self._helpers - @property - def private_key(self): - return self._private_key - settings = Settings() diff --git a/sidecar/cli/cli.py b/sidecar/cli/cli.py index de13f8e..7ef2c4a 100644 --- a/sidecar/cli/cli.py +++ b/sidecar/cli/cli.py @@ -1,4 +1,6 @@ import os +import shlex +import subprocess from pathlib import Path from typing import Optional @@ -6,7 +8,7 @@ import click_pathlib from ..app.command import Command, start_commands_parallel -from ..app.helpers import Role, load_helpers_from_network_config +from ..app.helpers import Role @click.group() @@ -17,32 +19,76 @@ def cli(): def start_helper_sidecar_command( config_path: Path, identity: int, + helper_port: int, + sidecar_port: int, root_path: Optional[Path] = None, + _env: Optional[dict[str, str]] = None, ): role = Role(int(identity)) network_config = config_path / Path("network.toml") root_path = root_path or Path(f"tmp/sidecar/{role.value}") root_path.mkdir(parents=True, exist_ok=True) - helpers = load_helpers_from_network_config(network_config) if role == Role.COORDINATOR: private_key_pem_path = config_path / Path("coordinator.key") else: private_key_pem_path = config_path / Path(f"h{role.value}.key") - helper = helpers[role] cmd = "uvicorn sidecar.app.main:app" + if _env is None: + _env = {} env = { **os.environ, + **_env, "ROLE": str(role.value), "ROOT_PATH": root_path, "CONFIG_PATH": config_path, "NETWORK_CONFIG_PATH": network_config, "PRIVATE_KEY_PEM_PATH": private_key_pem_path, - "UVICORN_PORT": str(helper.sidecar_port), + "HELPER_PORT": str(helper_port), + "UVICORN_PORT": str(sidecar_port), "UVICORN_HOST": "0.0.0.0", } return Command(cmd=cmd, env=env) +def start_traefik_command( + config_path: Path, + sidecar_port: int, + root_domain: str, + sidecar_domain: str, +): + sidecar_domain = sidecar_domain or f"sidecar.{root_domain}" + env = { + **os.environ, + "SIDECAR_DOMAIN": sidecar_domain, + "SIDECAR_PORT": str(sidecar_port), + "CERT_DIR": config_path, + } + cmd = "sudo -E ./traefik --configFile=sidecar/traefik/traefik.yaml" + return Command(cmd=cmd, env=env) + + +def start_traefik_local_command( + config_path: Path, + sidecar_ports: tuple[int, ...], + server_port: int, + root_domain: str, +): + env = { + **os.environ, + "CERT_DIR": config_path, + "SERVER_DOMAIN": root_domain, + "SERVER_PORT": str(server_port), + } + for identity, s_port in enumerate(sidecar_ports): + sidecar_domain = f"sidecar{identity}.{root_domain}" + env[f"SIDECAR_{identity}_DOMAIN"] = sidecar_domain + env[f"SIDECAR_{identity}_PORT"] = str(s_port) + + cmd = "traefik --configFile=sidecar/traefik/traefik-local.yaml" + return Command(cmd=cmd, env=env) + + +# pylint: disable=too-many-arguments @cli.command @click.option( "--config_path", @@ -51,18 +97,34 @@ def start_helper_sidecar_command( show_default=True, ) @click.option("--root_path", type=click_pathlib.Path(), default=None) +@click.option("--root_domain", type=str, default="ipa-helper.dev") +@click.option("--sidecar_domain", type=str, default="") +@click.option("--helper_port", type=int, default=7430) +@click.option("--sidecar_port", type=int, default=17430) @click.option("--identity", required=True, type=int) def start_helper_sidecar( config_path: Path, root_path: Optional[Path], + root_domain: str, + sidecar_domain: str, + helper_port: int, + sidecar_port: int, identity: int, ): - command = start_helper_sidecar_command( - config_path, - identity, - root_path, + sidecar_command = start_helper_sidecar_command( + config_path=config_path, + identity=identity, + helper_port=helper_port, + sidecar_port=sidecar_port, + root_path=root_path, + ) + traefik_command = start_traefik_command( + config_path=config_path, + sidecar_port=sidecar_port, + root_domain=root_domain, + sidecar_domain=sidecar_domain, ) - command.run_blocking_no_output_capture() + start_commands_parallel([sidecar_command, traefik_command]) @cli.command @@ -73,29 +135,55 @@ def start_helper_sidecar( show_default=True, ) @click.option("--root_path", type=click_pathlib.Path(), default=None) +@click.option("--helper_start_port", type=int, default=7430) +@click.option("--sidecar_start_port", type=int, default=17430) def start_local_dev( config_path: Path, root_path: Optional[Path], + helper_start_port: int, + sidecar_start_port: int, ): + root_domain: str = "draft.test" + server_port: int = 7530 npm_install_command = Command( cmd="npm --prefix server install", ) npm_install_command.run_blocking_no_output_capture() npm_run_dev_command = Command( - cmd="npm --prefix server run dev", + cmd=f"npm --prefix server run dev -- --port {server_port}", + ) + + helper_ports = {role: helper_start_port + int(role) for role in Role} + sidecar_ports = {role: sidecar_start_port + int(role) for role in Role} + + _env = {} + local_ca_process = subprocess.run( + shlex.split("mkcert -CAROOT"), + capture_output=True, + check=True, + ) + _env["SSL_CERT_FILE"] = ( + Path(local_ca_process.stdout.decode("utf8").strip()) / "rootCA.pem" ) - network_config = Path(config_path) / Path("network.toml") - helpers = load_helpers_from_network_config(network_config) sidecar_commands = [ start_helper_sidecar_command( - config_path, - helper.role, - root_path, + config_path=config_path, + identity=role, + helper_port=helper_ports[role], + sidecar_port=sidecar_ports[role], + root_path=root_path, + _env=_env, ) - for helper in helpers.values() + for role in Role ] - commands = [npm_run_dev_command] + sidecar_commands + traefik_command = start_traefik_local_command( + config_path=config_path, + sidecar_ports=tuple(sidecar_ports.values()), + server_port=server_port, + root_domain=root_domain, + ) + commands = sidecar_commands + [npm_run_dev_command, traefik_command] start_commands_parallel(commands) diff --git a/sidecar/traefik/dynamic-local/dynamic.yaml b/sidecar/traefik/dynamic-local/dynamic.yaml new file mode 100644 index 0000000..c2c84cf --- /dev/null +++ b/sidecar/traefik/dynamic-local/dynamic.yaml @@ -0,0 +1,53 @@ +http: + routers: + server-router: + entryPoints: + - "web-secure" + rule: "Host(`{{ env "SERVER_DOMAIN"}}`)" + service: "server-service" + tls: {} + sidecar-0-router: + entryPoints: + - "web-secure" + rule: "Host(`{{ env "SIDECAR_0_DOMAIN"}}`)" + service: "sidecar-0-service" + tls: {} + sidecar-1-router: + entryPoints: + - "web-secure" + rule: "Host(`{{ env "SIDECAR_1_DOMAIN"}}`)" + service: "sidecar-1-service" + tls: {} + sidecar-2-router: + entryPoints: + - "web-secure" + rule: "Host(`{{ env "SIDECAR_2_DOMAIN"}}`)" + service: "sidecar-2-service" + tls: {} + sidecar-3-router: + entryPoints: + - "web-secure" + rule: "Host(`{{ env "SIDECAR_3_DOMAIN"}}`)" + service: "sidecar-3-service" + tls: {} + services: + server-service: + loadBalancer: + servers: + - url: "http://localhost:{{ env "SERVER_PORT"}}" + sidecar-0-service: + loadBalancer: + servers: + - url: "http://localhost:{{ env "SIDECAR_0_PORT"}}" + sidecar-1-service: + loadBalancer: + servers: + - url: "http://localhost:{{ env "SIDECAR_1_PORT"}}" + sidecar-2-service: + loadBalancer: + servers: + - url: "http://localhost:{{ env "SIDECAR_2_PORT"}}" + sidecar-3-service: + loadBalancer: + servers: + - url: "http://localhost:{{ env "SIDECAR_3_PORT"}}" diff --git a/sidecar/traefik/dynamic-local/tls.yaml b/sidecar/traefik/dynamic-local/tls.yaml new file mode 100644 index 0000000..e59cc17 --- /dev/null +++ b/sidecar/traefik/dynamic-local/tls.yaml @@ -0,0 +1,4 @@ +tls: + certificates: + - certFile: {{ env "CERT_DIR" }}/cert.pem + keyFile: {{ env "CERT_DIR" }}/key.pem diff --git a/sidecar/traefik/dynamic/dynamic.yaml b/sidecar/traefik/dynamic/dynamic.yaml new file mode 100644 index 0000000..8db0ebf --- /dev/null +++ b/sidecar/traefik/dynamic/dynamic.yaml @@ -0,0 +1,14 @@ +http: + routers: + sidecar-router: + entryPoints: + - "web" + - "web-secure" + rule: "Host(`{{ env "SIDECAR_DOMAIN"}}`)" + service: "sidecar-service" + tls: {} + services: + sidecar-service: + loadBalancer: + servers: + - url: "http://localhost:{{ env "SIDECAR_PORT"}}" diff --git a/sidecar/traefik/dynamic/tls.yaml b/sidecar/traefik/dynamic/tls.yaml new file mode 100644 index 0000000..e59cc17 --- /dev/null +++ b/sidecar/traefik/dynamic/tls.yaml @@ -0,0 +1,4 @@ +tls: + certificates: + - certFile: {{ env "CERT_DIR" }}/cert.pem + keyFile: {{ env "CERT_DIR" }}/key.pem diff --git a/sidecar/traefik/traefik-local.yaml b/sidecar/traefik/traefik-local.yaml new file mode 100644 index 0000000..b980684 --- /dev/null +++ b/sidecar/traefik/traefik-local.yaml @@ -0,0 +1,9 @@ +entryPoints: + web: + address: ":80" + web-secure: + address: ":443" + +providers: + file: + directory: "sidecar/traefik/dynamic-local" diff --git a/sidecar/traefik/traefik.yaml b/sidecar/traefik/traefik.yaml new file mode 100644 index 0000000..1281337 --- /dev/null +++ b/sidecar/traefik/traefik.yaml @@ -0,0 +1,9 @@ +entryPoints: + web: + address: ":80" + web-secure: + address: ":443" + +providers: + file: + directory: "sidecar/traefik/dynamic"