From 5af7552656954af66e63ba96afd8915f2fd703ea Mon Sep 17 00:00:00 2001 From: David DiMaria Date: Tue, 30 Jul 2024 08:09:36 -0600 Subject: [PATCH 001/113] Create a storage trait and implement for s3 and filesystem --- Cargo.lock | 106 +++++++++++++++- quadratic-rust-shared/Cargo.toml | 4 + quadratic-rust-shared/src/aws/s3.rs | 47 ++++++- quadratic-rust-shared/src/error.rs | 18 +++ quadratic-rust-shared/src/lib.rs | 1 + .../src/storage/file_system.rs | 115 ++++++++++++++++++ quadratic-rust-shared/src/storage/mod.rs | 31 +++++ quadratic-rust-shared/src/storage/s3.rs | 54 ++++++++ 8 files changed, 369 insertions(+), 7 deletions(-) create mode 100644 quadratic-rust-shared/src/storage/file_system.rs create mode 100644 quadratic-rust-shared/src/storage/mod.rs create mode 100644 quadratic-rust-shared/src/storage/s3.rs diff --git a/Cargo.lock b/Cargo.lock index 5a1b68185d..9811bd3362 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -319,6 +319,17 @@ dependencies = [ "regex-syntax 0.8.4", ] +[[package]] +name = "assert-json-diff" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4259cbe96513d2f1073027a259fc2ca917feb3026a5a8d984e3628e490255cc0" +dependencies = [ + "extend", + "serde", + "serde_json", +] + [[package]] name = "async-compression" version = "0.4.11" @@ -634,9 +645,9 @@ dependencies = [ [[package]] name = "aws-smithy-http" -version = "0.60.8" +version = "0.60.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a7de001a1b9a25601016d8057ea16e31a45fdca3751304c8edf4ad72e706c08" +checksum = "d9cd0ae3d97daa0a2bf377a4d8e8e1362cae590c4a1aad0d40058ebca18eb91e" dependencies = [ "aws-smithy-eventstream", "aws-smithy-runtime-api", @@ -662,6 +673,22 @@ dependencies = [ "aws-smithy-types", ] +[[package]] +name = "aws-smithy-protocol-test" +version = "0.60.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "020468b04f916b36e0a791c4ebf80777ad2c25d8b9ebb8db14939e98a37abec0" +dependencies = [ + "assert-json-diff", + "aws-smithy-runtime-api", + "http 0.2.12", + "pretty_assertions", + "regex-lite", + "roxmltree", + "serde_json", + "thiserror", +] + [[package]] name = "aws-smithy-query" version = "0.60.7" @@ -674,12 +701,13 @@ dependencies = [ [[package]] name = "aws-smithy-runtime" -version = "1.5.5" +version = "1.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0d3965f6417a92a6d1009c5958a67042f57e46342afb37ca58f9ad26744ec73" +checksum = "ce87155eba55e11768b8c1afa607f3e864ae82f03caf63258b37455b0ad02537" dependencies = [ "aws-smithy-async", "aws-smithy-http", + "aws-smithy-protocol-test", "aws-smithy-runtime-api", "aws-smithy-types", "bytes", @@ -688,21 +716,26 @@ dependencies = [ "http 0.2.12", "http-body 0.4.6", "http-body 1.0.0", + "httparse", "hyper 0.14.29", "hyper-rustls", + "indexmap 2.2.6", "once_cell", "pin-project-lite", "pin-utils", "rustls", + "serde", + "serde_json", "tokio", "tracing", + "tracing-subscriber", ] [[package]] name = "aws-smithy-runtime-api" -version = "1.7.0" +version = "1.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b570ea39eb95bd32543f6e4032bce172cb6209b9bc8c83c770d08169e875afc" +checksum = "30819352ed0a04ecf6a2f3477e344d2d1ba33d43e0f09ad9047c12e0d923616f" dependencies = [ "aws-smithy-async", "aws-smithy-types", @@ -1467,6 +1500,12 @@ version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "339544cc9e2c4dc3fc7149fd630c5f22263a4fdf18a98afd0075784968b5cf00" +[[package]] +name = "diff" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56254986775e3233ffa9c4d7d3faaf6d36a2c09d30b20687e9f88bc8bafc16c8" + [[package]] name = "digest" version = "0.10.7" @@ -1595,6 +1634,18 @@ version = "2.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0206175f82b8d6bf6652ff7d71a1e27fd2e4efde587fd368662814d6ec1d9ce0" +[[package]] +name = "extend" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f47da3a72ec598d9c8937a7ebca8962a5c7a1f28444e38c2b33c771ba3f55f05" +dependencies = [ + "proc-macro-error", + "proc-macro2 1.0.86", + "quote 1.0.36", + "syn 1.0.109", +] + [[package]] name = "fake" version = "2.9.2" @@ -2902,6 +2953,16 @@ version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" +[[package]] +name = "pretty_assertions" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af7cee1a6c8a5b9208b3cb1061f10c0cb689087b3d8ce85fb9d2dd7a29b6ba66" +dependencies = [ + "diff", + "yansi", +] + [[package]] name = "proc-macro-error" version = "1.0.4" @@ -3170,10 +3231,14 @@ dependencies = [ "async-trait", "aws-config", "aws-sdk-s3", + "aws-smithy-async", + "aws-smithy-runtime", + "aws-smithy-runtime-api", "bigdecimal 0.3.1", "bytes", "chrono", "futures-util", + "http 1.1.0", "jsonwebtoken", "parquet", "redis", @@ -3442,6 +3507,15 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "roxmltree" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "921904a62e410e37e215c40381b7117f830d9d89ba60ab5236170541dd25646b" +dependencies = [ + "xmlparser", +] + [[package]] name = "rsa" version = "0.9.6" @@ -3698,6 +3772,7 @@ version = "1.0.117" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "455182ea6142b14f93f4bc5320a2b31c1f266b66a4a5c858b013302a5d8cbfc3" dependencies = [ + "indexmap 2.2.6", "itoa", "ryu", "serde", @@ -4666,6 +4741,16 @@ dependencies = [ "tracing-core", ] +[[package]] +name = "tracing-serde" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc6b213177105856957181934e4920de57730fc69bf42c37ee5bb664d406d9e1" +dependencies = [ + "serde", + "tracing-core", +] + [[package]] name = "tracing-subscriber" version = "0.3.18" @@ -4676,12 +4761,15 @@ dependencies = [ "nu-ansi-term", "once_cell", "regex", + "serde", + "serde_json", "sharded-slab", "smallvec", "thread_local", "tracing", "tracing-core", "tracing-log", + "tracing-serde", ] [[package]] @@ -5284,6 +5372,12 @@ version = "0.13.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "66fee0b777b0f5ac1c69bb06d361268faafa61cd4682ae064a171c16c433e9e4" +[[package]] +name = "yansi" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09041cd90cf85f7f8b2df60c646f853b7f535ce68f85244eb6731cf89fa498ec" + [[package]] name = "zerocopy" version = "0.7.34" diff --git a/quadratic-rust-shared/Cargo.toml b/quadratic-rust-shared/Cargo.toml index e7c52dfe59..3402f48943 100644 --- a/quadratic-rust-shared/Cargo.toml +++ b/quadratic-rust-shared/Cargo.toml @@ -29,4 +29,8 @@ tracing = "0.1.40" uuid = { version = "1.6.1", features = ["serde", "v4"] } [dev-dependencies] +aws-smithy-async = { version = "1.2.1", features = ["test-util"] } +aws-smithy-runtime = {version = "1.6.2", features = ["test-util"] } +aws-smithy-runtime-api = "1.7.1" +http = "1.1.0" tracing-test = "0.2.4" diff --git a/quadratic-rust-shared/src/aws/s3.rs b/quadratic-rust-shared/src/aws/s3.rs index dccb37fd4a..56b65cecf0 100644 --- a/quadratic-rust-shared/src/aws/s3.rs +++ b/quadratic-rust-shared/src/aws/s3.rs @@ -45,4 +45,49 @@ pub async fn upload_object( } #[cfg(test)] -mod tests {} +pub mod tests { + // use aws_config::{imds::Client as ImdsClient, provider_config::ProviderConfig}; + // use aws_sdk_s3::primitives::SdkBody; + // use aws_smithy_async::test_util::InstantSleep; + // use aws_smithy_runtime::client::http::test_util::{ReplayEvent, StaticReplayClient}; + // use aws_smithy_runtime_api::client::orchestrator::{HttpRequest, HttpResponse}; + // use http::Uri; + + // pub fn imds_request(path: &'static str, token: &str) -> HttpRequest { + // http::Request::builder() + // .uri(Uri::from_static(path)) + // .method("GET") + // .header("x-aws-ec2-metadata-token", token) + // .body(SdkBody::empty()) + // .unwrap() + // .try_into() + // .unwrap() + // } + + // pub fn imds_response(body: &'static str) -> HttpResponse { + // HttpResponse::try_from( + // http::Response::builder() + // .status(200) + // .body(SdkBody::from(body)) + // .unwrap(), + // ) + // .unwrap() + // } + + // pub fn make_imds_client(http_client: &StaticReplayClient) -> ImdsClient { + // tokio::time::pause(); + // ImdsClient::builder() + // .configure( + // &ProviderConfig::empty() + // .with_sleep_impl(InstantSleep::unlogged()) + // .with_http_client(http_client.clone()), + // ) + // .build() + // } + + // pub fn mock_imds_client(events: Vec) -> (ImdsClient, StaticReplayClient) { + // let http_client = StaticReplayClient::new(events); + // let client = make_imds_client(&http_client); + // (client, http_client) + // } +} diff --git a/quadratic-rust-shared/src/error.rs b/quadratic-rust-shared/src/error.rs index 16187fee36..c435d8d99f 100644 --- a/quadratic-rust-shared/src/error.rs +++ b/quadratic-rust-shared/src/error.rs @@ -43,6 +43,21 @@ pub enum Sql { Schema(String), } +#[derive(Error, Debug, Serialize, Deserialize, PartialEq, Clone)] +pub enum Storage { + #[error("Error creating directory {0}: {1}")] + CreateDirectory(String, String), + + #[error("Invalid key: {0}")] + InvalidKey(String), + + #[error("Error reading key {0}: {1}")] + Read(String, String), + + #[error("Error writing key {0}: {1}")] + Write(String, String), +} + #[derive(Error, Debug, Serialize, Deserialize, PartialEq, Clone)] pub enum SharedError { #[error("Error with Arrow: {0}")] @@ -69,6 +84,9 @@ pub enum SharedError { #[error("Error with SQL connector: {0}")] Sql(Sql), + #[error("Error with Storage: {0}")] + Storage(Storage), + #[error("Error with Uuid: {0}")] Uuid(String), } diff --git a/quadratic-rust-shared/src/lib.rs b/quadratic-rust-shared/src/lib.rs index b4065de1b0..991bc6e8c8 100644 --- a/quadratic-rust-shared/src/lib.rs +++ b/quadratic-rust-shared/src/lib.rs @@ -6,6 +6,7 @@ pub mod error; pub mod pubsub; pub mod quadratic_api; pub mod sql; +pub mod storage; // pub use aws::*; pub use error::*; diff --git a/quadratic-rust-shared/src/storage/file_system.rs b/quadratic-rust-shared/src/storage/file_system.rs new file mode 100644 index 0000000000..a06349edd4 --- /dev/null +++ b/quadratic-rust-shared/src/storage/file_system.rs @@ -0,0 +1,115 @@ +use async_trait::async_trait; +use bytes::Bytes; +use std::path::{Path, PathBuf}; +use tokio::fs::{create_dir_all, File}; +use tokio::io::{AsyncReadExt, AsyncWriteExt}; + +use super::Storage; +use crate::error::Result; +use crate::SharedError; +use crate::Storage as StorageError; + +#[derive(Debug)] +pub struct FileSystemConfig { + pub path: String, +} + +pub struct FileSystem { + pub config: FileSystemConfig, +} + +#[async_trait] +impl<'a> Storage<'a> for FileSystem { + type Config = FileSystemConfig; + + async fn read(&self, key: &str) -> Result { + let file_path = self.full_path(key, false).await?.0; + let mut bytes = vec![]; + let mut file = File::open(file_path) + .await + .map_err(|e| Self::read_error(key, &e))?; + + file.read_to_end(&mut bytes) + .await + .map_err(|e| Self::read_error(key, &e))?; + + Ok(bytes.into()) + } + + async fn write(&self, key: &'a str, data: &'a Bytes) -> Result<()> { + let file_path = self.full_path(key, true).await?.0; + let mut file = File::create(file_path) + .await + .map_err(|e| Self::write_error(key, &e))?; + file.write_all(data) + .await + .map_err(|e| Self::write_error(key, &e))?; + + Ok(()) + } +} + +impl FileSystem { + pub async fn full_path(&self, key: &str, create_dir: bool) -> Result<(PathBuf, PathBuf)> { + let FileSystemConfig { path } = &self.config; + let parts = key.split('-').collect::>(); + let invalid_key = || SharedError::Storage(StorageError::InvalidKey(key.to_owned())); + + // expecting uuid-sequence_number.grid + // e.g. aad29798-0bf9-4b25-ab45-e22efd37d446-0.grid + if parts.len() < 5 { + return Err(invalid_key()); + } + + let uuid = &parts[0..parts.len() - 1].join("-"); + let file_name = parts.last().ok_or_else(|| invalid_key())?; + let dir = Path::new(path).join(uuid); + let full_path = dir.join(file_name); + + if create_dir { + create_dir_all(dir.to_owned()).await.map_err(|e| { + SharedError::Storage(StorageError::CreateDirectory( + dir.to_string_lossy().into_owned(), + e.to_string(), + )) + })?; + } + + Ok((full_path, dir)) + } +} + +#[cfg(test)] +mod tests { + use tokio::fs::{remove_dir, remove_file}; + use uuid::Uuid; + + use super::*; + use std::env; + + fn config() -> FileSystemConfig { + FileSystemConfig { + path: env::temp_dir().to_str().unwrap().to_string(), + } + } + + #[tokio::test] + async fn file_system_write_and_read() { + let config = config(); + let storage = FileSystem { config }; + let file_name = Uuid::new_v4().to_string(); + let seqence_number = 0; + let key = &format!("{}-{}.grid", file_name, seqence_number); + let data = &Bytes::from("Hello, world!"); + + storage.write(key, data).await.unwrap(); + let read_data = storage.read(key).await.unwrap(); + + // cleanup + let (full_path, dir) = storage.full_path(key, false).await.unwrap(); + remove_file(full_path).await.unwrap(); + remove_dir(dir).await.unwrap(); + + assert_eq!(data, &read_data); + } +} diff --git a/quadratic-rust-shared/src/storage/mod.rs b/quadratic-rust-shared/src/storage/mod.rs new file mode 100644 index 0000000000..1791da0950 --- /dev/null +++ b/quadratic-rust-shared/src/storage/mod.rs @@ -0,0 +1,31 @@ +use async_trait::async_trait; +use bytes::Bytes; +use file_system::FileSystemConfig; +use s3::S3Config; + +use crate::{error::Result, SharedError, Storage as StorageError}; + +pub mod file_system; +pub mod s3; + +#[derive(Debug)] +pub enum Config<'a> { + S3(S3Config<'a>), + FileSystem(FileSystemConfig), +} + +#[async_trait] +pub trait Storage<'a> { + type Config; + + async fn read(&self, key: &str) -> Result; + async fn write(&self, key: &'a str, data: &'a Bytes) -> Result<()>; + + fn read_error(key: &str, e: impl ToString) -> SharedError { + SharedError::Storage(StorageError::Read(key.into(), e.to_string())) + } + + fn write_error(key: &str, e: impl ToString) -> SharedError { + SharedError::Storage(StorageError::Write(key.into(), e.to_string())) + } +} diff --git a/quadratic-rust-shared/src/storage/s3.rs b/quadratic-rust-shared/src/storage/s3.rs new file mode 100644 index 0000000000..86631acddb --- /dev/null +++ b/quadratic-rust-shared/src/storage/s3.rs @@ -0,0 +1,54 @@ +use async_trait::async_trait; +use aws_sdk_s3::Client; +use bytes::Bytes; + +use super::Storage; +use crate::{ + aws::s3::{download_object, upload_object}, + error::Result, +}; + +#[derive(Debug)] +pub struct S3Config<'a> { + pub client: Client, + pub bucket: &'a str, +} + +pub struct S3<'a> { + pub config: S3Config<'a>, +} + +#[async_trait] +impl<'a> Storage<'a> for S3<'a> { + type Config = S3Config<'a>; + + async fn read(&self, key: &str) -> Result { + let S3Config { client, bucket } = &self.config; + + let file = download_object(client, bucket, key) + .await + .map_err(|e| Self::read_error(key, &e))?; + + let bytes = file + .body + .collect() + .await + .map_err(|e| Self::read_error(key, &e))? + .into_bytes(); + + Ok(bytes) + } + + async fn write(&self, key: &'a str, data: &'a Bytes) -> Result<()> { + let S3Config { client, bucket } = &self.config; + + upload_object(client, bucket, key, data) + .await + .map_err(|e| Self::write_error(key, &e))?; + + Ok(()) + } +} + +#[cfg(test)] +mod tests {} From 0a0ba6be654749be4532e17dbe35de5b6ade1800 Mon Sep 17 00:00:00 2001 From: David DiMaria Date: Tue, 30 Jul 2024 11:43:55 -0600 Subject: [PATCH 002/113] Implement storage when reading/writing in the file service --- Cargo.lock | 5 +- quadratic-files/.env.docker | 9 +++- quadratic-files/.env.example | 9 +++- quadratic-files/.env.test | 11 ++++- quadratic-files/Cargo.toml | 1 + quadratic-files/src/config.rs | 23 ++++++++-- quadratic-files/src/error.rs | 4 +- quadratic-files/src/file.rs | 40 +++++++--------- quadratic-files/src/state/settings.rs | 46 +++++++++++++------ .../src/storage/file_system.rs | 15 ++++-- quadratic-rust-shared/src/storage/mod.rs | 40 ++++++++++++++-- quadratic-rust-shared/src/storage/s3.rs | 25 ++++++---- 12 files changed, 162 insertions(+), 66 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9811bd3362..fa0f44e861 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1023,9 +1023,9 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "bytes" -version = "1.6.0" +version = "1.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "514de17de45fdb8dc022b1a7975556c53c86f9f0aa5f534b98977b171857c2c9" +checksum = "a12916984aab3fa6e39d655a33e09c0071eb36d6ab3aea5c2d78551f1df6d952" [[package]] name = "bytes-utils" @@ -3150,6 +3150,7 @@ version = "0.1.0" dependencies = [ "axum", "axum-extra", + "bytes", "chrono", "dotenv", "envy", diff --git a/quadratic-files/.env.docker b/quadratic-files/.env.docker index 7d6e07a662..0276b02d33 100644 --- a/quadratic-files/.env.docker +++ b/quadratic-files/.env.docker @@ -15,7 +15,14 @@ PUBSUB_PASSWORD= PUBSUB_ACTIVE_CHANNELS=active_channels PUBSUB_PROCESSED_TRANSACTIONS_CHANNEL=processed_transactions +# Storage +STORAGE_TYPE=s3 # s3 or filesystem + +# Storage: s3 AWS_S3_REGION= AWS_S3_BUCKET_NAME=quadratic-api-docker AWS_S3_ACCESS_KEY_ID= -AWS_S3_SECRET_ACCESS_KEY= \ No newline at end of file +AWS_S3_SECRET_ACCESS_KEY= + +# Storage: filesystem +FILE_DIR=/storage \ No newline at end of file diff --git a/quadratic-files/.env.example b/quadratic-files/.env.example index 525b4a593f..2eb0159547 100644 --- a/quadratic-files/.env.example +++ b/quadratic-files/.env.example @@ -15,7 +15,14 @@ PUBSUB_PASSWORD= PUBSUB_ACTIVE_CHANNELS=active_channels PUBSUB_PROCESSED_TRANSACTIONS_CHANNEL=processed_transactions +# Storage +STORAGE_TYPE=s3 # s3 or filesystem + +# Storage: s3 AWS_S3_REGION=us-east-2 AWS_S3_BUCKET_NAME=quadratic-api-docker AWS_S3_ACCESS_KEY_ID=test -AWS_S3_SECRET_ACCESS_KEY=test \ No newline at end of file +AWS_S3_SECRET_ACCESS_KEY=test + +# Storage: filesystem +FILE_DIR=/storage \ No newline at end of file diff --git a/quadratic-files/.env.test b/quadratic-files/.env.test index ca08711da8..927fcc3716 100644 --- a/quadratic-files/.env.test +++ b/quadratic-files/.env.test @@ -15,7 +15,14 @@ PUBSUB_PASSWORD= PUBSUB_ACTIVE_CHANNELS=active_channels PUBSUB_PROCESSED_TRANSACTIONS_CHANNEL=processed_transactions +# Storage +STORAGE_TYPE=s3 # s3 or filesystem + +# Storage: s3 AWS_S3_REGION= -AWS_S3_BUCKET_NAME= +AWS_S3_BUCKET_NAME=quadratic-api-docker AWS_S3_ACCESS_KEY_ID= -AWS_S3_SECRET_ACCESS_KEY= \ No newline at end of file +AWS_S3_SECRET_ACCESS_KEY= + +# Storage: filesystem +FILE_DIR=/storage \ No newline at end of file diff --git a/quadratic-files/Cargo.toml b/quadratic-files/Cargo.toml index 88a93ba810..ac29ebf553 100644 --- a/quadratic-files/Cargo.toml +++ b/quadratic-files/Cargo.toml @@ -7,6 +7,7 @@ authors = ["David DiMaria "] [dependencies] axum = { version = "0.7.1", features = ["ws"] } axum-extra = { version = "0.9.0", features = ["typed-header"] } +bytes = "1.6.1" chrono = { version= "0.4.31", features = ["serde"] } dotenv = "0.15.0" envy = "0.4.2" diff --git a/quadratic-files/src/config.rs b/quadratic-files/src/config.rs index 8ecaaa362a..674f0a0382 100644 --- a/quadratic-files/src/config.rs +++ b/quadratic-files/src/config.rs @@ -9,6 +9,14 @@ use crate::error::{FilesError, Result}; use dotenv::dotenv; use quadratic_rust_shared::environment::Environment; use serde::Deserialize; +use strum_macros::Display; + +#[derive(Deserialize, Debug, Display)] +#[serde(rename_all = "kebab-case")] +pub(crate) enum StorageType { + S3, + FileSystem, +} #[allow(dead_code)] #[derive(Deserialize, Debug)] @@ -30,10 +38,17 @@ pub(crate) struct Config { pub(crate) quadratic_api_uri: String, pub(crate) m2m_auth_token: String, - pub(crate) aws_s3_region: String, - pub(crate) aws_s3_bucket_name: String, - pub(crate) aws_s3_access_key_id: String, - pub(crate) aws_s3_secret_access_key: String, + // Storage Type: s3 or filesystem + pub(crate) storage_type: StorageType, + + // StorageConfig::S3 + pub(crate) aws_s3_region: Option, + pub(crate) aws_s3_bucket_name: Option, + pub(crate) aws_s3_access_key_id: Option, + pub(crate) aws_s3_secret_access_key: Option, + + // StorageConfig::FileSystem + pub(crate) path: Option, } /// Load the global configuration from the environment into Config. diff --git a/quadratic-files/src/error.rs b/quadratic-files/src/error.rs index 7444ba7e6d..11bc5b9de4 100644 --- a/quadratic-files/src/error.rs +++ b/quadratic-files/src/error.rs @@ -34,8 +34,8 @@ pub(crate) enum FilesError { #[error("Internal server error: {0}")] InternalServer(String), - #[error("Unable to load file {0} from bucket {1}: {2}")] - LoadFile(String, String, String), + #[error("Unable to load file {0}: {1}")] + LoadFile(String, String), #[error("PubSub error: {0}")] PubSub(String), diff --git a/quadratic-files/src/file.rs b/quadratic-files/src/file.rs index be3dcedfc6..9b7a8c9fa7 100644 --- a/quadratic-files/src/file.rs +++ b/quadratic-files/src/file.rs @@ -12,12 +12,9 @@ use quadratic_core::{ }, }; use quadratic_rust_shared::{ - aws::{ - s3::{download_object, upload_object}, - Client, - }, pubsub::PubSub as PubSubTrait, quadratic_api::{get_file_checkpoint, set_file_checkpoint}, + storage::{Storage, StorageContainer}, }; use crate::{ @@ -45,20 +42,18 @@ pub(crate) fn apply_transaction(grid: &mut GridController, operations: Vec Result { - let file = download_object(client, bucket, key).await?; - let body = file - .body - .collect() + let body = storage + .read(key) .await - .map_err(|e| FilesError::LoadFile(key.into(), bucket.to_string(), e.to_string()))? - .into_bytes(); - let body = std::str::from_utf8(&body) - .map_err(|e| FilesError::LoadFile(key.into(), bucket.to_string(), e.to_string()))?; + .map_err(|e| FilesError::LoadFile(key.into(), e.to_string()))?; + + // TODO(ddimaria): remove this line when the binary file format PR is merged + let body = + std::str::from_utf8(&body).map_err(|e| FilesError::LoadFile(key.into(), e.to_string()))?; let grid = load_file(key, body)?; Ok(GridController::from_grid(grid, sequence_num)) @@ -70,16 +65,14 @@ pub(crate) fn key(file_id: Uuid, sequence: u64) -> String { /// Load a file from S3, add it to memory, process transactions and upload it back to S3 pub(crate) async fn process_transactions( - client: &Client, - bucket: &str, + storage: &StorageContainer, file_id: Uuid, checkpoint_sequence_num: u64, final_sequence_num: u64, operations: Vec, ) -> Result { let mut grid = get_and_load_object( - client, - bucket, + storage, &key(file_id, checkpoint_sequence_num), checkpoint_sequence_num, ) @@ -90,7 +83,7 @@ pub(crate) async fn process_transactions( apply_transaction(&mut grid, operations); let body = export_file(&key, grid.grid_mut())?; - upload_object(client, bucket, &key, &body).await?; + storage.write(&key, &body.into()).await?; Ok(final_sequence_num) } @@ -105,8 +98,7 @@ pub(crate) async fn process_queue_for_room( let channel = &file_id.to_string(); let Settings { - aws_client, - aws_s3_bucket_name, + storage, quadratic_api_uri, m2m_auth_token, .. @@ -173,8 +165,7 @@ pub(crate) async fn process_queue_for_room( // process the transactions and save the file to S3 let last_sequence_num = process_transactions( - aws_client, - aws_s3_bucket_name, + storage, *file_id, checkpoint_sequence_num, last_sequence_num, @@ -199,6 +190,7 @@ pub(crate) async fn process_queue_for_room( // update the checkpoint in quadratic-api let key = &key(*file_id, last_sequence_num); + set_file_checkpoint( quadratic_api_uri, m2m_auth_token, @@ -206,7 +198,7 @@ pub(crate) async fn process_queue_for_room( last_sequence_num, CURRENT_VERSION.into(), key.to_owned(), - aws_s3_bucket_name.to_owned(), + storage.path().to_owned(), ) .await?; diff --git a/quadratic-files/src/state/settings.rs b/quadratic-files/src/state/settings.rs index fa4b920aeb..aa3210f59c 100644 --- a/quadratic-files/src/state/settings.rs +++ b/quadratic-files/src/state/settings.rs @@ -1,33 +1,53 @@ -use quadratic_rust_shared::aws::{client, Client}; +use quadratic_rust_shared::aws::client; use quadratic_rust_shared::environment::Environment; +use quadratic_rust_shared::storage::file_system::{FileSystem, FileSystemConfig}; +use quadratic_rust_shared::storage::s3::{S3Config, S3}; +use quadratic_rust_shared::storage::StorageContainer; -use crate::config::Config; +use crate::config::{Config, StorageType}; #[derive(Debug)] pub(crate) struct Settings { pub(crate) quadratic_api_uri: String, pub(crate) m2m_auth_token: String, - pub(crate) aws_client: Client, - pub(crate) aws_s3_bucket_name: String, + pub(crate) storage: StorageContainer, pub(crate) pubsub_processed_transactions_channel: String, } impl Settings { + // Create a new Settings struct from the provided Config. + // Panics are OK here since this is set at startup and we want to fail fast. pub(crate) async fn new(config: &Config) -> Self { let is_local = config.environment == Environment::Docker || config.environment == Environment::Local; + let expected = |val: &Option, var: &str| { + val.to_owned() + .expect(&format!("Expected {} to have a value", var)) + }; + + let storage = match config.storage_type { + StorageType::S3 => StorageContainer::S3(S3::new(S3Config { + client: client( + &expected(&config.aws_s3_access_key_id, "AWS_S3_ACCESS_KEY_ID"), + &expected(&config.aws_s3_secret_access_key, "AWS_S3_SECRET_ACCESS_KEY"), + &expected(&config.aws_s3_region, "AWS_S3_REGION"), + "Quadratic File Service", + is_local, + ) + .await, + bucket: expected(&config.aws_s3_bucket_name, "AWS_S3_BUCKET_NAME"), + })), + StorageType::FileSystem => { + StorageContainer::FileSystem(FileSystem::new(FileSystemConfig { + path: expected(&config.path, "FILE_DIR"), + })) + } + }; + Settings { quadratic_api_uri: config.quadratic_api_uri.to_owned(), m2m_auth_token: config.m2m_auth_token.to_owned(), - aws_client: client( - &config.aws_s3_access_key_id, - &config.aws_s3_secret_access_key, - &config.aws_s3_region, - "Quadratic File Service", - is_local, - ) - .await, - aws_s3_bucket_name: config.aws_s3_bucket_name.to_owned(), + storage, pubsub_processed_transactions_channel: config .pubsub_processed_transactions_channel .to_owned(), diff --git a/quadratic-rust-shared/src/storage/file_system.rs b/quadratic-rust-shared/src/storage/file_system.rs index a06349edd4..3f13f6d58d 100644 --- a/quadratic-rust-shared/src/storage/file_system.rs +++ b/quadratic-rust-shared/src/storage/file_system.rs @@ -14,14 +14,13 @@ pub struct FileSystemConfig { pub path: String, } +#[derive(Debug)] pub struct FileSystem { pub config: FileSystemConfig, } #[async_trait] -impl<'a> Storage<'a> for FileSystem { - type Config = FileSystemConfig; - +impl Storage for FileSystem { async fn read(&self, key: &str) -> Result { let file_path = self.full_path(key, false).await?.0; let mut bytes = vec![]; @@ -36,7 +35,7 @@ impl<'a> Storage<'a> for FileSystem { Ok(bytes.into()) } - async fn write(&self, key: &'a str, data: &'a Bytes) -> Result<()> { + async fn write<'a>(&self, key: &'a str, data: &'a Bytes) -> Result<()> { let file_path = self.full_path(key, true).await?.0; let mut file = File::create(file_path) .await @@ -47,9 +46,17 @@ impl<'a> Storage<'a> for FileSystem { Ok(()) } + + fn path(&self) -> &str { + &self.config.path + } } impl FileSystem { + pub fn new(config: FileSystemConfig) -> Self { + Self { config } + } + pub async fn full_path(&self, key: &str, create_dir: bool) -> Result<(PathBuf, PathBuf)> { let FileSystemConfig { path } = &self.config; let parts = key.split('-').collect::>(); diff --git a/quadratic-rust-shared/src/storage/mod.rs b/quadratic-rust-shared/src/storage/mod.rs index 1791da0950..5bd7bc424c 100644 --- a/quadratic-rust-shared/src/storage/mod.rs +++ b/quadratic-rust-shared/src/storage/mod.rs @@ -9,17 +9,47 @@ pub mod file_system; pub mod s3; #[derive(Debug)] -pub enum Config<'a> { - S3(S3Config<'a>), +pub enum Config { + S3(S3Config), FileSystem(FileSystemConfig), } +#[derive(Debug)] +pub enum StorageContainer { + S3(s3::S3), + FileSystem(file_system::FileSystem), +} + +// TODO(ddimaria): this is a temp hack to get around some trait issues, do something better #[async_trait] -pub trait Storage<'a> { - type Config; +impl Storage for StorageContainer { + async fn read(&self, key: &str) -> Result { + match self { + Self::S3(s3) => s3.read(key).await, + Self::FileSystem(fs) => fs.read(key).await, + } + } + + async fn write<'a>(&self, key: &'a str, data: &'a Bytes) -> Result<()> { + match self { + Self::S3(s3) => s3.write(key, data).await, + Self::FileSystem(fs) => fs.write(key, data).await, + } + } + fn path(&self) -> &str { + match self { + Self::S3(s3) => s3.path(), + Self::FileSystem(fs) => fs.path(), + } + } +} + +#[async_trait] +pub trait Storage { async fn read(&self, key: &str) -> Result; - async fn write(&self, key: &'a str, data: &'a Bytes) -> Result<()>; + async fn write<'a>(&self, key: &'a str, data: &'a Bytes) -> Result<()>; + fn path(&self) -> &str; fn read_error(key: &str, e: impl ToString) -> SharedError { SharedError::Storage(StorageError::Read(key.into(), e.to_string())) diff --git a/quadratic-rust-shared/src/storage/s3.rs b/quadratic-rust-shared/src/storage/s3.rs index 86631acddb..672980b106 100644 --- a/quadratic-rust-shared/src/storage/s3.rs +++ b/quadratic-rust-shared/src/storage/s3.rs @@ -9,19 +9,18 @@ use crate::{ }; #[derive(Debug)] -pub struct S3Config<'a> { +pub struct S3Config { pub client: Client, - pub bucket: &'a str, + pub bucket: String, } -pub struct S3<'a> { - pub config: S3Config<'a>, +#[derive(Debug)] +pub struct S3 { + pub config: S3Config, } #[async_trait] -impl<'a> Storage<'a> for S3<'a> { - type Config = S3Config<'a>; - +impl Storage for S3 { async fn read(&self, key: &str) -> Result { let S3Config { client, bucket } = &self.config; @@ -39,7 +38,7 @@ impl<'a> Storage<'a> for S3<'a> { Ok(bytes) } - async fn write(&self, key: &'a str, data: &'a Bytes) -> Result<()> { + async fn write<'a>(&self, key: &'a str, data: &'a Bytes) -> Result<()> { let S3Config { client, bucket } = &self.config; upload_object(client, bucket, key, data) @@ -48,6 +47,16 @@ impl<'a> Storage<'a> for S3<'a> { Ok(()) } + + fn path(&self) -> &str { + &self.config.bucket + } +} + +impl S3 { + pub fn new(config: S3Config) -> Self { + Self { config } + } } #[cfg(test)] From e0f760cd764602a388d46b8a4af0e41247639057 Mon Sep 17 00:00:00 2001 From: David DiMaria Date: Tue, 30 Jul 2024 16:42:24 -0600 Subject: [PATCH 003/113] Serve files from the file service --- quadratic-files/.env.docker | 5 +- quadratic-files/.env.example | 5 +- quadratic-files/.env.test | 5 +- quadratic-files/src/auth.rs | 71 +++++++++++++++++++ quadratic-files/src/config.rs | 9 +-- quadratic-files/src/error.rs | 29 +++++++- quadratic-files/src/main.rs | 2 + quadratic-files/src/server.rs | 39 +++++++++- quadratic-files/src/state/mod.rs | 5 +- quadratic-files/src/state/settings.rs | 7 +- quadratic-files/src/storage.rs | 1 + quadratic-files/src/test_util.rs | 2 +- .../0.grid | 1 + .../1.grid | 1 + quadratic-files/storage/hello.txt | 1 + 15 files changed, 164 insertions(+), 19 deletions(-) create mode 100644 quadratic-files/src/auth.rs create mode 100644 quadratic-files/src/storage.rs create mode 100644 quadratic-files/storage/4666c467-0147-4510-87e2-23afdf24c82d/0.grid create mode 100644 quadratic-files/storage/4666c467-0147-4510-87e2-23afdf24c82d/1.grid create mode 100644 quadratic-files/storage/hello.txt diff --git a/quadratic-files/.env.docker b/quadratic-files/.env.docker index 0276b02d33..17c4c9e751 100644 --- a/quadratic-files/.env.docker +++ b/quadratic-files/.env.docker @@ -6,6 +6,7 @@ TRUNCATE_FILE_CHECK_S=3600 # 1 hour TRUNCATE_TRANSACTION_AGE_DAYS=5 # 5 days ENVIRONMENT=docker +AUTH0_JWKS_URI=https://dev-nje7dw8s.us.auth0.com/.well-known/jwks.json QUADRATIC_API_URI=http://quadratic-api:8000 M2M_AUTH_TOKEN=M2M_AUTH_TOKEN @@ -16,7 +17,7 @@ PUBSUB_ACTIVE_CHANNELS=active_channels PUBSUB_PROCESSED_TRANSACTIONS_CHANNEL=processed_transactions # Storage -STORAGE_TYPE=s3 # s3 or filesystem +STORAGE_TYPE=s3 # s3 or file-system # Storage: s3 AWS_S3_REGION= @@ -25,4 +26,4 @@ AWS_S3_ACCESS_KEY_ID= AWS_S3_SECRET_ACCESS_KEY= # Storage: filesystem -FILE_DIR=/storage \ No newline at end of file +STORAGE_DIR=./storage \ No newline at end of file diff --git a/quadratic-files/.env.example b/quadratic-files/.env.example index 2eb0159547..addc0f9bcb 100644 --- a/quadratic-files/.env.example +++ b/quadratic-files/.env.example @@ -6,6 +6,7 @@ FILES_PER_CHECK=100 TRUNCATE_FILE_CHECK_S=3600 # 1 hour TRUNCATE_TRANSACTION_AGE_DAYS=5 # 5 days +AUTH0_JWKS_URI=https://dev-nje7dw8s.us.auth0.com/.well-known/jwks.json QUADRATIC_API_URI=http://localhost:8000 M2M_AUTH_TOKEN=M2M_AUTH_TOKEN @@ -16,7 +17,7 @@ PUBSUB_ACTIVE_CHANNELS=active_channels PUBSUB_PROCESSED_TRANSACTIONS_CHANNEL=processed_transactions # Storage -STORAGE_TYPE=s3 # s3 or filesystem +STORAGE_TYPE=s3 # s3 or file-system # Storage: s3 AWS_S3_REGION=us-east-2 @@ -25,4 +26,4 @@ AWS_S3_ACCESS_KEY_ID=test AWS_S3_SECRET_ACCESS_KEY=test # Storage: filesystem -FILE_DIR=/storage \ No newline at end of file +STORAGE_DIR=./storage \ No newline at end of file diff --git a/quadratic-files/.env.test b/quadratic-files/.env.test index 927fcc3716..380e4bcac3 100644 --- a/quadratic-files/.env.test +++ b/quadratic-files/.env.test @@ -6,6 +6,7 @@ TRUNCATE_FILE_CHECK_S=3600 # 1 hour TRUNCATE_TRANSACTION_AGE_DAYS=5 # 5 days ENVIRONMENT=test +AUTH0_JWKS_URI=https://dev-nje7dw8s.us.auth0.com/.well-known/jwks.json QUADRATIC_API_URI=http://localhost:8000 M2M_AUTH_TOKEN=M2M_AUTH_TOKEN @@ -16,7 +17,7 @@ PUBSUB_ACTIVE_CHANNELS=active_channels PUBSUB_PROCESSED_TRANSACTIONS_CHANNEL=processed_transactions # Storage -STORAGE_TYPE=s3 # s3 or filesystem +STORAGE_TYPE=s3 # s3 or file-system # Storage: s3 AWS_S3_REGION= @@ -25,4 +26,4 @@ AWS_S3_ACCESS_KEY_ID= AWS_S3_SECRET_ACCESS_KEY= # Storage: filesystem -FILE_DIR=/storage \ No newline at end of file +STORAGE_DIR=./storage \ No newline at end of file diff --git a/quadratic-files/src/auth.rs b/quadratic-files/src/auth.rs new file mode 100644 index 0000000000..66b7708d00 --- /dev/null +++ b/quadratic-files/src/auth.rs @@ -0,0 +1,71 @@ +//! Authentication and authorization middleware. +//! +//! + +use axum::{ + async_trait, + extract::{FromRef, FromRequestParts}, + http::request::Parts, + RequestPartsExt, +}; +use axum_extra::{ + headers::{authorization::Bearer, Authorization}, + TypedHeader, +}; +use jsonwebtoken::jwk::JwkSet; +use quadratic_rust_shared::auth::jwt::authorize; +use serde::{Deserialize, Serialize}; + +use crate::error::{FilesError, Result}; + +/// The claims from the Quadratic/Auth0 JWT token. +/// We need our own implementation of this because we need to impl on it. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct Claims { + pub sub: String, + pub exp: usize, +} + +/// Instance of Axum's middleware that also contains a copy of state +#[cfg(not(test))] +pub fn get_middleware(jwks: JwkSet) -> axum::middleware::FromExtractorLayer { + axum::middleware::from_extractor_with_state::(jwks) +} + +// Middleware that accepts json for tests +#[cfg(test)] +pub fn get_middleware( + _jwks: JwkSet, +) -> tower_http::validate_request::ValidateRequestHeaderLayer< + tower_http::validate_request::AcceptHeader, +> { + tower_http::validate_request::ValidateRequestHeaderLayer::accept("application/json") +} + +/// Extract the claims from the request. +/// Anytime a claims parameter is added to a handler, this will automatically +/// be called. +#[async_trait] +impl FromRequestParts for Claims +where + JwkSet: FromRef, + S: Send + Sync, +{ + type Rejection = FilesError; + + async fn from_request_parts(parts: &mut Parts, jwks: &S) -> Result { + let jwks = JwkSet::from_ref(jwks); + // Extract the token from the authorization header + let TypedHeader(Authorization(bearer)) = parts + .extract::>>() + .await + .map_err(|e| FilesError::Authentication(e.to_string()))?; + + let token_data = authorize(&jwks, bearer.token(), false, true)?; + + Ok(token_data.claims) + } +} + +#[cfg(test)] +pub(crate) mod tests {} diff --git a/quadratic-files/src/config.rs b/quadratic-files/src/config.rs index 674f0a0382..b82b3c5dfe 100644 --- a/quadratic-files/src/config.rs +++ b/quadratic-files/src/config.rs @@ -35,20 +35,21 @@ pub(crate) struct Config { pub(crate) pubsub_active_channels: String, pub(crate) pubsub_processed_transactions_channel: String, + pub(crate) auth0_jwks_uri: String, pub(crate) quadratic_api_uri: String, pub(crate) m2m_auth_token: String, - // Storage Type: s3 or filesystem + // Storage Type: s3 or file-system pub(crate) storage_type: StorageType, - // StorageConfig::S3 + // StorageType::S3 pub(crate) aws_s3_region: Option, pub(crate) aws_s3_bucket_name: Option, pub(crate) aws_s3_access_key_id: Option, pub(crate) aws_s3_secret_access_key: Option, - // StorageConfig::FileSystem - pub(crate) path: Option, + // StorageType::FileSystem + pub(crate) storage_dir: Option, } /// Load the global configuration from the environment into Config. diff --git a/quadratic-files/src/error.rs b/quadratic-files/src/error.rs index 11bc5b9de4..d914b4ca50 100644 --- a/quadratic-files/src/error.rs +++ b/quadratic-files/src/error.rs @@ -5,14 +5,18 @@ //! Convert third party crate errors to application errors. //! Convert errors to responses. -use quadratic_rust_shared::{Aws, SharedError}; +use axum::{ + http::StatusCode, + response::{IntoResponse, Response}, +}; +use quadratic_rust_shared::{clean_errors, Auth, Aws, SharedError}; use serde::{Deserialize, Serialize}; use thiserror::Error; pub(crate) type Result = std::result::Result; #[derive(Error, Debug, Serialize, Deserialize, PartialEq, Clone)] -pub(crate) enum FilesError { +pub enum FilesError { #[error("Authentication error: {0}")] Authentication(String), @@ -59,9 +63,30 @@ pub(crate) enum FilesError { Unknown(String), } +// Convert FilesErrors into readable responses with appropriate status codes. +// These are the errors that are returned to the client. +impl IntoResponse for FilesError { + fn into_response(self) -> Response { + let (status, error) = match &self { + FilesError::Authentication(error) => (StatusCode::UNAUTHORIZED, clean_errors(error)), + FilesError::InternalServer(error) => { + (StatusCode::INTERNAL_SERVER_ERROR, clean_errors(error)) + } + _ => (StatusCode::INTERNAL_SERVER_ERROR, "Unknown".into()), + }; + + tracing::warn!("{} {}: {:?}", status, error, self); + + (status, error).into_response() + } +} + impl From for FilesError { fn from(error: SharedError) -> Self { match error { + SharedError::Auth(auth) => match auth { + Auth::Jwt(error) => FilesError::Authentication(error), + }, SharedError::Aws(aws) => match aws { Aws::S3(error) => FilesError::S3(error), }, diff --git a/quadratic-files/src/main.rs b/quadratic-files/src/main.rs index 880d4258c1..a2499ff47f 100644 --- a/quadratic-files/src/main.rs +++ b/quadratic-files/src/main.rs @@ -3,11 +3,13 @@ //! A file servic for that consumes transactions from a queue, applies them to //! a grid and writes them to S3. +mod auth; mod config; mod error; mod file; mod server; mod state; +mod storage; #[cfg(test)] mod test_util; mod truncate; diff --git a/quadratic-files/src/server.rs b/quadratic-files/src/server.rs index a9ecdeba7b..5cc02ed3a1 100644 --- a/quadratic-files/src/server.rs +++ b/quadratic-files/src/server.rs @@ -3,17 +3,23 @@ //! Handle bootstrapping and starting the HTTP server. Adds global state //! to be shared across all requests and threads. Adds tracing/logging. +use axum::extract::Request; use axum::http::StatusCode; use axum::response::IntoResponse; use axum::{routing::get, Extension, Router}; +use quadratic_rust_shared::auth::jwt::get_jwks; +use quadratic_rust_shared::storage::Storage; use std::time::Duration; use std::{net::SocketAddr, sync::Arc}; use tokio::time; +use tower::ServiceExt; +use tower_http::services::ServeDir; use tower_http::trace::{DefaultMakeSpan, TraceLayer}; use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt}; use crate::truncate::truncate_processed_transactions; use crate::{ + auth::get_middleware, config::config, error::{FilesError, Result}, file::process, @@ -25,11 +31,39 @@ const HEALTHCHECK_INTERVAL_S: u64 = 5; /// Construct the application router. This is separated out so that it can be /// integration tested. pub(crate) fn app(state: Arc) -> Router { + // get the auth middleware + let jwks = state + .settings + .jwks + .as_ref() + .expect("JWKS not found in state") + .to_owned(); + let auth = get_middleware(jwks); + let path = state.settings.storage.path().to_owned(); + + tracing::info!("Serving files from {path}"); + Router::new() - // routes + // protected routes + // + // + .nest_service( + "/storage", + get(|request: Request| async { + let service = ServeDir::new("storage"); + service.oneshot(request).await + }), + ) + // + // auth middleware + .route_layer(auth) + // + // unprotected routes .route("/health", get(healthcheck)) + // // state .layer(Extension(state)) + // // logger .layer( TraceLayer::new_for_http() @@ -49,7 +83,8 @@ pub(crate) async fn serve() -> Result<()> { .init(); let config = config()?; - let state = Arc::new(State::new(&config).await?); + let jwks = get_jwks(&config.auth0_jwks_uri).await?; + let state = Arc::new(State::new(&config, Some(jwks)).await?); let app = app(Arc::clone(&state)); let listener = tokio::net::TcpListener::bind(format!("{}:{}", config.host, config.port)) diff --git a/quadratic-files/src/state/mod.rs b/quadratic-files/src/state/mod.rs index cb6bc80108..b850330412 100644 --- a/quadratic-files/src/state/mod.rs +++ b/quadratic-files/src/state/mod.rs @@ -7,6 +7,7 @@ pub mod pubsub; pub mod settings; pub mod stats; +use jsonwebtoken::jwk::JwkSet; use quadratic_rust_shared::pubsub::redis_streams::RedisStreamsConfig; use quadratic_rust_shared::pubsub::Config as PubSubConfig; use tokio::sync::Mutex; @@ -26,7 +27,7 @@ pub(crate) struct State { } impl State { - pub(crate) async fn new(config: &Config) -> Result { + pub(crate) async fn new(config: &Config, jwks: Option) -> Result { let pubsub_config = PubSubConfig::RedisStreams(RedisStreamsConfig { host: config.pubsub_host.to_owned(), port: config.pubsub_port.to_owned(), @@ -36,7 +37,7 @@ impl State { Ok(State { pubsub: Mutex::new(PubSub::new(pubsub_config).await?), - settings: Settings::new(config).await, + settings: Settings::new(config, jwks).await, stats: Mutex::new(Stats::new()), }) } diff --git a/quadratic-files/src/state/settings.rs b/quadratic-files/src/state/settings.rs index aa3210f59c..891eee25ba 100644 --- a/quadratic-files/src/state/settings.rs +++ b/quadratic-files/src/state/settings.rs @@ -1,3 +1,4 @@ +use jsonwebtoken::jwk::JwkSet; use quadratic_rust_shared::aws::client; use quadratic_rust_shared::environment::Environment; use quadratic_rust_shared::storage::file_system::{FileSystem, FileSystemConfig}; @@ -8,6 +9,7 @@ use crate::config::{Config, StorageType}; #[derive(Debug)] pub(crate) struct Settings { + pub(crate) jwks: Option, pub(crate) quadratic_api_uri: String, pub(crate) m2m_auth_token: String, pub(crate) storage: StorageContainer, @@ -17,7 +19,7 @@ pub(crate) struct Settings { impl Settings { // Create a new Settings struct from the provided Config. // Panics are OK here since this is set at startup and we want to fail fast. - pub(crate) async fn new(config: &Config) -> Self { + pub(crate) async fn new(config: &Config, jwks: Option) -> Self { let is_local = config.environment == Environment::Docker || config.environment == Environment::Local; let expected = |val: &Option, var: &str| { @@ -39,12 +41,13 @@ impl Settings { })), StorageType::FileSystem => { StorageContainer::FileSystem(FileSystem::new(FileSystemConfig { - path: expected(&config.path, "FILE_DIR"), + path: expected(&config.storage_dir, "STORAGE_DIR"), })) } }; Settings { + jwks, quadratic_api_uri: config.quadratic_api_uri.to_owned(), m2m_auth_token: config.m2m_auth_token.to_owned(), storage, diff --git a/quadratic-files/src/storage.rs b/quadratic-files/src/storage.rs new file mode 100644 index 0000000000..8b13789179 --- /dev/null +++ b/quadratic-files/src/storage.rs @@ -0,0 +1 @@ + diff --git a/quadratic-files/src/test_util.rs b/quadratic-files/src/test_util.rs index 953693c847..d1d33932ca 100644 --- a/quadratic-files/src/test_util.rs +++ b/quadratic-files/src/test_util.rs @@ -12,7 +12,7 @@ use crate::state::State; pub(crate) async fn new_state() -> State { let config = config().unwrap(); - State::new(&config).await.unwrap() + State::new(&config, None).await.unwrap() } pub(crate) async fn new_arc_state() -> Arc { diff --git a/quadratic-files/storage/4666c467-0147-4510-87e2-23afdf24c82d/0.grid b/quadratic-files/storage/4666c467-0147-4510-87e2-23afdf24c82d/0.grid new file mode 100644 index 0000000000..50a1f90d7e --- /dev/null +++ b/quadratic-files/storage/4666c467-0147-4510-87e2-23afdf24c82d/0.grid @@ -0,0 +1 @@ +{"sheets":[{"name":"Sheet 1","id":{"id":"eeccf824-7b81-4644-a939-0dea8d195fe4"},"order":"a0","cells":[],"code_cells":[],"formats":[],"columns":[],"rows":[],"offsets":[[],[]],"borders":{}}],"version":"1.4"} \ No newline at end of file diff --git a/quadratic-files/storage/4666c467-0147-4510-87e2-23afdf24c82d/1.grid b/quadratic-files/storage/4666c467-0147-4510-87e2-23afdf24c82d/1.grid new file mode 100644 index 0000000000..f38ca0b1ae --- /dev/null +++ b/quadratic-files/storage/4666c467-0147-4510-87e2-23afdf24c82d/1.grid @@ -0,0 +1 @@ +{"sheets":[{"id":{"id":"eeccf824-7b81-4644-a939-0dea8d195fe4"},"name":"Sheet 1","color":null,"order":"a0","offsets":[[],[]],"columns":[[0,{"values":{"0":{"Text":"s"}},"align":{},"wrap":{},"numeric_format":{},"numeric_decimals":{},"numeric_commas":{},"bold":{},"italic":{},"text_color":{},"fill_color":{},"render_size":{}}]],"borders":{},"code_runs":[]}],"version":"1.5"} \ No newline at end of file diff --git a/quadratic-files/storage/hello.txt b/quadratic-files/storage/hello.txt new file mode 100644 index 0000000000..6769dd60bd --- /dev/null +++ b/quadratic-files/storage/hello.txt @@ -0,0 +1 @@ +Hello world! \ No newline at end of file From 26a5789ffeee6cdeac119094cfe04d350ad60370 Mon Sep 17 00:00:00 2001 From: David DiMaria Date: Tue, 30 Jul 2024 16:43:35 -0600 Subject: [PATCH 004/113] Abstract jwt and error functionality --- quadratic-connection/src/error.rs | 13 +------------ quadratic-connection/src/server.rs | 24 ++++-------------------- quadratic-rust-shared/src/auth/jwt.rs | 12 +++++++++++- quadratic-rust-shared/src/error.rs | 11 +++++++++++ 4 files changed, 27 insertions(+), 33 deletions(-) diff --git a/quadratic-connection/src/error.rs b/quadratic-connection/src/error.rs index cb9e218881..d1ea9598a1 100644 --- a/quadratic-connection/src/error.rs +++ b/quadratic-connection/src/error.rs @@ -9,7 +9,7 @@ use axum::{ http::StatusCode, response::{IntoResponse, Response}, }; -use quadratic_rust_shared::SharedError; +use quadratic_rust_shared::{clean_errors, SharedError}; use serde::{Deserialize, Serialize}; use thiserror::Error; @@ -52,17 +52,6 @@ pub(crate) fn proxy_error(e: impl ToString) -> ConnectionError { ConnectionError::Proxy(e.to_string()) } -fn clean_errors(error: impl ToString) -> String { - let mut cleaned = error.to_string(); - let remove = vec!["error returned from database: "]; - - for r in remove { - cleaned = format!("{:?}", cleaned).replace(r, ""); - } - - cleaned -} - impl From for ConnectionError { fn from(error: SharedError) -> Self { match error { diff --git a/quadratic-connection/src/server.rs b/quadratic-connection/src/server.rs index b01f75a46a..1e32fdb9cc 100644 --- a/quadratic-connection/src/server.rs +++ b/quadratic-connection/src/server.rs @@ -4,17 +4,15 @@ //! to be shared across all requests and threads. Adds tracing/logging. use axum::{ - http::{header::AUTHORIZATION, Method, StatusCode}, - response::IntoResponse, + http::{header::AUTHORIZATION, Method}, routing::{any, get, post}, Extension, Json, Router, }; -use jsonwebtoken::jwk::JwkSet; use quadratic_rust_shared::auth::jwt::get_jwks; use quadratic_rust_shared::sql::Connection; use serde::{Deserialize, Serialize}; use std::{iter::once, time::Duration}; -use tokio::{sync::OnceCell, time}; +use tokio::time; use tower_http::{ cors::{Any, CorsLayer}, sensitive_headers::SetSensitiveHeadersLayer, @@ -37,21 +35,6 @@ use crate::{ const HEALTHCHECK_INTERVAL_S: u64 = 5; -static JWKS: OnceCell = OnceCell::const_new(); - -/// Get the constant JWKS for use throughout the application -/// The panics are intentional and will happen at startup -pub(crate) async fn get_const_jwks() -> &'static JwkSet { - JWKS.get_or_init(|| async { - let config = config().expect("Invalid config"); - - get_jwks(&config.auth0_jwks_uri) - .await - .expect("Unable to get JWKS") - }) - .await -} - #[derive(Serialize, Deserialize)] pub(crate) struct SqlQuery { pub(crate) query: String, @@ -159,7 +142,7 @@ pub(crate) async fn serve() -> Result<()> { .init(); let config = config()?; - let jwks = get_const_jwks().await; + let jwks = get_jwks(&config.auth0_jwks_uri).await?; let state = State::new(&config, Some(jwks.clone()))?; let app = app(state.clone())?; @@ -238,6 +221,7 @@ pub(crate) mod tests { body::Body, http::{self, Request}, }; + use http::StatusCode; use tower::ServiceExt; use super::*; diff --git a/quadratic-rust-shared/src/auth/jwt.rs b/quadratic-rust-shared/src/auth/jwt.rs index 90ec2afac8..fa4cf84f33 100644 --- a/quadratic-rust-shared/src/auth/jwt.rs +++ b/quadratic-rust-shared/src/auth/jwt.rs @@ -1,10 +1,20 @@ -use jsonwebtoken::jwk::AlgorithmParameters; +use jsonwebtoken::jwk::{AlgorithmParameters, JwkSet}; use jsonwebtoken::{decode, decode_header, jwk, Algorithm, DecodingKey, TokenData, Validation}; use serde::de::DeserializeOwned; use std::str::FromStr; +use tokio::sync::OnceCell; use crate::error::{Auth, Result, SharedError}; +pub static JWKS: OnceCell = OnceCell::const_new(); + +/// Get the constant JWKS for use throughout the application +/// The panics are intentional and will happen at startup +pub async fn get_const_jwks(jwks_uri: &str) -> &'static JwkSet { + JWKS.get_or_init(|| async { get_jwks(jwks_uri).await.expect("Unable to get JWKS") }) + .await +} + /// Get the JWK set from a given URL. pub async fn get_jwks(url: &str) -> Result { let jwks = reqwest::get(url).await?.json::().await?; diff --git a/quadratic-rust-shared/src/error.rs b/quadratic-rust-shared/src/error.rs index c435d8d99f..078f420d90 100644 --- a/quadratic-rust-shared/src/error.rs +++ b/quadratic-rust-shared/src/error.rs @@ -91,6 +91,17 @@ pub enum SharedError { Uuid(String), } +pub fn clean_errors(error: impl ToString) -> String { + let mut cleaned = error.to_string(); + let remove = vec!["error returned from database: "]; + + for r in remove { + cleaned = format!("{:?}", cleaned).replace(r, ""); + } + + cleaned +} + impl From for SharedError { fn from(error: redis::RedisError) -> Self { SharedError::PubSub(error.to_string()) From e7a0eeab0ee80258ed4cab285dfd283acc605c4e Mon Sep 17 00:00:00 2001 From: David DiMaria Date: Tue, 30 Jul 2024 16:44:30 -0600 Subject: [PATCH 005/113] Ignore storage contents --- .gitignore | 1 + .../storage/4666c467-0147-4510-87e2-23afdf24c82d/0.grid | 1 - .../storage/4666c467-0147-4510-87e2-23afdf24c82d/1.grid | 1 - 3 files changed, 1 insertion(+), 2 deletions(-) delete mode 100644 quadratic-files/storage/4666c467-0147-4510-87e2-23afdf24c82d/0.grid delete mode 100644 quadratic-files/storage/4666c467-0147-4510-87e2-23afdf24c82d/1.grid diff --git a/.gitignore b/.gitignore index 3297816eda..475abc922a 100644 --- a/.gitignore +++ b/.gitignore @@ -40,6 +40,7 @@ quadratic-connection/target/ quadratic-core/target/ quadratic-core/tmp.txt quadratic-files/target/ +quadratic-files/storage quadratic-multiplayer/target/ quadratic-multiplayer/updateAlertVersion.json diff --git a/quadratic-files/storage/4666c467-0147-4510-87e2-23afdf24c82d/0.grid b/quadratic-files/storage/4666c467-0147-4510-87e2-23afdf24c82d/0.grid deleted file mode 100644 index 50a1f90d7e..0000000000 --- a/quadratic-files/storage/4666c467-0147-4510-87e2-23afdf24c82d/0.grid +++ /dev/null @@ -1 +0,0 @@ -{"sheets":[{"name":"Sheet 1","id":{"id":"eeccf824-7b81-4644-a939-0dea8d195fe4"},"order":"a0","cells":[],"code_cells":[],"formats":[],"columns":[],"rows":[],"offsets":[[],[]],"borders":{}}],"version":"1.4"} \ No newline at end of file diff --git a/quadratic-files/storage/4666c467-0147-4510-87e2-23afdf24c82d/1.grid b/quadratic-files/storage/4666c467-0147-4510-87e2-23afdf24c82d/1.grid deleted file mode 100644 index f38ca0b1ae..0000000000 --- a/quadratic-files/storage/4666c467-0147-4510-87e2-23afdf24c82d/1.grid +++ /dev/null @@ -1 +0,0 @@ -{"sheets":[{"id":{"id":"eeccf824-7b81-4644-a939-0dea8d195fe4"},"name":"Sheet 1","color":null,"order":"a0","offsets":[[],[]],"columns":[[0,{"values":{"0":{"Text":"s"}},"align":{},"wrap":{},"numeric_format":{},"numeric_decimals":{},"numeric_commas":{},"bold":{},"italic":{},"text_color":{},"fill_color":{},"render_size":{}}]],"borders":{},"code_runs":[]}],"version":"1.5"} \ No newline at end of file From b8c1fe44e044c7d8a169a3b287d01f27b2308356 Mon Sep 17 00:00:00 2001 From: David DiMaria Date: Tue, 30 Jul 2024 16:45:29 -0600 Subject: [PATCH 006/113] Rename aws dir to storage --- quadratic-api/src/routes/v0/files.$uuid.GET.ts | 2 +- quadratic-api/src/routes/v0/files.$uuid.thumbnail.POST.ts | 2 +- quadratic-api/src/routes/v0/files.GET.ts | 2 +- quadratic-api/src/routes/v0/teams.$uuid.GET.ts | 2 +- quadratic-api/src/{aws => storage}/s3.ts | 0 quadratic-api/src/utils/createFile.ts | 2 +- 6 files changed, 5 insertions(+), 5 deletions(-) rename quadratic-api/src/{aws => storage}/s3.ts (100%) diff --git a/quadratic-api/src/routes/v0/files.$uuid.GET.ts b/quadratic-api/src/routes/v0/files.$uuid.GET.ts index 3473a6f335..f040251d94 100644 --- a/quadratic-api/src/routes/v0/files.$uuid.GET.ts +++ b/quadratic-api/src/routes/v0/files.$uuid.GET.ts @@ -1,12 +1,12 @@ import { Response } from 'express'; import { ApiTypes } from 'quadratic-shared/typesAndSchemas'; import z from 'zod'; -import { generatePresignedUrl } from '../../aws/s3'; import dbClient from '../../dbClient'; import { getFile } from '../../middleware/getFile'; import { userOptionalMiddleware } from '../../middleware/user'; import { validateOptionalAccessToken } from '../../middleware/validateOptionalAccessToken'; import { validateRequestSchema } from '../../middleware/validateRequestSchema'; +import { generatePresignedUrl } from '../../storage/s3'; import { RequestWithOptionalUser } from '../../types/Request'; import { ResponseError } from '../../types/Response'; diff --git a/quadratic-api/src/routes/v0/files.$uuid.thumbnail.POST.ts b/quadratic-api/src/routes/v0/files.$uuid.thumbnail.POST.ts index 722ae7b737..e901b98dd8 100644 --- a/quadratic-api/src/routes/v0/files.$uuid.thumbnail.POST.ts +++ b/quadratic-api/src/routes/v0/files.$uuid.thumbnail.POST.ts @@ -3,13 +3,13 @@ import multer, { StorageEngine } from 'multer'; import multerS3 from 'multer-s3'; import { ApiTypes, FilePermissionSchema } from 'quadratic-shared/typesAndSchemas'; import z from 'zod'; -import { s3Client } from '../../aws/s3'; import dbClient from '../../dbClient'; import { AWS_S3_BUCKET_NAME } from '../../env-vars'; import { getFile } from '../../middleware/getFile'; import { userMiddleware } from '../../middleware/user'; import { validateAccessToken } from '../../middleware/validateAccessToken'; import { validateRequestSchema } from '../../middleware/validateRequestSchema'; +import { s3Client } from '../../storage/s3'; import { RequestWithFile, RequestWithUser } from '../../types/Request'; const { FILE_EDIT } = FilePermissionSchema.enum; diff --git a/quadratic-api/src/routes/v0/files.GET.ts b/quadratic-api/src/routes/v0/files.GET.ts index 1f51c9119c..0a3f76b6a6 100644 --- a/quadratic-api/src/routes/v0/files.GET.ts +++ b/quadratic-api/src/routes/v0/files.GET.ts @@ -1,9 +1,9 @@ import { Response } from 'express'; import { ApiTypes } from 'quadratic-shared/typesAndSchemas'; -import { generatePresignedUrl } from '../../aws/s3'; import dbClient from '../../dbClient'; import { userMiddleware } from '../../middleware/user'; import { validateAccessToken } from '../../middleware/validateAccessToken'; +import { generatePresignedUrl } from '../../storage/s3'; import { RequestWithUser } from '../../types/Request'; import { ResponseError } from '../../types/Response'; diff --git a/quadratic-api/src/routes/v0/teams.$uuid.GET.ts b/quadratic-api/src/routes/v0/teams.$uuid.GET.ts index 27ab4c226e..392adfcea1 100644 --- a/quadratic-api/src/routes/v0/teams.$uuid.GET.ts +++ b/quadratic-api/src/routes/v0/teams.$uuid.GET.ts @@ -2,12 +2,12 @@ import { Request, Response } from 'express'; import { ApiTypes } from 'quadratic-shared/typesAndSchemas'; import { z } from 'zod'; import { getUsersFromAuth0 } from '../../auth0/profile'; -import { generatePresignedUrl } from '../../aws/s3'; import dbClient from '../../dbClient'; import { getTeam } from '../../middleware/getTeam'; import { userMiddleware } from '../../middleware/user'; import { validateAccessToken } from '../../middleware/validateAccessToken'; import { parseRequest } from '../../middleware/validateRequestSchema'; +import { generatePresignedUrl } from '../../storage/s3'; import { RequestWithUser } from '../../types/Request'; import { getFilePermissions } from '../../utils/permissions'; diff --git a/quadratic-api/src/aws/s3.ts b/quadratic-api/src/storage/s3.ts similarity index 100% rename from quadratic-api/src/aws/s3.ts rename to quadratic-api/src/storage/s3.ts diff --git a/quadratic-api/src/utils/createFile.ts b/quadratic-api/src/utils/createFile.ts index 9bffc40ffd..ca39af2d9a 100644 --- a/quadratic-api/src/utils/createFile.ts +++ b/quadratic-api/src/utils/createFile.ts @@ -1,5 +1,5 @@ -import { uploadStringAsFileS3 } from '../aws/s3'; import dbClient from '../dbClient'; +import { uploadStringAsFileS3 } from '../storage/s3'; export async function createFile({ contents, From 550b484890b76007773327f83133d6491c508672 Mon Sep 17 00:00:00 2001 From: David DiMaria Date: Thu, 1 Aug 2024 09:43:41 -0600 Subject: [PATCH 007/113] Handle thumbnail uploads, start presigned URL work --- .gitignore | 1 + Cargo.lock | 2 + quadratic-api/.env.docker | 4 + quadratic-api/.env.example | 13 ++- quadratic-api/.env.test | 15 ++- quadratic-api/src/env-vars.ts | 15 ++- quadratic-api/src/routes/v0/examples.POST.ts | 7 ++ .../src/routes/v0/files.$uuid.GET.ts | 6 +- .../routes/v0/files.$uuid.thumbnail.POST.ts | 23 +---- quadratic-api/src/routes/v0/files.GET.ts | 4 +- quadratic-api/src/routes/v0/files.POST.ts | 8 +- .../src/routes/v0/teams.$uuid.GET.ts | 4 +- quadratic-api/src/storage/fileSystem.ts | 93 +++++++++++++++++++ quadratic-api/src/storage/s3.ts | 23 ++++- quadratic-api/src/storage/storage.ts | 53 +++++++++++ quadratic-api/src/utils/createFile.ts | 6 +- .../web-workers/quadraticCore/worker/core.ts | 7 +- quadratic-client/src/shared/api/apiClient.ts | 1 - quadratic-files/.env.docker | 4 +- quadratic-files/.env.example | 4 +- quadratic-files/.env.test | 4 +- quadratic-files/Cargo.toml | 5 +- quadratic-files/src/error.rs | 20 ++-- quadratic-files/src/server.rs | 48 +++++++--- quadratic-files/src/storage.rs | 53 +++++++++++ quadratic-files/storage/hello.txt | 1 - quadratic-rust-shared/src/storage/mod.rs | 30 +++--- 27 files changed, 359 insertions(+), 95 deletions(-) create mode 100644 quadratic-api/src/storage/fileSystem.ts create mode 100644 quadratic-api/src/storage/storage.ts delete mode 100644 quadratic-files/storage/hello.txt diff --git a/.gitignore b/.gitignore index 475abc922a..7bb5dd529a 100644 --- a/.gitignore +++ b/.gitignore @@ -66,6 +66,7 @@ docker/redis/data docker/postgres/data docker/postgres-connection/data docker/mysql-connection/data +docker/file-storage # JMeter jmeter.log diff --git a/Cargo.lock b/Cargo.lock index fa0f44e861..347f4b994c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -805,6 +805,7 @@ checksum = "3a6c9af12842a67734c9a2e355436e5d03b22383ed60cf13cd0c18fbfe3dcbcf" dependencies = [ "async-trait", "axum-core", + "axum-macros", "base64 0.21.7", "bytes", "futures-util", @@ -3170,6 +3171,7 @@ dependencies = [ "strum_macros 0.25.3", "thiserror", "tokio", + "tokio-util", "tower", "tower-http", "tracing", diff --git a/quadratic-api/.env.docker b/quadratic-api/.env.docker index e0f5ac53d8..a28e8b7f5a 100644 --- a/quadratic-api/.env.docker +++ b/quadratic-api/.env.docker @@ -3,3 +3,7 @@ DATABASE_URL='postgresql://postgres:postgres@postgres:5432/postgres' # Hex string to be used as the key for enctyption, use npm run key:generate ENCRYPTION_KEY=eb4758047f74bdb2603cce75c4370327ca2c3662c4786867659126da8e64dfcc + +# Storage +STORAGE_TYPE=file-system +QUADRATIC_FILE_URI=http://127.0.0.1:3002 \ No newline at end of file diff --git a/quadratic-api/.env.example b/quadratic-api/.env.example index 986edfc86a..2eaf2b05c1 100644 --- a/quadratic-api/.env.example +++ b/quadratic-api/.env.example @@ -11,11 +11,6 @@ AUTH0_CLIENT_ID=DCPCvqyU5Q0bJD8Q3QmJEoV48x1zLH7W AUTH0_CLIENT_SECRET=94dp3PDcxlI9ZDqBSvkdjQHWgGdx0ZSeyTr5-Rn3Kcts-ZyTdj1FLlJjCyqrTXEG AUTH0_AUDIENCE=community-quadratic -AWS_S3_REGION=us-east-2 -AWS_S3_ACCESS_KEY_ID=test -AWS_S3_SECRET_ACCESS_KEY=test -AWS_S3_BUCKET_NAME=quadratic-api-docker - OPENAI_API_KEY= SENTRY_DSN= @@ -26,3 +21,11 @@ STRIPE_SECRET_KEY=STRIPE_SECRET_KEY STRIPE_WEBHOOK_SECRET=STRIPE_WEBHOOK_SECRET ENCRYPTION_KEY=eb4758047f74bdb2603cce75c4370327ca2c3662c4786867659126da8e64dfcc + +# Storage +STORAGE_TYPE=file-system # s3 or file-system +QUADRATIC_FILE_URI=http://127.0.0.1:3002 +AWS_S3_REGION=us-east-2 +AWS_S3_ACCESS_KEY_ID=test +AWS_S3_SECRET_ACCESS_KEY=test +AWS_S3_BUCKET_NAME=quadratic-api-docker diff --git a/quadratic-api/.env.test b/quadratic-api/.env.test index 0aa0efdebc..1c3a71065d 100644 --- a/quadratic-api/.env.test +++ b/quadratic-api/.env.test @@ -4,13 +4,20 @@ AUTH0_ISSUER='https://auth-dev.quadratic.to/' AUTH0_CLIENT_ID="AUTH0_CLIENT_ID" AUTH0_CLIENT_SECRET="AUTH0_CLIENT_SECRET" AUTH0_DOMAIN="AUTH0_DOMAIN" -AWS_S3_REGION=us-west-2 -AWS_S3_BUCKET_NAME=AWS_S3_BUCKET_NAME -AWS_S3_ACCESS_KEY_ID=AWS_S3_ACCESS_KEY_ID -AWS_S3_SECRET_ACCESS_KEY=AWS_S3_SECRET_ACCESS_KEY M2M_AUTH_TOKEN=M2M_AUTH_TOKEN STRIPE_SECRET_KEY=STRIPE_SECRET_KEY STRIPE_WEBHOOK_SECRET=STRIPE_WEBHOOK_SECRET # Hex string to be used as the key for enctyption, use npm run key:generate ENCRYPTION_KEY=eb4758047f74bdb2603cce75c4370327ca2c3662c4786867659126da8e64dfcc + +# Storage +STORAGE_TYPE=file-system # s3 or file-system +QUADRATIC_FILE_URI=http://127.0.0.1:3002 +AWS_S3_REGION=us-west-2 +AWS_S3_BUCKET_NAME=AWS_S3_BUCKET_NAME +AWS_S3_ACCESS_KEY_ID=AWS_S3_ACCESS_KEY_ID +AWS_S3_SECRET_ACCESS_KEY=AWS_S3_SECRET_ACCESS_KEY +AWS_S3_ENDPOINT=http://0.0.0.0:4566 + + diff --git a/quadratic-api/src/env-vars.ts b/quadratic-api/src/env-vars.ts index 018f528800..26ea45ed45 100644 --- a/quadratic-api/src/env-vars.ts +++ b/quadratic-api/src/env-vars.ts @@ -9,6 +9,11 @@ export const SENTRY_DSN = process.env.SENTRY_DSN; export const NODE_ENV = process.env.NODE_ENV || 'development'; export const PORT = process.env.PORT || 8000; export const ENVIRONMENT = process.env.ENVIRONMENT; +export const QUADRATIC_FILE_URI = process.env.QUADRATIC_FILE_URI as string; +export const AWS_S3_REGION = process.env.AWS_S3_REGION as string; +export const AWS_S3_ACCESS_KEY_ID = process.env.AWS_S3_ACCESS_KEY_ID as string; +export const AWS_S3_SECRET_ACCESS_KEY = process.env.AWS_S3_SECRET_ACCESS_KEY as string; +export const AWS_S3_BUCKET_NAME = process.env.AWS_S3_BUCKET_NAME as string; // Required export const AUTH0_DOMAIN = process.env.AUTH0_DOMAIN as string; @@ -17,12 +22,9 @@ export const AUTH0_CLIENT_SECRET = process.env.AUTH0_CLIENT_SECRET as string; export const AUTH0_JWKS_URI = process.env.AUTH0_JWKS_URI as string; export const AUTH0_ISSUER = process.env.AUTH0_ISSUER as string; export const AUTH0_AUDIENCE = process.env.AUTH0_AUDIENCE as string; -export const AWS_S3_REGION = process.env.AWS_S3_REGION as string; -export const AWS_S3_ACCESS_KEY_ID = process.env.AWS_S3_ACCESS_KEY_ID as string; -export const AWS_S3_SECRET_ACCESS_KEY = process.env.AWS_S3_SECRET_ACCESS_KEY as string; -export const AWS_S3_BUCKET_NAME = process.env.AWS_S3_BUCKET_NAME as string; export const STRIPE_SECRET_KEY = process.env.STRIPE_SECRET_KEY as string; export const ENCRYPTION_KEY = process.env.ENCRYPTION_KEY as string; +export const STORAGE_TYPE = process.env.STORAGE_TYPE as string; [ 'AUTH0_DOMAIN', 'AUTH0_CLIENT_ID', @@ -30,10 +32,7 @@ export const ENCRYPTION_KEY = process.env.ENCRYPTION_KEY as string; 'AUTH0_JWKS_URI', 'AUTH0_ISSUER', 'AUTH0_AUDIENCE', - 'AWS_S3_REGION', - 'AWS_S3_ACCESS_KEY_ID', - 'AWS_S3_SECRET_ACCESS_KEY', - 'AWS_S3_BUCKET_NAME', + 'STORAGE_TYPE', 'STRIPE_SECRET_KEY', 'ENCRYPTION_KEY', ].forEach(ensureEnvVarExists); diff --git a/quadratic-api/src/routes/v0/examples.POST.ts b/quadratic-api/src/routes/v0/examples.POST.ts index ce48e3ab3e..f01227b824 100644 --- a/quadratic-api/src/routes/v0/examples.POST.ts +++ b/quadratic-api/src/routes/v0/examples.POST.ts @@ -27,6 +27,12 @@ async function handler(req: RequestWithUser, res: Response void) => { - cb(null, { fieldName: file.fieldname }); - }, - key: (req: Request, file: Express.Multer.File, cb: (error: Error | null, key: string) => void) => { - const fileUuid = req.params.uuid; - cb(null, `${fileUuid}-${file.originalname}`); - }, - }) as StorageEngine, -}); - async function handler(req: RequestWithUser & RequestWithFile, res: Response) { const { params: { uuid }, @@ -65,6 +48,6 @@ export default [ ), validateAccessToken, userMiddleware, - uploadThumbnailToS3.single('thumbnail'), + uploadMiddleware().single('thumbnail'), handler, ]; diff --git a/quadratic-api/src/routes/v0/files.GET.ts b/quadratic-api/src/routes/v0/files.GET.ts index 0a3f76b6a6..71be69ef34 100644 --- a/quadratic-api/src/routes/v0/files.GET.ts +++ b/quadratic-api/src/routes/v0/files.GET.ts @@ -3,7 +3,7 @@ import { ApiTypes } from 'quadratic-shared/typesAndSchemas'; import dbClient from '../../dbClient'; import { userMiddleware } from '../../middleware/user'; import { validateAccessToken } from '../../middleware/validateAccessToken'; -import { generatePresignedUrl } from '../../storage/s3'; +import { getFileUrl } from '../../storage/storage'; import { RequestWithUser } from '../../types/Request'; import { ResponseError } from '../../types/Response'; @@ -44,7 +44,7 @@ async function handler(req: RequestWithUser, res: Response { if (file.thumbnail) { - file.thumbnail = await generatePresignedUrl(file.thumbnail); + file.thumbnail = await getFileUrl(file.thumbnail); } }) ); diff --git a/quadratic-api/src/routes/v0/files.POST.ts b/quadratic-api/src/routes/v0/files.POST.ts index b10b45c31c..12b5e63d1b 100644 --- a/quadratic-api/src/routes/v0/files.POST.ts +++ b/quadratic-api/src/routes/v0/files.POST.ts @@ -23,6 +23,12 @@ async function handler(req: RequestWithUser, res: Response { if (file.thumbnail) { - file.thumbnail = await generatePresignedUrl(file.thumbnail); + file.thumbnail = await getPresignedFileUrl(file.thumbnail); } }) ); diff --git a/quadratic-api/src/storage/fileSystem.ts b/quadratic-api/src/storage/fileSystem.ts new file mode 100644 index 0000000000..647d9e8e81 --- /dev/null +++ b/quadratic-api/src/storage/fileSystem.ts @@ -0,0 +1,93 @@ +import { Request } from 'express'; +import multer from 'multer'; +import stream, { Readable } from 'node:stream'; +import { QUADRATIC_FILE_URI } from '../env-vars'; +import { encryptFromEnv } from '../utils/crypto'; +import { UploadFileResponse } from './storage'; + +const generateUrl = (key: string): string => `${QUADRATIC_FILE_URI}/storage/${key}`; +const generatePresignedUrl = (key: string): string => generateUrl(`presigned/${key}`); + +export const getStorageUrl = (key: string): string => { + return generateUrl(key); +}; + +export const getPresignedStorageUrl = (key: string): string => { + const encrypted = encryptFromEnv(key); + return generatePresignedUrl(encrypted); +}; + +export const upload = async (key: string, contents: string | Uint8Array, jwt: string): Promise => { + const url = generateUrl(key); + + console.warn('Uploading to', url); + + try { + const response = await fetch(url, { + method: 'POST', + body: contents, + headers: { + 'Content-Type': 'text/plain', + Authorization: `${jwt}`, + }, + }).then((res) => res.json()); + + return response; + } catch (e) { + console.error(e); + throw new Error(`Failed to upload file to ${url}`); + } +}; + +function streamToByteArray(stream: Readable): Promise { + return new Promise((resolve, reject) => { + const chunks: Buffer[] = []; + + stream.on('data', (chunk: Buffer) => { + chunks.push(chunk); + }); + + stream.on('end', () => { + const buffer = Buffer.concat(chunks); + resolve(new Uint8Array(buffer)); + }); + + stream.on('error', (err: Error) => { + reject(err); + }); + }); +} + +export const multerFileSystemStorage: multer.Multer = multer({ + storage: { + _handleFile( + req: Request, + file: Express.Multer.File, + cb: (error?: any, info?: Partial) => void + ): void { + const fileUuid = req.params.uuid; + const key = `${fileUuid}-${file.originalname}`; + const jwt = req.header('Authorization'); + + if (!jwt) { + cb('No authorization header'); + return; + } + + const passThrough = new stream.PassThrough(); + file.stream.pipe(passThrough); + + streamToByteArray(passThrough) + .then((data) => { + upload(key, data, jwt) + .then((_response) => cb(null, file)) + .catch((error) => cb(error)); + }) + .catch((error) => cb(error)); + }, + + _removeFile(_req: Request, _file: Express.Multer.File, cb: (error: Error | null) => void): void { + cb(null); + }, + }, +}); diff --git a/quadratic-api/src/storage/s3.ts b/quadratic-api/src/storage/s3.ts index 96b1728930..4cebed9c1e 100644 --- a/quadratic-api/src/storage/s3.ts +++ b/quadratic-api/src/storage/s3.ts @@ -1,5 +1,8 @@ import { GetObjectCommand, PutObjectCommand, S3Client } from '@aws-sdk/client-s3'; import { getSignedUrl } from '@aws-sdk/s3-request-presigner'; +import { Request } from 'express'; +import multer, { StorageEngine } from 'multer'; +import multerS3 from 'multer-s3'; import { AWS_S3_ACCESS_KEY_ID, AWS_S3_BUCKET_NAME, @@ -7,6 +10,8 @@ import { AWS_S3_SECRET_ACCESS_KEY, ENVIRONMENT, } from '../env-vars'; +import { UploadFileResponse } from './storage'; + const endpoint = ENVIRONMENT === 'docker' ? 'http://localstack:4566' : ENVIRONMENT === 'local' ? 'http://0.0.0.0:4566' : undefined; @@ -21,7 +26,7 @@ export const s3Client = new S3Client({ forcePathStyle: true, }); -export const uploadStringAsFileS3 = async (fileKey: string, contents: string) => { +export const uploadStringAsFileS3 = async (fileKey: string, contents: string): Promise => { const command = new PutObjectCommand({ Bucket: AWS_S3_BUCKET_NAME, Key: fileKey, @@ -42,8 +47,22 @@ export const uploadStringAsFileS3 = async (fileKey: string, contents: string) => } }; +export const multerS3Storage: multer.Multer = multer({ + storage: multerS3({ + s3: s3Client, + bucket: AWS_S3_BUCKET_NAME, + metadata: (req: Request, file: Express.Multer.File, cb: (error: Error | null, metadata: any) => void) => { + cb(null, { fieldName: file.fieldname }); + }, + key: (req: Request, file: Express.Multer.File, cb: (error: Error | null, key: string) => void) => { + const fileUuid = req.params.uuid; + cb(null, `${fileUuid}-${file.originalname}`); + }, + }) as StorageEngine, +}); + // Get file URL from S3 -export const generatePresignedUrl = async (key: string) => { +export const generatePresignedUrl = async (key: string): Promise => { const command = new GetObjectCommand({ Bucket: AWS_S3_BUCKET_NAME, Key: key, diff --git a/quadratic-api/src/storage/storage.ts b/quadratic-api/src/storage/storage.ts new file mode 100644 index 0000000000..6483436e9a --- /dev/null +++ b/quadratic-api/src/storage/storage.ts @@ -0,0 +1,53 @@ +import multer from 'multer'; +import { STORAGE_TYPE } from '../env-vars'; +import { getPresignedStorageUrl, getStorageUrl, multerFileSystemStorage, upload } from './fileSystem'; +import { generatePresignedUrl, multerS3Storage, uploadStringAsFileS3 } from './s3'; + +export type UploadFileResponse = { + bucket: string; + key: string; +}; + +export const getFileUrl = async (key: string) => { + switch (STORAGE_TYPE) { + case 's3': + return await generatePresignedUrl(key); + case 'file-system': + return getStorageUrl(key); + default: + throw new Error(`Unsupported storage type in getFileUrl(): ${STORAGE_TYPE}`); + } +}; + +export const getPresignedFileUrl = async (key: string) => { + switch (STORAGE_TYPE) { + case 's3': + return await generatePresignedUrl(key); + case 'file-system': + return getPresignedStorageUrl(key); + default: + throw new Error(`Unsupported storage type in getFileUrl(): ${STORAGE_TYPE}`); + } +}; + +export const uploadFile = async (key: string, contents: string, jwt: string): Promise => { + switch (STORAGE_TYPE) { + case 's3': + return await uploadStringAsFileS3(key, contents); + case 'file-system': + return await upload(key, contents, jwt); + default: + throw new Error(`Unsupported storage type in uploadFile(): ${STORAGE_TYPE}`); + } +}; + +export const uploadMiddleware = (): multer.Multer => { + switch (STORAGE_TYPE) { + case 's3': + return multerS3Storage; + case 'file-system': + return multerFileSystemStorage as unknown as multer.Multer; + default: + throw new Error(`Unsupported storage type in uploadMiddleware(): ${STORAGE_TYPE}`); + } +}; diff --git a/quadratic-api/src/utils/createFile.ts b/quadratic-api/src/utils/createFile.ts index ca39af2d9a..e8626c0be1 100644 --- a/quadratic-api/src/utils/createFile.ts +++ b/quadratic-api/src/utils/createFile.ts @@ -1,5 +1,5 @@ import dbClient from '../dbClient'; -import { uploadStringAsFileS3 } from '../storage/s3'; +import { uploadFile } from '../storage/storage'; export async function createFile({ contents, @@ -8,6 +8,7 @@ export async function createFile({ version, teamId, isPrivate, + jwt, }: { contents: string; name: string; @@ -15,6 +16,7 @@ export async function createFile({ version: string; teamId: number; isPrivate?: boolean; + jwt: string; }) { return await dbClient.$transaction(async (transaction) => { // Create file in db @@ -36,7 +38,7 @@ export async function createFile({ // Upload file contents to S3 and create a checkpoint const { uuid, id: fileId } = dbFile; - const response = await uploadStringAsFileS3(`${uuid}-0.grid`, contents); + const response = await uploadFile(`${uuid}-0.grid`, contents, jwt); await transaction.fileCheckpoint.create({ data: { diff --git a/quadratic-client/src/app/web-workers/quadraticCore/worker/core.ts b/quadratic-client/src/app/web-workers/quadraticCore/worker/core.ts index 44a0ca5574..6fe2961164 100644 --- a/quadratic-client/src/app/web-workers/quadraticCore/worker/core.ts +++ b/quadratic-client/src/app/web-workers/quadraticCore/worker/core.ts @@ -51,7 +51,12 @@ class Core { private renderQueue: Function[] = []; private async loadGridFile(file: string): Promise { - const res = await fetch(file); + const jwt = await coreClient.getJwt(); + const res = await fetch(file, { + headers: { + Authorization: `Bearer ${jwt}`, + }, + }); return await res.text(); } diff --git a/quadratic-client/src/shared/api/apiClient.ts b/quadratic-client/src/shared/api/apiClient.ts index 7547a91034..c78a591f69 100644 --- a/quadratic-client/src/shared/api/apiClient.ts +++ b/quadratic-client/src/shared/api/apiClient.ts @@ -75,7 +75,6 @@ export const apiClient = { ); }, async delete(uuid: string, inviteId: string) { - console.log(`DELETE to /v0/teams/${uuid}/invites/${inviteId}`); return fetchFromApi( `/v0/teams/${uuid}/invites/${inviteId}`, { diff --git a/quadratic-files/.env.docker b/quadratic-files/.env.docker index 17c4c9e751..132746ccf1 100644 --- a/quadratic-files/.env.docker +++ b/quadratic-files/.env.docker @@ -25,5 +25,5 @@ AWS_S3_BUCKET_NAME=quadratic-api-docker AWS_S3_ACCESS_KEY_ID= AWS_S3_SECRET_ACCESS_KEY= -# Storage: filesystem -STORAGE_DIR=./storage \ No newline at end of file +# Storage: file-system +STORAGE_DIR=./../docker/file-storage \ No newline at end of file diff --git a/quadratic-files/.env.example b/quadratic-files/.env.example index addc0f9bcb..21d7b8c77c 100644 --- a/quadratic-files/.env.example +++ b/quadratic-files/.env.example @@ -25,5 +25,5 @@ AWS_S3_BUCKET_NAME=quadratic-api-docker AWS_S3_ACCESS_KEY_ID=test AWS_S3_SECRET_ACCESS_KEY=test -# Storage: filesystem -STORAGE_DIR=./storage \ No newline at end of file +# Storage: file-system +STORAGE_DIR=./../docker/file-storage \ No newline at end of file diff --git a/quadratic-files/.env.test b/quadratic-files/.env.test index 380e4bcac3..f6f8b48338 100644 --- a/quadratic-files/.env.test +++ b/quadratic-files/.env.test @@ -25,5 +25,5 @@ AWS_S3_BUCKET_NAME=quadratic-api-docker AWS_S3_ACCESS_KEY_ID= AWS_S3_SECRET_ACCESS_KEY= -# Storage: filesystem -STORAGE_DIR=./storage \ No newline at end of file +# Storage: file-system +STORAGE_DIR=./../docker/file-storage \ No newline at end of file diff --git a/quadratic-files/Cargo.toml b/quadratic-files/Cargo.toml index ac29ebf553..51bd57a08f 100644 --- a/quadratic-files/Cargo.toml +++ b/quadratic-files/Cargo.toml @@ -5,7 +5,7 @@ edition = "2021" authors = ["David DiMaria "] [dependencies] -axum = { version = "0.7.1", features = ["ws"] } +axum = { version = "0.7.1", features = ["macros"] } axum-extra = { version = "0.9.0", features = ["typed-header"] } bytes = "1.6.1" chrono = { version= "0.4.31", features = ["serde"] } @@ -26,8 +26,9 @@ strum = "0.25.0" strum_macros = "0.25.3" thiserror = "1.0.50" tokio = { version = "1.34.0", features = ["full"] } +tokio-util = "0.7.11" tower = { version = "0.4.13", features = ["util"] } -tower-http = { version = "0.5.0", features = ["fs", "trace"] } +tower-http = { version = "0.5.0", features = ["cors", "fs", "trace"] } tracing = "0.1.40" tracing-subscriber = { version = "0.3.18", features = ["env-filter"] } uuid = { version = "1.6.1", features = ["serde", "v4"] } diff --git a/quadratic-files/src/error.rs b/quadratic-files/src/error.rs index d914b4ca50..8c77aa9d06 100644 --- a/quadratic-files/src/error.rs +++ b/quadratic-files/src/error.rs @@ -9,7 +9,7 @@ use axum::{ http::StatusCode, response::{IntoResponse, Response}, }; -use quadratic_rust_shared::{clean_errors, Auth, Aws, SharedError}; +use quadratic_rust_shared::{clean_errors, SharedError}; use serde::{Deserialize, Serialize}; use thiserror::Error; @@ -44,6 +44,9 @@ pub enum FilesError { #[error("PubSub error: {0}")] PubSub(String), + #[error("QuadraticApi error: {0}")] + QuadraticApi(String), + #[error("Error requesting data: {0}")] Request(String), @@ -53,6 +56,9 @@ pub enum FilesError { #[error("Error serializing or deserializing: {0}")] Serialization(String), + #[error("File storage error: {0}")] + Storage(String), + #[error("Transaction queue error: {0}")] TransactionQueue(String), @@ -84,14 +90,12 @@ impl IntoResponse for FilesError { impl From for FilesError { fn from(error: SharedError) -> Self { match error { - SharedError::Auth(auth) => match auth { - Auth::Jwt(error) => FilesError::Authentication(error), - }, - SharedError::Aws(aws) => match aws { - Aws::S3(error) => FilesError::S3(error), - }, + SharedError::Auth(error) => FilesError::Authentication(error.to_string()), + SharedError::Aws(error) => FilesError::S3(error.to_string()), SharedError::PubSub(error) => FilesError::PubSub(error), - _ => FilesError::Unknown(format!("Unknown Quadratic API error: {error}")), + SharedError::QuadraticApi(error) => FilesError::QuadraticApi(error), + SharedError::Storage(error) => FilesError::Storage(error.to_string()), + _ => FilesError::Unknown(format!("Unknown SharedError: {error}")), } } } diff --git a/quadratic-files/src/server.rs b/quadratic-files/src/server.rs index 5cc02ed3a1..5c78509d92 100644 --- a/quadratic-files/src/server.rs +++ b/quadratic-files/src/server.rs @@ -3,8 +3,7 @@ //! Handle bootstrapping and starting the HTTP server. Adds global state //! to be shared across all requests and threads. Adds tracing/logging. -use axum::extract::Request; -use axum::http::StatusCode; +use axum::http::{Method, StatusCode}; use axum::response::IntoResponse; use axum::{routing::get, Extension, Router}; use quadratic_rust_shared::auth::jwt::get_jwks; @@ -12,11 +11,11 @@ use quadratic_rust_shared::storage::Storage; use std::time::Duration; use std::{net::SocketAddr, sync::Arc}; use tokio::time; -use tower::ServiceExt; -use tower_http::services::ServeDir; +use tower_http::cors::{Any, CorsLayer}; use tower_http::trace::{DefaultMakeSpan, TraceLayer}; use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt}; +use crate::storage::get_storage; use crate::truncate::truncate_processed_transactions; use crate::{ auth::get_middleware, @@ -24,9 +23,10 @@ use crate::{ error::{FilesError, Result}, file::process, state::State, + storage::upload_storage, }; -const HEALTHCHECK_INTERVAL_S: u64 = 5; +const HEALTHCHECK_INTERVAL_S: u64 = 30; /// Construct the application router. This is separated out so that it can be /// integration tested. @@ -41,29 +41,53 @@ pub(crate) fn app(state: Arc) -> Router { let auth = get_middleware(jwks); let path = state.settings.storage.path().to_owned(); + let cors = CorsLayer::new() + .allow_methods([Method::GET, Method::POST]) + // + // allow requests from any origin + .allow_origin(Any) + // + // TODO(ddimaria): uncomment when we move proxy to a separate service + // + // .allow_headers([ + // CONTENT_TYPE, + // AUTHORIZATION, + // ACCEPT, + // ORIGIN, + // HeaderName::from_static("proxy"), + // ]) + // + // required for the proxy + .allow_headers(Any) + .expose_headers(Any); + tracing::info!("Serving files from {path}"); Router::new() // protected routes // - // - .nest_service( - "/storage", - get(|request: Request| async { - let service = ServeDir::new("storage"); - service.oneshot(request).await - }), + // get a file from storage + .route( + "/storage/:key", + get(get_storage) + // + // upload a file + .post(upload_storage), ) // // auth middleware .route_layer(auth) // // unprotected routes + // .route("/health", get(healthcheck)) // // state .layer(Extension(state)) // + // cors + .layer(cors) + // // logger .layer( TraceLayer::new_for_http() diff --git a/quadratic-files/src/storage.rs b/quadratic-files/src/storage.rs index 8b13789179..9a1f83a982 100644 --- a/quadratic-files/src/storage.rs +++ b/quadratic-files/src/storage.rs @@ -1 +1,54 @@ +use axum::{ + body::to_bytes, + debug_handler, + extract::{Path, Request}, + response::IntoResponse, + Extension, Json, +}; +use quadratic_rust_shared::storage::Storage; +use serde::Serialize; +use std::sync::Arc; +use crate::error::{FilesError, Result}; +use crate::state::State; + +#[derive(Debug, Serialize)] +pub(crate) struct UploadStorageResponse { + bucket: String, + key: String, +} + +#[debug_handler] +pub(crate) async fn get_storage( + Path(file_name): Path, + state: Extension>, +) -> Result { + tracing::info!("Get file {}", file_name,); + + let file = state.settings.storage.read(&file_name).await?; + Ok(file.into_response()) +} + +#[debug_handler] +pub(crate) async fn upload_storage( + Path(file_name): Path, + state: Extension>, + request: Request, +) -> Result> { + tracing::info!( + "Uploading file {} to {}", + file_name, + state.settings.storage.path() + ); + + let bytes = to_bytes(request.into_body(), usize::MAX) + .await + .map_err(|e| FilesError::Storage(e.to_string()))?; + + state.settings.storage.write(&file_name, &bytes).await?; + + Ok(Json(UploadStorageResponse { + bucket: state.settings.storage.path().to_owned(), + key: file_name, + })) +} diff --git a/quadratic-files/storage/hello.txt b/quadratic-files/storage/hello.txt deleted file mode 100644 index 6769dd60bd..0000000000 --- a/quadratic-files/storage/hello.txt +++ /dev/null @@ -1 +0,0 @@ -Hello world! \ No newline at end of file diff --git a/quadratic-rust-shared/src/storage/mod.rs b/quadratic-rust-shared/src/storage/mod.rs index 5bd7bc424c..96814881bb 100644 --- a/quadratic-rust-shared/src/storage/mod.rs +++ b/quadratic-rust-shared/src/storage/mod.rs @@ -20,6 +20,21 @@ pub enum StorageContainer { FileSystem(file_system::FileSystem), } +#[async_trait] +pub trait Storage { + async fn read(&self, key: &str) -> Result; + async fn write<'a>(&self, key: &'a str, data: &'a Bytes) -> Result<()>; + fn path(&self) -> &str; + + fn read_error(key: &str, e: impl ToString) -> SharedError { + SharedError::Storage(StorageError::Read(key.into(), e.to_string())) + } + + fn write_error(key: &str, e: impl ToString) -> SharedError { + SharedError::Storage(StorageError::Write(key.into(), e.to_string())) + } +} + // TODO(ddimaria): this is a temp hack to get around some trait issues, do something better #[async_trait] impl Storage for StorageContainer { @@ -44,18 +59,3 @@ impl Storage for StorageContainer { } } } - -#[async_trait] -pub trait Storage { - async fn read(&self, key: &str) -> Result; - async fn write<'a>(&self, key: &'a str, data: &'a Bytes) -> Result<()>; - fn path(&self) -> &str; - - fn read_error(key: &str, e: impl ToString) -> SharedError { - SharedError::Storage(StorageError::Read(key.into(), e.to_string())) - } - - fn write_error(key: &str, e: impl ToString) -> SharedError { - SharedError::Storage(StorageError::Write(key.into(), e.to_string())) - } -} From 4d613db49fa82e8af5bbb8ab0916236650f7d9c1 Mon Sep 17 00:00:00 2001 From: David DiMaria Date: Fri, 2 Aug 2024 13:51:52 -0600 Subject: [PATCH 008/113] Presigned URLs, cleanup, documentation, tests --- Cargo.lock | 52 +++++++++++++ .../routes/v0/files.$uuid.thumbnail.POST.ts | 2 + quadratic-api/src/storage/fileSystem.ts | 7 +- quadratic-files/Cargo.toml | 2 +- quadratic-files/src/config.rs | 1 + quadratic-files/src/error.rs | 13 +++- quadratic-files/src/server.rs | 9 ++- quadratic-files/src/state/settings.rs | 6 +- quadratic-files/src/storage.rs | 40 ++++++++-- quadratic-rust-shared/Cargo.toml | 3 + quadratic-rust-shared/package.json | 19 +++++ quadratic-rust-shared/src/arrow/error.rs | 8 ++ quadratic-rust-shared/src/arrow/mod.rs | 1 + quadratic-rust-shared/src/auth/error.rs | 8 ++ quadratic-rust-shared/src/auth/jwt.rs | 3 +- quadratic-rust-shared/src/auth/mod.rs | 1 + quadratic-rust-shared/src/aws/error.rs | 8 ++ quadratic-rust-shared/src/aws/mod.rs | 1 + quadratic-rust-shared/src/aws/s3.rs | 7 +- quadratic-rust-shared/src/crypto/aes_cbc.rs | 75 +++++++++++++++++++ quadratic-rust-shared/src/crypto/error.rs | 8 ++ quadratic-rust-shared/src/crypto/mod.rs | 2 + quadratic-rust-shared/src/error.rs | 58 +++----------- quadratic-rust-shared/src/lib.rs | 1 + quadratic-rust-shared/src/sql/error.rs | 17 +++++ quadratic-rust-shared/src/sql/mod.rs | 1 + .../src/sql/mysql_connection.rs | 23 +++--- .../src/sql/postgres_connection.rs | 23 +++--- quadratic-rust-shared/src/storage/error.rs | 17 +++++ .../src/storage/file_system.rs | 23 ++++-- quadratic-rust-shared/src/storage/mod.rs | 17 ++++- quadratic-rust-shared/src/storage/s3.rs | 18 ++++- 32 files changed, 375 insertions(+), 99 deletions(-) create mode 100644 quadratic-rust-shared/package.json create mode 100644 quadratic-rust-shared/src/arrow/error.rs create mode 100644 quadratic-rust-shared/src/auth/error.rs create mode 100644 quadratic-rust-shared/src/aws/error.rs create mode 100644 quadratic-rust-shared/src/crypto/aes_cbc.rs create mode 100644 quadratic-rust-shared/src/crypto/error.rs create mode 100644 quadratic-rust-shared/src/crypto/mod.rs create mode 100644 quadratic-rust-shared/src/sql/error.rs create mode 100644 quadratic-rust-shared/src/storage/error.rs diff --git a/Cargo.lock b/Cargo.lock index 347f4b994c..75a32cda5b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -23,6 +23,17 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" +[[package]] +name = "aes" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" +dependencies = [ + "cfg-if", + "cipher", + "cpufeatures", +] + [[package]] name = "ahash" version = "0.8.11" @@ -1004,6 +1015,15 @@ dependencies = [ "generic-array", ] +[[package]] +name = "block-padding" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8894febbff9f758034a5b8e12d87918f56dfc64a8e1fe757d65e29041538d93" +dependencies = [ + "generic-array", +] + [[package]] name = "bumpalo" version = "3.16.0" @@ -1061,6 +1081,15 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" +[[package]] +name = "cbc" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26b52a9543ae338f279b96b0b9fed9c8093744685043739079ce85cd58f289a6" +dependencies = [ + "cipher", +] + [[package]] name = "cc" version = "1.0.99" @@ -1115,6 +1144,16 @@ dependencies = [ "half", ] +[[package]] +name = "cipher" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" +dependencies = [ + "crypto-common", + "inout", +] + [[package]] name = "clap" version = "3.2.25" @@ -2257,6 +2296,16 @@ dependencies = [ "serde", ] +[[package]] +name = "inout" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0c10553d664a4d0bcff9f4215d0aac67a639cc68ef660840afe309b807bc9f5" +dependencies = [ + "block-padding", + "generic-array", +] + [[package]] name = "integer-encoding" version = "3.0.4" @@ -3230,6 +3279,7 @@ dependencies = [ name = "quadratic-rust-shared" version = "0.1.0" dependencies = [ + "aes", "arrow", "async-trait", "aws-config", @@ -3239,8 +3289,10 @@ dependencies = [ "aws-smithy-runtime-api", "bigdecimal 0.3.1", "bytes", + "cbc", "chrono", "futures-util", + "hex", "http 1.1.0", "jsonwebtoken", "parquet", diff --git a/quadratic-api/src/routes/v0/files.$uuid.thumbnail.POST.ts b/quadratic-api/src/routes/v0/files.$uuid.thumbnail.POST.ts index bae2496a46..dc81f8e362 100644 --- a/quadratic-api/src/routes/v0/files.$uuid.thumbnail.POST.ts +++ b/quadratic-api/src/routes/v0/files.$uuid.thumbnail.POST.ts @@ -23,6 +23,8 @@ async function handler(req: RequestWithUser & RequestWithFile, res: Response) { return res.status(403).json({ error: { message: 'Permission denied' } }); } + console.log('req.file', req.file); + // update the file object with the thumbnail URL await dbClient.file.update({ where: { diff --git a/quadratic-api/src/storage/fileSystem.ts b/quadratic-api/src/storage/fileSystem.ts index 647d9e8e81..f6c7ecc759 100644 --- a/quadratic-api/src/storage/fileSystem.ts +++ b/quadratic-api/src/storage/fileSystem.ts @@ -2,6 +2,7 @@ import { Request } from 'express'; import multer from 'multer'; import stream, { Readable } from 'node:stream'; import { QUADRATIC_FILE_URI } from '../env-vars'; +import { UploadFile } from '../types/Request'; import { encryptFromEnv } from '../utils/crypto'; import { UploadFileResponse } from './storage'; @@ -20,8 +21,6 @@ export const getPresignedStorageUrl = (key: string): string => { export const upload = async (key: string, contents: string | Uint8Array, jwt: string): Promise => { const url = generateUrl(key); - console.warn('Uploading to', url); - try { const response = await fetch(url, { method: 'POST', @@ -62,13 +61,15 @@ export const multerFileSystemStorage: multer.Multer = multer({ storage: { _handleFile( req: Request, - file: Express.Multer.File, + file: Express.Multer.File & UploadFile, cb: (error?: any, info?: Partial) => void ): void { const fileUuid = req.params.uuid; const key = `${fileUuid}-${file.originalname}`; const jwt = req.header('Authorization'); + file.key = key; + if (!jwt) { cb('No authorization header'); return; diff --git a/quadratic-files/Cargo.toml b/quadratic-files/Cargo.toml index 51bd57a08f..618089bf72 100644 --- a/quadratic-files/Cargo.toml +++ b/quadratic-files/Cargo.toml @@ -28,7 +28,7 @@ thiserror = "1.0.50" tokio = { version = "1.34.0", features = ["full"] } tokio-util = "0.7.11" tower = { version = "0.4.13", features = ["util"] } -tower-http = { version = "0.5.0", features = ["cors", "fs", "trace"] } +tower-http = { version = "0.5.0", features = ["cors", "fs", "trace", "validate-request"] } tracing = "0.1.40" tracing-subscriber = { version = "0.3.18", features = ["env-filter"] } uuid = { version = "1.6.1", features = ["serde", "v4"] } diff --git a/quadratic-files/src/config.rs b/quadratic-files/src/config.rs index b82b3c5dfe..ea0061e32f 100644 --- a/quadratic-files/src/config.rs +++ b/quadratic-files/src/config.rs @@ -50,6 +50,7 @@ pub(crate) struct Config { // StorageType::FileSystem pub(crate) storage_dir: Option, + pub(crate) storage_encryption_keys: Option>, } /// Load the global configuration from the environment into Config. diff --git a/quadratic-files/src/error.rs b/quadratic-files/src/error.rs index 8c77aa9d06..762fb58a18 100644 --- a/quadratic-files/src/error.rs +++ b/quadratic-files/src/error.rs @@ -9,7 +9,7 @@ use axum::{ http::StatusCode, response::{IntoResponse, Response}, }; -use quadratic_rust_shared::{clean_errors, SharedError}; +use quadratic_rust_shared::{clean_errors, storage::error::Storage as StorageError, SharedError}; use serde::{Deserialize, Serialize}; use thiserror::Error; @@ -41,6 +41,9 @@ pub enum FilesError { #[error("Unable to load file {0}: {1}")] LoadFile(String, String), + #[error("Not Found: {0}")] + NotFound(String), + #[error("PubSub error: {0}")] PubSub(String), @@ -78,10 +81,11 @@ impl IntoResponse for FilesError { FilesError::InternalServer(error) => { (StatusCode::INTERNAL_SERVER_ERROR, clean_errors(error)) } + FilesError::NotFound(error) => (StatusCode::NOT_FOUND, clean_errors(error)), _ => (StatusCode::INTERNAL_SERVER_ERROR, "Unknown".into()), }; - tracing::warn!("{} {}: {:?}", status, error, self); + tracing::warn!("{:?}", self); (status, error).into_response() } @@ -94,7 +98,10 @@ impl From for FilesError { SharedError::Aws(error) => FilesError::S3(error.to_string()), SharedError::PubSub(error) => FilesError::PubSub(error), SharedError::QuadraticApi(error) => FilesError::QuadraticApi(error), - SharedError::Storage(error) => FilesError::Storage(error.to_string()), + SharedError::Storage(error) => match error { + StorageError::Read(key, _) => FilesError::NotFound(format!("File {key} not found")), + _ => FilesError::Storage(error.to_string()), + }, _ => FilesError::Unknown(format!("Unknown SharedError: {error}")), } } diff --git a/quadratic-files/src/server.rs b/quadratic-files/src/server.rs index 5c78509d92..056776d734 100644 --- a/quadratic-files/src/server.rs +++ b/quadratic-files/src/server.rs @@ -15,7 +15,7 @@ use tower_http::cors::{Any, CorsLayer}; use tower_http::trace::{DefaultMakeSpan, TraceLayer}; use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt}; -use crate::storage::get_storage; +use crate::storage::{get_presigned_storage, get_storage}; use crate::truncate::truncate_processed_transactions; use crate::{ auth::get_middleware, @@ -64,7 +64,7 @@ pub(crate) fn app(state: Arc) -> Router { tracing::info!("Serving files from {path}"); Router::new() - // protected routes + // PROTECTED ROUTES (via JWT) // // get a file from storage .route( @@ -78,10 +78,13 @@ pub(crate) fn app(state: Arc) -> Router { // auth middleware .route_layer(auth) // - // unprotected routes + // UNPROTECTED ROUTES // .route("/health", get(healthcheck)) // + // presigned urls + .route("/storage/presigned/:key", get(get_presigned_storage)) + // // state .layer(Extension(state)) // diff --git a/quadratic-files/src/state/settings.rs b/quadratic-files/src/state/settings.rs index 891eee25ba..92db063cca 100644 --- a/quadratic-files/src/state/settings.rs +++ b/quadratic-files/src/state/settings.rs @@ -24,7 +24,7 @@ impl Settings { config.environment == Environment::Docker || config.environment == Environment::Local; let expected = |val: &Option, var: &str| { val.to_owned() - .expect(&format!("Expected {} to have a value", var)) + .unwrap_or_else(|| panic!("Expected {} to have a value", var)) }; let storage = match config.storage_type { @@ -42,6 +42,10 @@ impl Settings { StorageType::FileSystem => { StorageContainer::FileSystem(FileSystem::new(FileSystemConfig { path: expected(&config.storage_dir, "STORAGE_DIR"), + encryption_keys: config + .storage_encryption_keys + .to_owned() + .expect("Expected STORAGE_ENCRYPTION_KEYS to have a value"), })) } }; diff --git a/quadratic-files/src/storage.rs b/quadratic-files/src/storage.rs index 9a1f83a982..34a31c1344 100644 --- a/quadratic-files/src/storage.rs +++ b/quadratic-files/src/storage.rs @@ -1,11 +1,13 @@ use axum::{ body::to_bytes, - debug_handler, extract::{Path, Request}, response::IntoResponse, Extension, Json, }; -use quadratic_rust_shared::storage::Storage; +use quadratic_rust_shared::{ + crypto::aes_cbc::decrypt_from_api, + storage::{Storage, StorageConfig}, +}; use serde::Serialize; use std::sync::Arc; @@ -18,24 +20,50 @@ pub(crate) struct UploadStorageResponse { key: String, } -#[debug_handler] +/// Get a file from storage pub(crate) async fn get_storage( Path(file_name): Path, state: Extension>, ) -> Result { - tracing::info!("Get file {}", file_name,); + tracing::trace!("Get file {}", file_name); let file = state.settings.storage.read(&file_name).await?; Ok(file.into_response()) } -#[debug_handler] +/// Get a file from storage from a presigned URL (encrypted) +pub(crate) async fn get_presigned_storage( + Path(encrypted_file_name): Path, + state: Extension>, +) -> Result { + tracing::trace!("Get presigned file {}", encrypted_file_name); + + match state.settings.storage.config() { + StorageConfig::FileSystem(config) => { + // For now, we only support one encryption key. + // In the future, implement key traversal on decryption failures. + let key = config + .encryption_keys + .first() + .ok_or(FilesError::Storage("No encryption keys found".to_string()))?; + let file_name = decrypt_from_api(key, &encrypted_file_name)?; + let file = state.settings.storage.read(&file_name).await?; + + Ok(file.into_response()) + } + _ => Err(FilesError::Storage( + "Presigned URLs supported in FileSystem storage options".to_string(), + )), + } +} + +/// Upload a file to storage pub(crate) async fn upload_storage( Path(file_name): Path, state: Extension>, request: Request, ) -> Result> { - tracing::info!( + tracing::trace!( "Uploading file {} to {}", file_name, state.settings.storage.path() diff --git a/quadratic-rust-shared/Cargo.toml b/quadratic-rust-shared/Cargo.toml index 3402f48943..5acc748689 100644 --- a/quadratic-rust-shared/Cargo.toml +++ b/quadratic-rust-shared/Cargo.toml @@ -6,14 +6,17 @@ edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] +aes = "0.8.4" arrow = "51.0.0" async-trait = "0.1.80" aws-config = { version= "1.1.1", features = ["behavior-version-latest"] } aws-sdk-s3 = {version = "1.12.0", features = ["behavior-version-latest", "rt-tokio"] } bigdecimal = "0.3.0" # need this fixed to the sqlx dependency bytes = "1.6.0" +cbc = { version = "0.1.2", features = ["alloc"] } chrono = "0.4.31" futures-util = "0.3.30" +hex = "0.4.3" jsonwebtoken = "9.2.0" parquet = { version = "51.0.0", default-features = false, features = ["arrow", "arrow-array", "flate2", "snap"] } redis = { version = "0.25.3", features = ["tokio-comp"] } diff --git a/quadratic-rust-shared/package.json b/quadratic-rust-shared/package.json new file mode 100644 index 0000000000..39dba83e85 --- /dev/null +++ b/quadratic-rust-shared/package.json @@ -0,0 +1,19 @@ +{ + "name": "quadratic-rust-shared", + "description": "Shared Rust code in Quadratic", + "version": "0.1.0", + "dependencies": {}, + "devDependencies": {}, + "scripts": { + "start": "RUST_LOG=info cargo run", + "build": "cargo build", + "dev": "RUST_LOG=info cargo watch -x 'run'", + "test": "cargo test", + "test:watch": "RUST_LOG=info cargo watch -x 'test'", + "lint": "cargo clippy --all-targets --all-features -- -D warnings", + "coverage": "npm run coverage:gen && npm run coverage:html && npm run coverage:view", + "coverage:gen": "CARGO_INCREMENTAL=0 RUSTFLAGS='-Cinstrument-coverage' LLVM_PROFILE_FILE='coverage/cargo-test-%p-%m.profraw' cargo test", + "coverage:html": "grcov . --binary-path ./target/debug/deps/ -s . -t html --branch --ignore-not-existing --ignore '../*' --ignore '/*' -o coverage/html", + "coverage:view": "open coverage/html/index.html" + } +} diff --git a/quadratic-rust-shared/src/arrow/error.rs b/quadratic-rust-shared/src/arrow/error.rs new file mode 100644 index 0000000000..b2204c5734 --- /dev/null +++ b/quadratic-rust-shared/src/arrow/error.rs @@ -0,0 +1,8 @@ +use serde::{Deserialize, Serialize}; +use thiserror::Error; + +#[derive(Error, Debug, Serialize, Deserialize, PartialEq, Clone)] +pub enum Arrow { + #[error("Arrow error: {0}")] + External(String), +} diff --git a/quadratic-rust-shared/src/arrow/mod.rs b/quadratic-rust-shared/src/arrow/mod.rs index e69de29bb2..a91e735174 100644 --- a/quadratic-rust-shared/src/arrow/mod.rs +++ b/quadratic-rust-shared/src/arrow/mod.rs @@ -0,0 +1 @@ +pub mod error; diff --git a/quadratic-rust-shared/src/auth/error.rs b/quadratic-rust-shared/src/auth/error.rs new file mode 100644 index 0000000000..cf82f97170 --- /dev/null +++ b/quadratic-rust-shared/src/auth/error.rs @@ -0,0 +1,8 @@ +use serde::{Deserialize, Serialize}; +use thiserror::Error; + +#[derive(Error, Debug, Serialize, Deserialize, PartialEq, Clone)] +pub enum Auth { + #[error("JWT error: {0}")] + Jwt(String), +} diff --git a/quadratic-rust-shared/src/auth/jwt.rs b/quadratic-rust-shared/src/auth/jwt.rs index fa4cf84f33..9e6383b334 100644 --- a/quadratic-rust-shared/src/auth/jwt.rs +++ b/quadratic-rust-shared/src/auth/jwt.rs @@ -4,7 +4,8 @@ use serde::de::DeserializeOwned; use std::str::FromStr; use tokio::sync::OnceCell; -use crate::error::{Auth, Result, SharedError}; +use crate::auth::error::Auth; +use crate::error::{Result, SharedError}; pub static JWKS: OnceCell = OnceCell::const_new(); diff --git a/quadratic-rust-shared/src/auth/mod.rs b/quadratic-rust-shared/src/auth/mod.rs index 417233c083..73491e1474 100644 --- a/quadratic-rust-shared/src/auth/mod.rs +++ b/quadratic-rust-shared/src/auth/mod.rs @@ -1 +1,2 @@ +pub mod error; pub mod jwt; diff --git a/quadratic-rust-shared/src/aws/error.rs b/quadratic-rust-shared/src/aws/error.rs new file mode 100644 index 0000000000..2846e2197d --- /dev/null +++ b/quadratic-rust-shared/src/aws/error.rs @@ -0,0 +1,8 @@ +use serde::{Deserialize, Serialize}; +use thiserror::Error; + +#[derive(Error, Debug, Serialize, Deserialize, PartialEq, Clone)] +pub enum Aws { + #[error("Error communicating with AWS: {0}")] + S3(String), +} diff --git a/quadratic-rust-shared/src/aws/mod.rs b/quadratic-rust-shared/src/aws/mod.rs index b4048e379f..8ec4d50623 100644 --- a/quadratic-rust-shared/src/aws/mod.rs +++ b/quadratic-rust-shared/src/aws/mod.rs @@ -1,3 +1,4 @@ +pub mod error; pub mod s3; pub use aws_config::{retry::RetryConfig, BehaviorVersion, Region}; diff --git a/quadratic-rust-shared/src/aws/s3.rs b/quadratic-rust-shared/src/aws/s3.rs index 56b65cecf0..fae7188580 100644 --- a/quadratic-rust-shared/src/aws/s3.rs +++ b/quadratic-rust-shared/src/aws/s3.rs @@ -4,7 +4,8 @@ use aws_sdk_s3::{ Client, }; -use crate::error::{Aws, Result, SharedError}; +use crate::aws::error::Aws as AwsError; +use crate::error::{Result, SharedError}; pub async fn download_object(client: &Client, bucket: &str, key: &str) -> Result { client @@ -14,7 +15,7 @@ pub async fn download_object(client: &Client, bucket: &str, key: &str) -> Result .send() .await .map_err(|error| { - SharedError::Aws(Aws::S3(format!( + SharedError::Aws(AwsError::S3(format!( "Error retrieving file {} from bucket {}: {:?}.", key, bucket, error ))) @@ -37,7 +38,7 @@ pub async fn upload_object( .send() .await .map_err(|error| { - SharedError::Aws(Aws::S3(format!( + SharedError::Aws(AwsError::S3(format!( "Error uploading file {key} to bucket {bucket}: {:?}.", error ))) diff --git a/quadratic-rust-shared/src/crypto/aes_cbc.rs b/quadratic-rust-shared/src/crypto/aes_cbc.rs new file mode 100644 index 0000000000..94313a3cf6 --- /dev/null +++ b/quadratic-rust-shared/src/crypto/aes_cbc.rs @@ -0,0 +1,75 @@ +use std::fmt::Debug; + +use aes::{ + cipher::{block_padding::Pkcs7, BlockDecryptMut, BlockEncryptMut, KeyIvInit}, + Aes256, +}; +use bytes::Bytes; +use cbc::{Decryptor, Encryptor}; + +use crate::{crypto::error::Crypto as CryptoError, error::Result, SharedError}; + +type Aes256CbcEnc = Encryptor; +type Aes256CbcDec = Decryptor; + +/// Encrypt data using AES-256-CBC. +pub fn encrypt(key: &[u8; 32], iv: &[u8; 16], data: &[u8]) -> Result { + let encryptor = Aes256CbcEnc::new(key.into(), iv.into()); + let encrypted = encryptor.encrypt_padded_vec_mut::(data).to_owned(); + + Ok(encrypted.into()) +} + +/// Convenience function to handle errors when decrypting data. +fn decrypt_error(e: impl Debug) -> SharedError { + let error = CryptoError::AesCbcDecode(format!("Error decoding data: {:?}", e)); + SharedError::Crypto(error) +} + +/// Decrypt data using AES-256-CBC. +pub fn decrypt(key: &[u8; 32], iv: &[u8; 16], data: &[u8]) -> Result { + let decryptor = Aes256CbcDec::new(key.into(), iv.into()); + let decrypted = decryptor + .decrypt_padded_vec_mut::(data) + .map_err(decrypt_error)? + .to_owned(); + + Ok(decrypted.into()) +} + +/// Decrypt data from the Quadratic API, which prepends the IV to the data and is hex encoded. +pub fn decrypt_from_api(key: &str, data: &str) -> Result { + let key = hex::decode(key).map_err(decrypt_error)?; + let key = key.try_into().map_err(decrypt_error)?; + let parts = data.split(":").collect::>(); + let decoded_iv = hex::decode(parts[0]).map_err(decrypt_error)?; + let iv = decoded_iv.as_slice().try_into().map_err(decrypt_error)?; + let decoded_data = hex::decode(parts[1]).map_err(decrypt_error)?; + let data = decoded_data.as_slice(); + let decrypted = decrypt(&key, iv, data)?; + let decrypted_string = String::from_utf8(decrypted.to_vec()).map_err(decrypt_error)?; + + Ok(decrypted_string) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn encrypt_and_decrypt_aes_cbc() { + let key = [0x42; 32]; + let iv = [0x24; 16]; + let text = b"Hello, world!"; + + let encrypted = encrypt(&key, &iv, text).unwrap(); + let decrypted = decrypt(&key, &iv, &encrypted).unwrap(); + + assert_eq!(text, decrypted.as_ref()); + + let api_data = format!("{}:{}", hex::encode(iv), hex::encode(encrypted)); + let decrypted = decrypt_from_api(&hex::encode(key), &api_data).unwrap(); + + assert_eq!(text, decrypted.as_bytes()); + } +} diff --git a/quadratic-rust-shared/src/crypto/error.rs b/quadratic-rust-shared/src/crypto/error.rs new file mode 100644 index 0000000000..1f6fdaf22d --- /dev/null +++ b/quadratic-rust-shared/src/crypto/error.rs @@ -0,0 +1,8 @@ +use serde::{Deserialize, Serialize}; +use thiserror::Error; + +#[derive(Error, Debug, Serialize, Deserialize, PartialEq, Clone)] +pub enum Crypto { + #[error("Error decoding {0}")] + AesCbcDecode(String), +} diff --git a/quadratic-rust-shared/src/crypto/mod.rs b/quadratic-rust-shared/src/crypto/mod.rs new file mode 100644 index 0000000000..b0681b7703 --- /dev/null +++ b/quadratic-rust-shared/src/crypto/mod.rs @@ -0,0 +1,2 @@ +pub mod aes_cbc; +pub mod error; diff --git a/quadratic-rust-shared/src/error.rs b/quadratic-rust-shared/src/error.rs index 078f420d90..c0302355db 100644 --- a/quadratic-rust-shared/src/error.rs +++ b/quadratic-rust-shared/src/error.rs @@ -8,55 +8,14 @@ use serde::{Deserialize, Serialize}; use thiserror::Error; -pub type Result = std::result::Result; - -#[derive(Error, Debug, Serialize, Deserialize, PartialEq, Clone)] -pub enum Arrow { - #[error("Arrow error: {0}")] - External(String), -} - -#[derive(Error, Debug, Serialize, Deserialize, PartialEq, Clone)] -pub enum Auth { - #[error("JWT error: {0}")] - Jwt(String), -} - -#[derive(Error, Debug, Serialize, Deserialize, PartialEq, Clone)] -pub enum Aws { - #[error("Error communicating with AWS: {0}")] - S3(String), -} - -#[derive(Error, Debug, Serialize, Deserialize, PartialEq, Clone)] -pub enum Sql { - #[error("Error connecting to database: {0}")] - Connect(String), - - #[error("Error converting results to Parquet: {0}")] - ParquetConversion(String), - - #[error("Error executing query: {0}")] - Query(String), - - #[error("Error creating schema: {0}")] - Schema(String), -} - -#[derive(Error, Debug, Serialize, Deserialize, PartialEq, Clone)] -pub enum Storage { - #[error("Error creating directory {0}: {1}")] - CreateDirectory(String, String), - - #[error("Invalid key: {0}")] - InvalidKey(String), - - #[error("Error reading key {0}: {1}")] - Read(String, String), +use crate::arrow::error::Arrow; +use crate::auth::error::Auth; +use crate::aws::error::Aws; +use crate::crypto::error::Crypto; +use crate::sql::error::Sql; +use crate::storage::error::Storage; - #[error("Error writing key {0}: {1}")] - Write(String, String), -} +pub type Result = std::result::Result; #[derive(Error, Debug, Serialize, Deserialize, PartialEq, Clone)] pub enum SharedError { @@ -69,6 +28,9 @@ pub enum SharedError { #[error("Error communicating with AWS: {0}")] Aws(Aws), + #[error("Error with Crypto: {0}")] + Crypto(Crypto), + #[error("Error communicating with the Quadratic API: {0}")] QuadraticApi(String), diff --git a/quadratic-rust-shared/src/lib.rs b/quadratic-rust-shared/src/lib.rs index 991bc6e8c8..bac2f9ca30 100644 --- a/quadratic-rust-shared/src/lib.rs +++ b/quadratic-rust-shared/src/lib.rs @@ -1,6 +1,7 @@ pub mod arrow; pub mod auth; pub mod aws; +pub mod crypto; pub mod environment; pub mod error; pub mod pubsub; diff --git a/quadratic-rust-shared/src/sql/error.rs b/quadratic-rust-shared/src/sql/error.rs new file mode 100644 index 0000000000..ec9974c3ed --- /dev/null +++ b/quadratic-rust-shared/src/sql/error.rs @@ -0,0 +1,17 @@ +use serde::{Deserialize, Serialize}; +use thiserror::Error; + +#[derive(Error, Debug, Serialize, Deserialize, PartialEq, Clone)] +pub enum Sql { + #[error("Error connecting to database: {0}")] + Connect(String), + + #[error("Error converting results to Parquet: {0}")] + ParquetConversion(String), + + #[error("Error executing query: {0}")] + Query(String), + + #[error("Error creating schema: {0}")] + Schema(String), +} diff --git a/quadratic-rust-shared/src/sql/mod.rs b/quadratic-rust-shared/src/sql/mod.rs index 2973eb53d4..ed5f893eaa 100644 --- a/quadratic-rust-shared/src/sql/mod.rs +++ b/quadratic-rust-shared/src/sql/mod.rs @@ -24,6 +24,7 @@ use crate::{ vec_time_arrow_type_to_array_ref, }; +pub mod error; pub mod mysql_connection; pub mod postgres_connection; diff --git a/quadratic-rust-shared/src/sql/mysql_connection.rs b/quadratic-rust-shared/src/sql/mysql_connection.rs index a0a3c88cdd..2002afa32d 100644 --- a/quadratic-rust-shared/src/sql/mysql_connection.rs +++ b/quadratic-rust-shared/src/sql/mysql_connection.rs @@ -13,11 +13,11 @@ use sqlx::{ }; use uuid::Uuid; -use crate::error::{Result, SharedError, Sql}; +use crate::error::{Result, SharedError}; use crate::sql::{ArrowType, Connection}; use crate::{ convert_mysql_type, - sql::{DatabaseSchema, SchemaColumn, SchemaTable}, + sql::{error::Sql as SqlError, DatabaseSchema, SchemaColumn, SchemaTable}, }; #[derive(Debug, Serialize, Deserialize)] @@ -67,7 +67,9 @@ impl Connection for MySqlConnection { if let Some(ref port) = self.port { options = options.port(port.parse::().map_err(|_| { - SharedError::Sql(Sql::Connect("Could not parse port into a number".into())) + SharedError::Sql(SqlError::Connect( + "Could not parse port into a number".into(), + )) })?); } @@ -75,10 +77,9 @@ impl Connection for MySqlConnection { options = options.database(database); } - let pool = options - .connect() - .await - .map_err(|e| SharedError::Sql(Sql::Connect(format!("{:?}: {e}", self.database))))?; + let pool = options.connect().await.map_err(|e| { + SharedError::Sql(SqlError::Connect(format!("{:?}: {e}", self.database))) + })?; Ok(pool) } @@ -97,7 +98,7 @@ impl Connection for MySqlConnection { let mut stream = sqlx::query(sql).fetch(&mut pool); while let Some(row) = stream.next().await { - let row = row.map_err(|e| SharedError::Sql(Sql::Query(e.to_string())))?; + let row = row.map_err(|e| SharedError::Sql(SqlError::Query(e.to_string())))?; bytes += row.len() as u64; if bytes > max_bytes { @@ -111,7 +112,7 @@ impl Connection for MySqlConnection { rows = sqlx::query(sql) .fetch_all(&mut pool) .await - .map_err(|e| SharedError::Sql(Sql::Query(e.to_string())))?; + .map_err(|e| SharedError::Sql(SqlError::Query(e.to_string())))?; } Ok((rows, over_the_limit)) @@ -119,7 +120,9 @@ impl Connection for MySqlConnection { async fn schema(&self, pool: Self::Conn) -> Result { let database = self.database.as_ref().ok_or_else(|| { - SharedError::Sql(Sql::Schema("Database name is required for MySQL".into())) + SharedError::Sql(SqlError::Schema( + "Database name is required for MySQL".into(), + )) })?; let sql = format!(" diff --git a/quadratic-rust-shared/src/sql/postgres_connection.rs b/quadratic-rust-shared/src/sql/postgres_connection.rs index 4ee1f249b0..c1406ca169 100644 --- a/quadratic-rust-shared/src/sql/postgres_connection.rs +++ b/quadratic-rust-shared/src/sql/postgres_connection.rs @@ -13,11 +13,11 @@ use sqlx::{ }; use uuid::Uuid; -use crate::error::{Result, SharedError, Sql}; +use crate::error::{Result, SharedError}; use crate::sql::{ArrowType, Connection}; use crate::{ convert_pg_type, - sql::{DatabaseSchema, SchemaColumn, SchemaTable}, + sql::{error::Sql as SqlError, DatabaseSchema, SchemaColumn, SchemaTable}, }; #[derive(Debug, Serialize, Deserialize)] @@ -67,7 +67,9 @@ impl Connection for PostgresConnection { if let Some(ref port) = self.port { options = options.port(port.parse::().map_err(|_| { - SharedError::Sql(Sql::Connect("Could not parse port into a number".into())) + SharedError::Sql(SqlError::Connect( + "Could not parse port into a number".into(), + )) })?); } @@ -75,10 +77,9 @@ impl Connection for PostgresConnection { options = options.database(database); } - let pool = options - .connect() - .await - .map_err(|e| SharedError::Sql(Sql::Connect(format!("{:?}: {e}", self.database))))?; + let pool = options.connect().await.map_err(|e| { + SharedError::Sql(SqlError::Connect(format!("{:?}: {e}", self.database))) + })?; Ok(pool) } @@ -97,7 +98,7 @@ impl Connection for PostgresConnection { let mut stream = sqlx::query(sql).fetch(&mut pool); while let Some(row) = stream.next().await { - let row = row.map_err(|e| SharedError::Sql(Sql::Query(e.to_string())))?; + let row = row.map_err(|e| SharedError::Sql(SqlError::Query(e.to_string())))?; bytes += row.len() as u64; if bytes > max_bytes { @@ -112,7 +113,7 @@ impl Connection for PostgresConnection { rows = sqlx::query(sql) .fetch_all(&mut pool) .await - .map_err(|e| SharedError::Sql(Sql::Query(e.to_string())))?; + .map_err(|e| SharedError::Sql(SqlError::Query(e.to_string())))?; } Ok((rows, over_the_limit)) @@ -120,7 +121,9 @@ impl Connection for PostgresConnection { async fn schema(&self, pool: Self::Conn) -> Result { let database = self.database.as_ref().ok_or_else(|| { - SharedError::Sql(Sql::Schema("Database name is required for MySQL".into())) + SharedError::Sql(SqlError::Schema( + "Database name is required for MySQL".into(), + )) })?; let sql = format!(" diff --git a/quadratic-rust-shared/src/storage/error.rs b/quadratic-rust-shared/src/storage/error.rs new file mode 100644 index 0000000000..dffd5ab46b --- /dev/null +++ b/quadratic-rust-shared/src/storage/error.rs @@ -0,0 +1,17 @@ +use serde::{Deserialize, Serialize}; +use thiserror::Error; + +#[derive(Error, Debug, Serialize, Deserialize, PartialEq, Clone)] +pub enum Storage { + #[error("Error creating directory {0}: {1}")] + CreateDirectory(String, String), + + #[error("Invalid key: {0}")] + InvalidKey(String), + + #[error("Error reading key {0}: {1}")] + Read(String, String), + + #[error("Error writing key {0}: {1}")] + Write(String, String), +} diff --git a/quadratic-rust-shared/src/storage/file_system.rs b/quadratic-rust-shared/src/storage/file_system.rs index 3f13f6d58d..e058f7314b 100644 --- a/quadratic-rust-shared/src/storage/file_system.rs +++ b/quadratic-rust-shared/src/storage/file_system.rs @@ -6,21 +6,25 @@ use tokio::io::{AsyncReadExt, AsyncWriteExt}; use super::Storage; use crate::error::Result; +use crate::storage::error::Storage as StorageError; use crate::SharedError; -use crate::Storage as StorageError; -#[derive(Debug)] +#[derive(Debug, Clone)] pub struct FileSystemConfig { pub path: String, + pub encryption_keys: Vec, } -#[derive(Debug)] +#[derive(Debug, Clone)] pub struct FileSystem { pub config: FileSystemConfig, } #[async_trait] impl Storage for FileSystem { + type Config = FileSystemConfig; + + /// Read the file from the file system and return the bytes. async fn read(&self, key: &str) -> Result { let file_path = self.full_path(key, false).await?.0; let mut bytes = vec![]; @@ -35,6 +39,7 @@ impl Storage for FileSystem { Ok(bytes.into()) } + /// Write the bytes to the file system. async fn write<'a>(&self, key: &'a str, data: &'a Bytes) -> Result<()> { let file_path = self.full_path(key, true).await?.0; let mut file = File::create(file_path) @@ -47,9 +52,15 @@ impl Storage for FileSystem { Ok(()) } + /// Return the path to the file system. fn path(&self) -> &str { &self.config.path } + + /// Return the configuration + fn config(&self) -> Self::Config { + self.config.clone() + } } impl FileSystem { @@ -57,8 +68,9 @@ impl FileSystem { Self { config } } + /// Return the full path to the file and the directory. pub async fn full_path(&self, key: &str, create_dir: bool) -> Result<(PathBuf, PathBuf)> { - let FileSystemConfig { path } = &self.config; + let FileSystemConfig { path, .. } = &self.config; let parts = key.split('-').collect::>(); let invalid_key = || SharedError::Storage(StorageError::InvalidKey(key.to_owned())); @@ -69,7 +81,7 @@ impl FileSystem { } let uuid = &parts[0..parts.len() - 1].join("-"); - let file_name = parts.last().ok_or_else(|| invalid_key())?; + let file_name = parts.last().ok_or_else(invalid_key)?; let dir = Path::new(path).join(uuid); let full_path = dir.join(file_name); @@ -97,6 +109,7 @@ mod tests { fn config() -> FileSystemConfig { FileSystemConfig { path: env::temp_dir().to_str().unwrap().to_string(), + encryption_keys: vec![], } } diff --git a/quadratic-rust-shared/src/storage/mod.rs b/quadratic-rust-shared/src/storage/mod.rs index 96814881bb..4a8660db62 100644 --- a/quadratic-rust-shared/src/storage/mod.rs +++ b/quadratic-rust-shared/src/storage/mod.rs @@ -3,13 +3,14 @@ use bytes::Bytes; use file_system::FileSystemConfig; use s3::S3Config; -use crate::{error::Result, SharedError, Storage as StorageError}; +use crate::{error::Result, storage::error::Storage as StorageError, SharedError}; +pub mod error; pub mod file_system; pub mod s3; #[derive(Debug)] -pub enum Config { +pub enum StorageConfig { S3(S3Config), FileSystem(FileSystemConfig), } @@ -22,9 +23,12 @@ pub enum StorageContainer { #[async_trait] pub trait Storage { + type Config; + async fn read(&self, key: &str) -> Result; async fn write<'a>(&self, key: &'a str, data: &'a Bytes) -> Result<()>; fn path(&self) -> &str; + fn config(&self) -> Self::Config; fn read_error(key: &str, e: impl ToString) -> SharedError { SharedError::Storage(StorageError::Read(key.into(), e.to_string())) @@ -38,6 +42,8 @@ pub trait Storage { // TODO(ddimaria): this is a temp hack to get around some trait issues, do something better #[async_trait] impl Storage for StorageContainer { + type Config = StorageConfig; + async fn read(&self, key: &str) -> Result { match self { Self::S3(s3) => s3.read(key).await, @@ -58,4 +64,11 @@ impl Storage for StorageContainer { Self::FileSystem(fs) => fs.path(), } } + + fn config(&self) -> StorageConfig { + match self { + Self::S3(s3) => StorageConfig::S3(s3.config()), + Self::FileSystem(fs) => StorageConfig::FileSystem(fs.config()), + } + } } diff --git a/quadratic-rust-shared/src/storage/s3.rs b/quadratic-rust-shared/src/storage/s3.rs index 672980b106..f9fcdd70c2 100644 --- a/quadratic-rust-shared/src/storage/s3.rs +++ b/quadratic-rust-shared/src/storage/s3.rs @@ -8,19 +8,22 @@ use crate::{ error::Result, }; -#[derive(Debug)] +#[derive(Debug, Clone)] pub struct S3Config { pub client: Client, pub bucket: String, } -#[derive(Debug)] +#[derive(Debug, Clone)] pub struct S3 { pub config: S3Config, } #[async_trait] impl Storage for S3 { + type Config = S3Config; + + /// Read the file from the S3 bucket and return the bytes. async fn read(&self, key: &str) -> Result { let S3Config { client, bucket } = &self.config; @@ -38,6 +41,7 @@ impl Storage for S3 { Ok(bytes) } + /// Write the bytes to the S3 bucket. async fn write<'a>(&self, key: &'a str, data: &'a Bytes) -> Result<()> { let S3Config { client, bucket } = &self.config; @@ -48,9 +52,15 @@ impl Storage for S3 { Ok(()) } + /// Return the S3 bucket. fn path(&self) -> &str { &self.config.bucket } + + /// Return the configuration + fn config(&self) -> Self::Config { + self.config.clone() + } } impl S3 { @@ -60,4 +70,6 @@ impl S3 { } #[cfg(test)] -mod tests {} +mod tests { + // TODO(ddimaria): add tests once we have S3 mocks in place +} From 310c8ab5bb8f7ca0d01c0405fa0c08ac8f68a20c Mon Sep 17 00:00:00 2001 From: David DiMaria Date: Fri, 2 Aug 2024 16:37:46 -0600 Subject: [PATCH 009/113] Cleanup --- .../src/routes/v0/files.$uuid.thumbnail.POST.ts | 2 -- quadratic-api/src/storage/fileSystem.ts | 11 +++++++++++ quadratic-api/src/storage/s3.ts | 4 +++- quadratic-api/src/storage/storage.ts | 6 +++++- quadratic-files/src/server.rs | 14 -------------- 5 files changed, 19 insertions(+), 18 deletions(-) diff --git a/quadratic-api/src/routes/v0/files.$uuid.thumbnail.POST.ts b/quadratic-api/src/routes/v0/files.$uuid.thumbnail.POST.ts index dc81f8e362..bae2496a46 100644 --- a/quadratic-api/src/routes/v0/files.$uuid.thumbnail.POST.ts +++ b/quadratic-api/src/routes/v0/files.$uuid.thumbnail.POST.ts @@ -23,8 +23,6 @@ async function handler(req: RequestWithUser & RequestWithFile, res: Response) { return res.status(403).json({ error: { message: 'Permission denied' } }); } - console.log('req.file', req.file); - // update the file object with the thumbnail URL await dbClient.file.update({ where: { diff --git a/quadratic-api/src/storage/fileSystem.ts b/quadratic-api/src/storage/fileSystem.ts index f6c7ecc759..90f9c953bb 100644 --- a/quadratic-api/src/storage/fileSystem.ts +++ b/quadratic-api/src/storage/fileSystem.ts @@ -9,15 +9,18 @@ import { UploadFileResponse } from './storage'; const generateUrl = (key: string): string => `${QUADRATIC_FILE_URI}/storage/${key}`; const generatePresignedUrl = (key: string): string => generateUrl(`presigned/${key}`); +// Get the URL for a given file (key) for the file service. export const getStorageUrl = (key: string): string => { return generateUrl(key); }; +// Get a presigned URL for a given file (key) for the file service. export const getPresignedStorageUrl = (key: string): string => { const encrypted = encryptFromEnv(key); return generatePresignedUrl(encrypted); }; +// Upload a file to the file service. export const upload = async (key: string, contents: string | Uint8Array, jwt: string): Promise => { const url = generateUrl(key); @@ -38,6 +41,7 @@ export const upload = async (key: string, contents: string | Uint8Array, jwt: st } }; +// Collect a full stream and place in a byte array. function streamToByteArray(stream: Readable): Promise { return new Promise((resolve, reject) => { const chunks: Buffer[] = []; @@ -57,6 +61,10 @@ function streamToByteArray(stream: Readable): Promise { }); } +// Multer storage engine for file-system storage. +// +// This middleware is used to handled client upload files and send them to +// the file service. export const multerFileSystemStorage: multer.Multer = multer({ storage: { _handleFile( @@ -75,9 +83,11 @@ export const multerFileSystemStorage: multer.Multer = multer({ return; } + // Create a pass-through stream to pipe the file stream to const passThrough = new stream.PassThrough(); file.stream.pipe(passThrough); + // Collect the stream and upload to the file service streamToByteArray(passThrough) .then((data) => { upload(key, data, jwt) @@ -87,6 +97,7 @@ export const multerFileSystemStorage: multer.Multer = multer({ .catch((error) => cb(error)); }, + // only implement if needed (not currently used) _removeFile(_req: Request, _file: Express.Multer.File, cb: (error: Error | null) => void): void { cb(null); }, diff --git a/quadratic-api/src/storage/s3.ts b/quadratic-api/src/storage/s3.ts index 4cebed9c1e..d0a57c0acd 100644 --- a/quadratic-api/src/storage/s3.ts +++ b/quadratic-api/src/storage/s3.ts @@ -26,6 +26,7 @@ export const s3Client = new S3Client({ forcePathStyle: true, }); +// Upload a string as a file to S3 export const uploadStringAsFileS3 = async (fileKey: string, contents: string): Promise => { const command = new PutObjectCommand({ Bucket: AWS_S3_BUCKET_NAME, @@ -47,6 +48,7 @@ export const uploadStringAsFileS3 = async (fileKey: string, contents: string): P } }; +// Multer storage engine for S3 export const multerS3Storage: multer.Multer = multer({ storage: multerS3({ s3: s3Client, @@ -61,7 +63,7 @@ export const multerS3Storage: multer.Multer = multer({ }) as StorageEngine, }); -// Get file URL from S3 +// Get the presigned file URL from S3 export const generatePresignedUrl = async (key: string): Promise => { const command = new GetObjectCommand({ Bucket: AWS_S3_BUCKET_NAME, diff --git a/quadratic-api/src/storage/storage.ts b/quadratic-api/src/storage/storage.ts index 6483436e9a..1feaa15d6a 100644 --- a/quadratic-api/src/storage/storage.ts +++ b/quadratic-api/src/storage/storage.ts @@ -8,6 +8,7 @@ export type UploadFileResponse = { key: string; }; +// Get the URL for a given file (key). export const getFileUrl = async (key: string) => { switch (STORAGE_TYPE) { case 's3': @@ -19,6 +20,7 @@ export const getFileUrl = async (key: string) => { } }; +// Get a presigned URL for a given file (key). export const getPresignedFileUrl = async (key: string) => { switch (STORAGE_TYPE) { case 's3': @@ -26,10 +28,11 @@ export const getPresignedFileUrl = async (key: string) => { case 'file-system': return getPresignedStorageUrl(key); default: - throw new Error(`Unsupported storage type in getFileUrl(): ${STORAGE_TYPE}`); + throw new Error(`Unsupported storage type in getPresignedFileUrl(): ${STORAGE_TYPE}`); } }; +// Upload a file (key). export const uploadFile = async (key: string, contents: string, jwt: string): Promise => { switch (STORAGE_TYPE) { case 's3': @@ -41,6 +44,7 @@ export const uploadFile = async (key: string, contents: string, jwt: string): Pr } }; +// Multer middleware for file uploads. export const uploadMiddleware = (): multer.Multer => { switch (STORAGE_TYPE) { case 's3': diff --git a/quadratic-files/src/server.rs b/quadratic-files/src/server.rs index 056776d734..8a829f3599 100644 --- a/quadratic-files/src/server.rs +++ b/quadratic-files/src/server.rs @@ -43,21 +43,7 @@ pub(crate) fn app(state: Arc) -> Router { let cors = CorsLayer::new() .allow_methods([Method::GET, Method::POST]) - // - // allow requests from any origin .allow_origin(Any) - // - // TODO(ddimaria): uncomment when we move proxy to a separate service - // - // .allow_headers([ - // CONTENT_TYPE, - // AUTHORIZATION, - // ACCEPT, - // ORIGIN, - // HeaderName::from_static("proxy"), - // ]) - // - // required for the proxy .allow_headers(Any) .expose_headers(Any); From c3846530cac4d823c30bbeb2f39c44a4901a4cde Mon Sep 17 00:00:00 2001 From: David DiMaria Date: Mon, 5 Aug 2024 09:55:16 -0600 Subject: [PATCH 010/113] Fix jest test mock --- quadratic-api/jest.setup.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/quadratic-api/jest.setup.js b/quadratic-api/jest.setup.js index 6f9e6c3d93..13b0a6e071 100644 --- a/quadratic-api/jest.setup.js +++ b/quadratic-api/jest.setup.js @@ -16,7 +16,7 @@ jest.mock('./src/middleware/validateAccessToken', () => { }; }); -jest.mock('./src/aws/s3', () => { +jest.mock('./src/storage/storage', () => { return { s3Client: {}, generatePresignedUrl: jest.fn().mockImplementation(async (str) => str), From 443ab196de0a45209abc5b21720785bc5e1b5d52 Mon Sep 17 00:00:00 2001 From: David DiMaria Date: Mon, 5 Aug 2024 10:11:20 -0600 Subject: [PATCH 011/113] Fix docker compose down command in quadratic-connection --- quadratic-connection/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/quadratic-connection/package.json b/quadratic-connection/package.json index f19de47186..7e30bd504f 100644 --- a/quadratic-connection/package.json +++ b/quadratic-connection/package.json @@ -17,6 +17,6 @@ "coverage:html": "grcov . --binary-path ./target/debug/deps/ -s . -t html --branch --ignore-not-existing --ignore '../*' --ignore '/*' -o coverage/html", "coverage:view": "open coverage/html/index.html", "docker:up": "docker compose --profile quadratic-connection-test up -d && sleep 3", - "docker:down": "docker compose down --profile quadratic-connection-test" + "docker:down": "docker compose --profile quadratic-connection-test down" } } From f6a3e6b918e533343d6cd15a8a07b681e78a12e9 Mon Sep 17 00:00:00 2001 From: David DiMaria Date: Mon, 5 Aug 2024 14:12:21 -0600 Subject: [PATCH 012/113] Fix quadratic-api test mocks --- quadratic-api/.env.docker | 2 +- quadratic-api/.env.example | 2 +- quadratic-api/.env.test | 2 +- quadratic-api/jest.setup.js | 8 ++++++-- quadratic-api/src/storage/storage.ts | 1 + 5 files changed, 10 insertions(+), 5 deletions(-) diff --git a/quadratic-api/.env.docker b/quadratic-api/.env.docker index a28e8b7f5a..33231b72e0 100644 --- a/quadratic-api/.env.docker +++ b/quadratic-api/.env.docker @@ -5,5 +5,5 @@ DATABASE_URL='postgresql://postgres:postgres@postgres:5432/postgres' ENCRYPTION_KEY=eb4758047f74bdb2603cce75c4370327ca2c3662c4786867659126da8e64dfcc # Storage -STORAGE_TYPE=file-system +STORAGE_TYPE=s3 # s3 or file-system QUADRATIC_FILE_URI=http://127.0.0.1:3002 \ No newline at end of file diff --git a/quadratic-api/.env.example b/quadratic-api/.env.example index 2eaf2b05c1..2fbddae355 100644 --- a/quadratic-api/.env.example +++ b/quadratic-api/.env.example @@ -23,7 +23,7 @@ STRIPE_WEBHOOK_SECRET=STRIPE_WEBHOOK_SECRET ENCRYPTION_KEY=eb4758047f74bdb2603cce75c4370327ca2c3662c4786867659126da8e64dfcc # Storage -STORAGE_TYPE=file-system # s3 or file-system +STORAGE_TYPE=s3 # s3 or file-system QUADRATIC_FILE_URI=http://127.0.0.1:3002 AWS_S3_REGION=us-east-2 AWS_S3_ACCESS_KEY_ID=test diff --git a/quadratic-api/.env.test b/quadratic-api/.env.test index 1c3a71065d..c508c8761e 100644 --- a/quadratic-api/.env.test +++ b/quadratic-api/.env.test @@ -12,7 +12,7 @@ STRIPE_WEBHOOK_SECRET=STRIPE_WEBHOOK_SECRET ENCRYPTION_KEY=eb4758047f74bdb2603cce75c4370327ca2c3662c4786867659126da8e64dfcc # Storage -STORAGE_TYPE=file-system # s3 or file-system +STORAGE_TYPE=s3 # s3 or file-system QUADRATIC_FILE_URI=http://127.0.0.1:3002 AWS_S3_REGION=us-west-2 AWS_S3_BUCKET_NAME=AWS_S3_BUCKET_NAME diff --git a/quadratic-api/jest.setup.js b/quadratic-api/jest.setup.js index 13b0a6e071..46579e6f08 100644 --- a/quadratic-api/jest.setup.js +++ b/quadratic-api/jest.setup.js @@ -1,3 +1,5 @@ +const { multerS3Storage } = require('./src/storage/s3'); + // For auth we expect the following Authorization header format: // Bearer ValidToken {user.sub} jest.mock('./src/middleware/validateAccessToken', () => { @@ -19,10 +21,12 @@ jest.mock('./src/middleware/validateAccessToken', () => { jest.mock('./src/storage/storage', () => { return { s3Client: {}, - generatePresignedUrl: jest.fn().mockImplementation(async (str) => str), - uploadStringAsFileS3: jest.fn().mockImplementation(async () => { + getFileUrl: jest.fn().mockImplementation(async (str) => str), + getPresignedFileUrl: jest.fn().mockImplementation(async (str) => str), + uploadFile: jest.fn().mockImplementation(async () => { return { bucket: 'test-bucket', key: 'test-key' }; }), + uploadMiddleware: () => multerS3Storage, }; }); diff --git a/quadratic-api/src/storage/storage.ts b/quadratic-api/src/storage/storage.ts index 1feaa15d6a..ba8791d769 100644 --- a/quadratic-api/src/storage/storage.ts +++ b/quadratic-api/src/storage/storage.ts @@ -22,6 +22,7 @@ export const getFileUrl = async (key: string) => { // Get a presigned URL for a given file (key). export const getPresignedFileUrl = async (key: string) => { + console.warn('getPresignedFileUrl', getPresignedFileUrl); switch (STORAGE_TYPE) { case 's3': return await generatePresignedUrl(key); From 35ad668066bc2d6bc78409ac99c8528480b79588 Mon Sep 17 00:00:00 2001 From: David DiMaria Date: Tue, 6 Aug 2024 09:48:11 -0600 Subject: [PATCH 013/113] Abstract auth functionality in API, switch using env var AUTH_TYPE --- quadratic-api/.env.docker | 3 +++ quadratic-api/.env.example | 16 +++++++++------- quadratic-api/.env.test | 13 ++++++++----- quadratic-api/src/auth/auth.ts | 16 ++++++++++++++++ .../src/{auth0/profile.ts => auth/auth0.ts} | 0 quadratic-api/src/env-vars.ts | 4 +++- quadratic-api/src/middleware/user.ts | 4 ++-- quadratic-api/src/routes/v0/education.POST.ts | 4 ++-- .../src/routes/v0/files.$uuid.invites.POST.ts | 4 ++-- .../src/routes/v0/files.$uuid.sharing.GET.ts | 4 ++-- quadratic-api/src/routes/v0/teams.$uuid.GET.ts | 4 ++-- .../src/routes/v0/teams.$uuid.invites.POST.ts | 4 ++-- quadratic-client/.env.docker | 3 +++ quadratic-client/.env.example | 9 ++++++--- 14 files changed, 60 insertions(+), 28 deletions(-) create mode 100644 quadratic-api/src/auth/auth.ts rename quadratic-api/src/{auth0/profile.ts => auth/auth0.ts} (100%) diff --git a/quadratic-api/.env.docker b/quadratic-api/.env.docker index 33231b72e0..b08992efb7 100644 --- a/quadratic-api/.env.docker +++ b/quadratic-api/.env.docker @@ -4,6 +4,9 @@ DATABASE_URL='postgresql://postgres:postgres@postgres:5432/postgres' # Hex string to be used as the key for enctyption, use npm run key:generate ENCRYPTION_KEY=eb4758047f74bdb2603cce75c4370327ca2c3662c4786867659126da8e64dfcc +# Auth +AUTH_TYPE=auth0 # auth0 or ? + # Storage STORAGE_TYPE=s3 # s3 or file-system QUADRATIC_FILE_URI=http://127.0.0.1:3002 \ No newline at end of file diff --git a/quadratic-api/.env.example b/quadratic-api/.env.example index 2fbddae355..155365901c 100644 --- a/quadratic-api/.env.example +++ b/quadratic-api/.env.example @@ -4,13 +4,6 @@ DATABASE_URL=postgresql://postgres:postgres@0.0.0.0:5432/postgres CORS='*' -AUTH0_JWKS_URI=https://quadratic-community.us.auth0.com/.well-known/jwks.json -AUTH0_ISSUER=https://quadratic-community.us.auth0.com/ -AUTH0_DOMAIN=quadratic-community.us.auth0.com -AUTH0_CLIENT_ID=DCPCvqyU5Q0bJD8Q3QmJEoV48x1zLH7W -AUTH0_CLIENT_SECRET=94dp3PDcxlI9ZDqBSvkdjQHWgGdx0ZSeyTr5-Rn3Kcts-ZyTdj1FLlJjCyqrTXEG -AUTH0_AUDIENCE=community-quadratic - OPENAI_API_KEY= SENTRY_DSN= @@ -22,6 +15,15 @@ STRIPE_WEBHOOK_SECRET=STRIPE_WEBHOOK_SECRET ENCRYPTION_KEY=eb4758047f74bdb2603cce75c4370327ca2c3662c4786867659126da8e64dfcc +# Auth +AUTH_TYPE=auth0 # auth0 or ? +AUTH0_JWKS_URI=https://quadratic-community.us.auth0.com/.well-known/jwks.json +AUTH0_ISSUER=https://quadratic-community.us.auth0.com/ +AUTH0_DOMAIN=quadratic-community.us.auth0.com +AUTH0_CLIENT_ID=DCPCvqyU5Q0bJD8Q3QmJEoV48x1zLH7W +AUTH0_CLIENT_SECRET=94dp3PDcxlI9ZDqBSvkdjQHWgGdx0ZSeyTr5-Rn3Kcts-ZyTdj1FLlJjCyqrTXEG +AUTH0_AUDIENCE=community-quadratic + # Storage STORAGE_TYPE=s3 # s3 or file-system QUADRATIC_FILE_URI=http://127.0.0.1:3002 diff --git a/quadratic-api/.env.test b/quadratic-api/.env.test index c508c8761e..a0e6f64c1c 100644 --- a/quadratic-api/.env.test +++ b/quadratic-api/.env.test @@ -1,9 +1,4 @@ DATABASE_URL='postgresql://prisma:prisma@localhost:5433/quadratic-api’' -AUTH0_JWKS_URI='https://dev.us.auth0.com/.well-known/jwks.json' -AUTH0_ISSUER='https://auth-dev.quadratic.to/' -AUTH0_CLIENT_ID="AUTH0_CLIENT_ID" -AUTH0_CLIENT_SECRET="AUTH0_CLIENT_SECRET" -AUTH0_DOMAIN="AUTH0_DOMAIN" M2M_AUTH_TOKEN=M2M_AUTH_TOKEN STRIPE_SECRET_KEY=STRIPE_SECRET_KEY STRIPE_WEBHOOK_SECRET=STRIPE_WEBHOOK_SECRET @@ -11,6 +6,14 @@ STRIPE_WEBHOOK_SECRET=STRIPE_WEBHOOK_SECRET # Hex string to be used as the key for enctyption, use npm run key:generate ENCRYPTION_KEY=eb4758047f74bdb2603cce75c4370327ca2c3662c4786867659126da8e64dfcc +# Auth +AUTH_TYPE=auth0 # auth0 or ? +AUTH0_JWKS_URI='https://dev.us.auth0.com/.well-known/jwks.json' +AUTH0_ISSUER='https://auth-dev.quadratic.to/' +AUTH0_CLIENT_ID="AUTH0_CLIENT_ID" +AUTH0_CLIENT_SECRET="AUTH0_CLIENT_SECRET" +AUTH0_DOMAIN="AUTH0_DOMAIN" + # Storage STORAGE_TYPE=s3 # s3 or file-system QUADRATIC_FILE_URI=http://127.0.0.1:3002 diff --git a/quadratic-api/src/auth/auth.ts b/quadratic-api/src/auth/auth.ts new file mode 100644 index 0000000000..83008b7ced --- /dev/null +++ b/quadratic-api/src/auth/auth.ts @@ -0,0 +1,16 @@ +import { AUTH_TYPE } from '../env-vars'; +import { getUsersFromAuth0 } from './auth0'; + +export type UsersRequest = { + id: number; + auth0Id: string; +}; + +export const getUsers = async (users: UsersRequest[]) => { + switch (AUTH_TYPE) { + case 'auth0': + return await getUsersFromAuth0(users); + default: + throw new Error(`Unsupported auth type in getUsers(): ${AUTH_TYPE}`); + } +}; diff --git a/quadratic-api/src/auth0/profile.ts b/quadratic-api/src/auth/auth0.ts similarity index 100% rename from quadratic-api/src/auth0/profile.ts rename to quadratic-api/src/auth/auth0.ts diff --git a/quadratic-api/src/env-vars.ts b/quadratic-api/src/env-vars.ts index 26ea45ed45..0532c9de7b 100644 --- a/quadratic-api/src/env-vars.ts +++ b/quadratic-api/src/env-vars.ts @@ -25,6 +25,7 @@ export const AUTH0_AUDIENCE = process.env.AUTH0_AUDIENCE as string; export const STRIPE_SECRET_KEY = process.env.STRIPE_SECRET_KEY as string; export const ENCRYPTION_KEY = process.env.ENCRYPTION_KEY as string; export const STORAGE_TYPE = process.env.STORAGE_TYPE as string; +export const AUTH_TYPE = process.env.AUTH_TYPE as string; [ 'AUTH0_DOMAIN', 'AUTH0_CLIENT_ID', @@ -32,9 +33,10 @@ export const STORAGE_TYPE = process.env.STORAGE_TYPE as string; 'AUTH0_JWKS_URI', 'AUTH0_ISSUER', 'AUTH0_AUDIENCE', - 'STORAGE_TYPE', 'STRIPE_SECRET_KEY', 'ENCRYPTION_KEY', + 'STORAGE_TYPE', + 'AUTH_TYPE', ].forEach(ensureEnvVarExists); // Required in prod, optional locally diff --git a/quadratic-api/src/middleware/user.ts b/quadratic-api/src/middleware/user.ts index 4bf5dfe2cd..46c2207804 100644 --- a/quadratic-api/src/middleware/user.ts +++ b/quadratic-api/src/middleware/user.ts @@ -1,5 +1,5 @@ import { NextFunction, Request, Response } from 'express'; -import { getUsersFromAuth0 } from '../auth0/profile'; +import { getUsers } from '../auth/auth'; import dbClient from '../dbClient'; import { addUserToTeam } from '../internal/addUserToTeam'; import { RequestWithAuth, RequestWithOptionalAuth, RequestWithUser } from '../types/Request'; @@ -8,7 +8,7 @@ const runFirstTimeUserLogic = async (user: Awaited { diff --git a/quadratic-api/src/routes/v0/files.$uuid.invites.POST.ts b/quadratic-api/src/routes/v0/files.$uuid.invites.POST.ts index fef6dc3cc9..7da048b755 100644 --- a/quadratic-api/src/routes/v0/files.$uuid.invites.POST.ts +++ b/quadratic-api/src/routes/v0/files.$uuid.invites.POST.ts @@ -2,7 +2,7 @@ import * as Sentry from '@sentry/node'; import { Response } from 'express'; import { ApiSchemas, ApiTypes, FilePermissionSchema } from 'quadratic-shared/typesAndSchemas'; import { z } from 'zod'; -import { getUsersFromAuth0, lookupUsersFromAuth0ByEmail } from '../../auth0/profile'; +import { getUsers, lookupUsersFromAuth0ByEmail } from '../../auth/auth'; import dbClient from '../../dbClient'; import { sendEmail } from '../../email/sendEmail'; import { templates } from '../../email/templates'; @@ -62,7 +62,7 @@ async function handler(req: RequestWithUser, res: Response user)); + const auth0UsersById = await getUsers(dbUsers.map(({ user }) => user)); // Get signed thumbnail URLs await Promise.all( diff --git a/quadratic-api/src/routes/v0/teams.$uuid.invites.POST.ts b/quadratic-api/src/routes/v0/teams.$uuid.invites.POST.ts index 0af3fe7461..06a4892753 100644 --- a/quadratic-api/src/routes/v0/teams.$uuid.invites.POST.ts +++ b/quadratic-api/src/routes/v0/teams.$uuid.invites.POST.ts @@ -2,7 +2,7 @@ import * as Sentry from '@sentry/node'; import { Response } from 'express'; import { ApiSchemas, ApiTypes } from 'quadratic-shared/typesAndSchemas'; import { z } from 'zod'; -import { getUsersFromAuth0, lookupUsersFromAuth0ByEmail } from '../../auth0/profile'; +import { getUsers, lookupUsersFromAuth0ByEmail } from '../../auth/auth'; import dbClient from '../../dbClient'; import { sendEmail } from '../../email/sendEmail'; import { templates } from '../../email/templates'; @@ -68,7 +68,7 @@ async function handler(req: RequestWithUser, res: Response Date: Wed, 7 Aug 2024 14:55:01 -0600 Subject: [PATCH 014/113] Full kratos integration into docker compose --- docker-compose.base.yml | 1 - docker-compose.yml | 74 +++++++++++++ docker/ory-auth/config/identity.schema.json | 47 ++++++++ docker/ory-auth/config/kratos.yml | 102 ++++++++++++++++++ docker/postgres/scripts/init.sh | 18 ++++ .../menus/TopBar/SubMenus/QuadraticMenu.tsx | 4 +- quadratic-client/src/{ => auth}/auth.ts | 2 +- quadratic-client/src/auth/auth0.ts | 0 quadratic-client/src/auth/ory.ts | 0 quadratic-client/src/router.tsx | 2 +- 10 files changed, 245 insertions(+), 5 deletions(-) create mode 100644 docker/ory-auth/config/identity.schema.json create mode 100644 docker/ory-auth/config/kratos.yml create mode 100755 docker/postgres/scripts/init.sh rename quadratic-client/src/{ => auth}/auth.ts (99%) create mode 100644 quadratic-client/src/auth/auth0.ts create mode 100644 quadratic-client/src/auth/ory.ts diff --git a/docker-compose.base.yml b/docker-compose.base.yml index 534485e0bc..99bdd897b9 100644 --- a/docker-compose.base.yml +++ b/docker-compose.base.yml @@ -23,7 +23,6 @@ services: POSTGRES_USER: postgres PGUSER: postgres POSTGRES_PASSWORD: postgres - POSTGRES_DB: postgres healthcheck: test: ["CMD-SHELL", "pg_isready -U postgres -d postgres"] interval: 10s diff --git a/docker-compose.yml b/docker-compose.yml index 9540d13fec..71d16f21ba 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -15,8 +15,11 @@ services: extends: file: docker-compose.base.yml service: postgres + environment: + ADDITIONAL_DATABASES: kratos volumes: - ./docker/postgres/data:/var/lib/postgresql/data + - ./docker/postgres/scripts:/docker-entrypoint-initdb.d profiles: - base - all @@ -154,6 +157,77 @@ services: - quadratic-connection - all + # Auth Providers + + ory-auth: + image: oryd/kratos:v1.2.0 + ports: + - "4433:4433" # public + - "4434:4434" # admin + command: serve -c /etc/config/kratos/kratos.yml --dev --watch-courier + volumes: + - ./docker/ory-auth/config:/etc/config/kratos + environment: + DSN: postgresql://postgres:postgres@host.docker.internal:5432/kratos?sslmode=disable + LOG_LEVEL: trace + restart: unless-stopped + depends_on: + - postgres + - ory-auth-migrate + profiles: + - ory + - all + networks: + - host + + ory-auth-migrate: + image: oryd/kratos:v1.2.0 + command: migrate -c /etc/config/kratos/kratos.yml sql -e --yes + volumes: + - ./docker/ory-auth/config:/etc/config/kratos + environment: + DSN: postgresql://postgres:postgres@host.docker.internal:5432/kratos?sslmode=disable + restart: on-failure + depends_on: + - postgres + profiles: + - ory + - all + networks: + - host + + ory-auth-node: + image: oryd/kratos-selfservice-ui-node:v1.2.0 + ports: + - "4455:4455" + environment: + PORT: 4455 + SECURITY_MODE: + KRATOS_PUBLIC_URL: http://host.docker.internal:4433/ + KRATOS_BROWSER_URL: http://127.0.0.1:4433/ + COOKIE_SECRET: changeme + CSRF_COOKIE_NAME: ory_csrf_ui + CSRF_COOKIE_SECRET: changeme + restart: on-failure + profiles: + - ory + - all + networks: + - host + + ory-auth-mail: + image: oryd/mailslurper:latest-smtps + ports: + - "1025:1025" + - "4436:4436" + - "4437:4437" + - "8080:8080" + profiles: + - ory + - all + networks: + - host + # Databases to be used for testing by the connection service postgres-connection: diff --git a/docker/ory-auth/config/identity.schema.json b/docker/ory-auth/config/identity.schema.json new file mode 100644 index 0000000000..a953fc68ec --- /dev/null +++ b/docker/ory-auth/config/identity.schema.json @@ -0,0 +1,47 @@ +{ + "$id": "https://schemas.ory.sh/presets/kratos/quickstart/email-password/identity.schema.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Person", + "type": "object", + "properties": { + "traits": { + "type": "object", + "properties": { + "email": { + "type": "string", + "format": "email", + "title": "E-Mail", + "minLength": 3, + "ory.sh/kratos": { + "credentials": { + "password": { + "identifier": true + } + }, + "verification": { + "via": "email" + }, + "recovery": { + "via": "email" + } + } + }, + "name": { + "type": "object", + "properties": { + "first": { + "title": "First Name", + "type": "string" + }, + "last": { + "title": "Last Name", + "type": "string" + } + } + } + }, + "required": ["email"], + "additionalProperties": false + } + } +} diff --git a/docker/ory-auth/config/kratos.yml b/docker/ory-auth/config/kratos.yml new file mode 100644 index 0000000000..50c79ea997 --- /dev/null +++ b/docker/ory-auth/config/kratos.yml @@ -0,0 +1,102 @@ +version: v0.13.0 + +dsn: memory + +serve: + public: + base_url: http://127.0.0.1:4433/ + cors: + enabled: true + admin: + base_url: http://kratos:4434/ + +selfservice: + default_browser_return_url: http://127.0.0.1:4455/welcome + allowed_return_urls: + - http://127.0.0.1:4455 + - http://localhost:19006/Callback + - exp://localhost:8081/--/Callback + + methods: + password: + enabled: true + totp: + config: + issuer: Kratos + enabled: true + lookup_secret: + enabled: true + link: + enabled: true + code: + enabled: true + + flows: + error: + ui_url: http://127.0.0.1:4455/error + + settings: + ui_url: http://127.0.0.1:4455/settings + privileged_session_max_age: 15m + required_aal: highest_available + + recovery: + enabled: true + ui_url: http://127.0.0.1:4455/recovery + use: code + + verification: + enabled: true + ui_url: http://127.0.0.1:4455/verification + use: code + after: + default_browser_return_url: http://127.0.0.1:4455/welcome + + logout: + after: + default_browser_return_url: http://127.0.0.1:4455/login + + login: + ui_url: http://127.0.0.1:4455/login + lifespan: 10m + + registration: + lifespan: 10m + ui_url: http://127.0.0.1:4455/registration + after: + password: + hooks: + - hook: session + - hook: show_verification_ui + +log: + level: debug + format: text + leak_sensitive_values: true + +secrets: + cookie: + - PLEASE-CHANGE-ME-I-AM-VERY-INSECURE + cipher: + - 32-LONG-SECRET-NOT-SECURE-AT-ALL + +ciphers: + algorithm: xchacha20-poly1305 + +hashers: + algorithm: bcrypt + bcrypt: + cost: 8 + +identity: + default_schema_id: default + schemas: + - id: default + url: file:///etc/config/kratos/identity.schema.json + +courier: + smtp: + connection_uri: smtps://test:test@host.docker.internal:1025/?skip_ssl_verify=true + +feature_flags: + use_continue_with_transitions: true \ No newline at end of file diff --git a/docker/postgres/scripts/init.sh b/docker/postgres/scripts/init.sh new file mode 100755 index 0000000000..49102ba731 --- /dev/null +++ b/docker/postgres/scripts/init.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +set -e +set -u + +function create_user_and_database() { + local database=$1 + echo "Creating database '$database' with user '$POSTGRES_USER'" + psql -c "CREATE DATABASE $database;" || { echo "Failed to create database '$database'"; exit 1; } + echo "Database '$database' created" +} + +if [ -n "$ADDITIONAL_DATABASES" ]; then + for i in ${ADDITIONAL_DATABASES//,/ } + do + create_user_and_database $1 + done +fi diff --git a/quadratic-client/src/app/ui/menus/TopBar/SubMenus/QuadraticMenu.tsx b/quadratic-client/src/app/ui/menus/TopBar/SubMenus/QuadraticMenu.tsx index 971947ec15..362e6a29df 100644 --- a/quadratic-client/src/app/ui/menus/TopBar/SubMenus/QuadraticMenu.tsx +++ b/quadratic-client/src/app/ui/menus/TopBar/SubMenus/QuadraticMenu.tsx @@ -1,7 +1,7 @@ import { quadraticCore } from '@/app/web-workers/quadraticCore/quadraticCore'; import { useRootRouteLoaderData } from '@/routes/_root'; -import { useFileRouteLoaderData } from '@/shared/hooks/useFileRouteLoaderData'; import { useGlobalSnackbar } from '@/shared/components/GlobalSnackbarProvider'; +import { useFileRouteLoaderData } from '@/shared/hooks/useFileRouteLoaderData'; import { isMac } from '@/shared/utils/isMac'; import { Check } from '@mui/icons-material'; import { Menu, MenuDivider, MenuItem, SubMenu } from '@szhsin/react-menu'; @@ -11,7 +11,7 @@ import { isMobile } from 'react-device-detect'; import { useParams } from 'react-router'; import { useNavigate, useSubmit } from 'react-router-dom'; import { useRecoilState } from 'recoil'; -import { authClient } from '../../../../../auth'; +import { authClient } from '../../../../../auth/auth'; import { copyAction, createNewFileAction, diff --git a/quadratic-client/src/auth.ts b/quadratic-client/src/auth/auth.ts similarity index 99% rename from quadratic-client/src/auth.ts rename to quadratic-client/src/auth/auth.ts index 6734967654..b6d5374c88 100644 --- a/quadratic-client/src/auth.ts +++ b/quadratic-client/src/auth/auth.ts @@ -2,7 +2,7 @@ import { Auth0Client, User, createAuth0Client } from '@auth0/auth0-spa-js'; import * as Sentry from '@sentry/react'; import { useEffect } from 'react'; import { LoaderFunction, LoaderFunctionArgs, redirect } from 'react-router-dom'; -import { ROUTES } from './shared/constants/routes'; +import { ROUTES } from '../shared/constants/routes'; const AUTH0_DOMAIN = import.meta.env.VITE_AUTH0_DOMAIN || ''; const AUTH0_CLIENT_ID = import.meta.env.VITE_AUTH0_CLIENT_ID || ''; diff --git a/quadratic-client/src/auth/auth0.ts b/quadratic-client/src/auth/auth0.ts new file mode 100644 index 0000000000..e69de29bb2 diff --git a/quadratic-client/src/auth/ory.ts b/quadratic-client/src/auth/ory.ts new file mode 100644 index 0000000000..e69de29bb2 diff --git a/quadratic-client/src/router.tsx b/quadratic-client/src/router.tsx index 0a520c7cae..8a62ea0a57 100644 --- a/quadratic-client/src/router.tsx +++ b/quadratic-client/src/router.tsx @@ -13,7 +13,7 @@ import { createRoutesFromElements, redirect, } from 'react-router-dom'; -import { protectedRouteLoaderWrapper } from './auth'; +import { protectedRouteLoaderWrapper } from './auth/auth'; import * as RootRoute from './routes/_root'; export const router = createBrowserRouter( From 8b5ac52df54cdde20c3c3e0dfe2c795bc3b6ea87 Mon Sep 17 00:00:00 2001 From: David DiMaria Date: Thu, 8 Aug 2024 09:40:06 -0600 Subject: [PATCH 015/113] Abstract auth0Client into a generic AuthClient implementation --- .../app/ui/menus/CodeEditor/AiAssistant.tsx | 2 +- .../multiplayerWebWorker/multiplayer.ts | 2 +- .../pythonWebWorker/pythonWebWorker.ts | 2 +- .../quadraticCore/quadraticCore.ts | 2 +- quadratic-client/src/auth/auth.ts | 118 +++--------------- quadratic-client/src/auth/auth0.ts | 97 ++++++++++++++ .../dashboard/components/EducationDialog.tsx | 2 +- quadratic-client/src/routes/_dashboard.tsx | 2 +- quadratic-client/src/routes/_root.tsx | 2 +- quadratic-client/src/routes/file.$uuid.tsx | 2 +- quadratic-client/src/routes/login-result.tsx | 2 +- quadratic-client/src/routes/login.tsx | 2 +- quadratic-client/src/routes/logout.tsx | 2 +- .../routes/teams.$teamUuid.files.create.tsx | 2 +- .../src/shared/api/connectionClient.ts | 2 +- .../src/shared/api/fetchFromApi.ts | 2 +- 16 files changed, 131 insertions(+), 112 deletions(-) diff --git a/quadratic-client/src/app/ui/menus/CodeEditor/AiAssistant.tsx b/quadratic-client/src/app/ui/menus/CodeEditor/AiAssistant.tsx index 122cf8aed8..9cb57a65f6 100644 --- a/quadratic-client/src/app/ui/menus/CodeEditor/AiAssistant.tsx +++ b/quadratic-client/src/app/ui/menus/CodeEditor/AiAssistant.tsx @@ -7,7 +7,7 @@ import { TooltipHint } from '@/app/ui/components/TooltipHint'; import { AI } from '@/app/ui/icons'; import { useCodeEditor } from '@/app/ui/menus/CodeEditor/CodeEditorContext'; import { useConnectionSchemaFetcher } from '@/app/ui/menus/CodeEditor/useConnectionSchemaFetcher'; -import { authClient } from '@/auth'; +import { authClient } from '@/auth/auth'; import { useRootRouteLoaderData } from '@/routes/_root'; import { apiClient } from '@/shared/api/apiClient'; import { Textarea } from '@/shared/shadcn/ui/textarea'; diff --git a/quadratic-client/src/app/web-workers/multiplayerWebWorker/multiplayer.ts b/quadratic-client/src/app/web-workers/multiplayerWebWorker/multiplayer.ts index 46b3524867..4ae06c092f 100644 --- a/quadratic-client/src/app/web-workers/multiplayerWebWorker/multiplayer.ts +++ b/quadratic-client/src/app/web-workers/multiplayerWebWorker/multiplayer.ts @@ -8,7 +8,7 @@ import { pixiAppSettings } from '@/app/gridGL/pixiApp/PixiAppSettings'; import { SheetPosTS } from '@/app/gridGL/types/size'; import type { CodeRun } from '@/app/web-workers/CodeRun'; import { LanguageState } from '@/app/web-workers/languageTypes'; -import { authClient, parseDomain } from '@/auth'; +import { authClient, parseDomain } from '@/auth/auth'; import { displayName } from '@/shared/utils/userUtil'; import { User } from '@auth0/auth0-spa-js'; import * as Sentry from '@sentry/react'; diff --git a/quadratic-client/src/app/web-workers/pythonWebWorker/pythonWebWorker.ts b/quadratic-client/src/app/web-workers/pythonWebWorker/pythonWebWorker.ts index 4e7899871c..7bbf4266d4 100644 --- a/quadratic-client/src/app/web-workers/pythonWebWorker/pythonWebWorker.ts +++ b/quadratic-client/src/app/web-workers/pythonWebWorker/pythonWebWorker.ts @@ -1,6 +1,6 @@ import { events } from '@/app/events/events'; import { LanguageState } from '@/app/web-workers/languageTypes'; -import { authClient } from '@/auth'; +import { authClient } from '@/auth/auth'; import mixpanel from 'mixpanel-browser'; import { quadraticCore } from '../quadraticCore/quadraticCore'; import { ClientPythonMessage, PythonClientGetJwt, PythonClientMessage } from './pythonClientMessages'; diff --git a/quadratic-client/src/app/web-workers/quadraticCore/quadraticCore.ts b/quadratic-client/src/app/web-workers/quadraticCore/quadraticCore.ts index b5a3376b73..0f6fe9b840 100644 --- a/quadratic-client/src/app/web-workers/quadraticCore/quadraticCore.ts +++ b/quadratic-client/src/app/web-workers/quadraticCore/quadraticCore.ts @@ -27,7 +27,7 @@ import { SheetRect, SummarizeSelectionResult, } from '@/app/quadratic-core-types'; -import { authClient } from '@/auth'; +import { authClient } from '@/auth/auth'; import { Rectangle } from 'pixi.js'; import { renderWebWorker } from '../renderWebWorker/renderWebWorker'; import { diff --git a/quadratic-client/src/auth/auth.ts b/quadratic-client/src/auth/auth.ts index b6d5374c88..bce9490670 100644 --- a/quadratic-client/src/auth/auth.ts +++ b/quadratic-client/src/auth/auth.ts @@ -1,46 +1,12 @@ -import { Auth0Client, User, createAuth0Client } from '@auth0/auth0-spa-js'; -import * as Sentry from '@sentry/react'; +import { auth0Client } from '@/auth/auth0'; +import { User } from '@auth0/auth0-spa-js'; import { useEffect } from 'react'; import { LoaderFunction, LoaderFunctionArgs, redirect } from 'react-router-dom'; import { ROUTES } from '../shared/constants/routes'; -const AUTH0_DOMAIN = import.meta.env.VITE_AUTH0_DOMAIN || ''; -const AUTH0_CLIENT_ID = import.meta.env.VITE_AUTH0_CLIENT_ID || ''; -const AUTH0_AUDIENCE = import.meta.env.VITE_AUTH0_AUDIENCE; -const AUTH0_ISSUER = import.meta.env.VITE_AUTH0_ISSUER; +const AUTH_TYPE = import.meta.env.VITE_AUTH_TYPE || ''; -// verify all AUTH0 env variables are set -if (!(AUTH0_DOMAIN && AUTH0_CLIENT_ID && AUTH0_AUDIENCE && AUTH0_ISSUER)) { - const message = 'Auth0 variables are not configured correctly.'; - Sentry.captureEvent({ - message, - level: 'fatal', - }); -} - -// Create the client as a module-scoped promise so all loaders will wait -// for this one single instance of client to resolve -let auth0ClientPromise: Promise; -async function getClient() { - if (!auth0ClientPromise) { - auth0ClientPromise = createAuth0Client({ - domain: AUTH0_DOMAIN, - clientId: AUTH0_CLIENT_ID, - issuer: AUTH0_ISSUER, - authorizationParams: { - audience: AUTH0_AUDIENCE, - }, - cacheLocation: 'localstorage', - useRefreshTokens: true, - // remove the subdomain from the cookie domain so that the ws server can access it - cookieDomain: parseDomain(window.location.host), - }); - } - const auth0Client = await auth0ClientPromise; - return auth0Client; -} - -interface AuthClient { +export interface AuthClient { isAuthenticated(): Promise; user(): Promise; login(redirectTo: string, isSignupFlow?: boolean): Promise; @@ -49,61 +15,17 @@ interface AuthClient { getTokenOrRedirect(): Promise; } -export const authClient: AuthClient = { - async isAuthenticated() { - const client = await getClient(); - const isAuthenticated = await client.isAuthenticated(); - return isAuthenticated; - }, - async user() { - const client = await getClient(); - const user = await client.getUser(); - return user; - }, - async login(redirectTo: string, isSignupFlow: boolean = false) { - const client = await getClient(); - await client.loginWithRedirect({ - authorizationParams: { - screen_hint: isSignupFlow ? 'signup' : 'login', - redirect_uri: - window.location.origin + - ROUTES.LOGIN_RESULT + - '?' + - new URLSearchParams([['redirectTo', redirectTo]]).toString(), - }, - }); - await waitForAuth0ClientToRedirect(); - }, - async handleSigninRedirect() { - const query = window.location.search; - if (query.includes('code=') && query.includes('state=')) { - const client = await getClient(); - await client.handleRedirectCallback(); - } - }, - async logout() { - const client = await getClient(); - await client.logout({ logoutParams: { returnTo: window.location.origin } }); - await waitForAuth0ClientToRedirect(); - }, - /** - * Tries to get a token for the current user from the auth0 client. - * If the token is still valid, it'll pull it from a cache. If it’s expired, - * it will fail and we will manually redirect the user to auth0 to re-authenticate - * and get a new token. - */ - async getTokenOrRedirect() { - const client = await getClient(); - try { - const token = await client.getTokenSilently(); - return token; - } catch (e) { - await this.login(new URL(window.location.href).pathname); - return ''; - } - }, +const getAuthClient = () => { + switch (AUTH_TYPE) { + case 'auth0': + return auth0Client; + default: + throw new Error(`Unsupported auth type in getAuthClient(): ${AUTH_TYPE}`); + } }; +export const authClient: AuthClient = getAuthClient(); + /** * Utility function for use in route loaders. * If the user is not logged in (or don't have an auth token) and tries to @@ -134,20 +56,20 @@ export function protectedRouteLoaderWrapper(loaderFn: LoaderFunction): LoaderFun } /** - * In cases where we call the auth0 client and it redirects the user to the - * auth0 website (e.g. for `.login` and `.logout`, presumably via changing - * `window.location`) we have to manually wait for the auth0 client. + * In cases where we call the auth client and it redirects the user to the + * auth website (e.g. for `.login` and `.logout`, presumably via changing + * `window.location`) we have to manually wait for the auth client. * - * Why? Because even though auth0's client APIs are async, they seem to + * Why? Because even though auth's client APIs are async, they seem to * complete immediately and our app's code continues before `window.location` * kicks in. * - * So this function ensures our whole app pauses while the auth0 lib does its - * thing and kicks the user over to auth0.com + * So this function ensures our whole app pauses while the auth lib does its + * thing and kicks the user over to auth.com * * We only use this when we _want_ to pause everything and wait to redirect */ -export function waitForAuth0ClientToRedirect() { +export function waitForAuthClientToRedirect() { return new Promise((resolve) => setTimeout(resolve, 10000)); } diff --git a/quadratic-client/src/auth/auth0.ts b/quadratic-client/src/auth/auth0.ts index e69de29bb2..9dc3490521 100644 --- a/quadratic-client/src/auth/auth0.ts +++ b/quadratic-client/src/auth/auth0.ts @@ -0,0 +1,97 @@ +import { Auth0Client, createAuth0Client } from '@auth0/auth0-spa-js'; +import * as Sentry from '@sentry/react'; +import { ROUTES } from '../shared/constants/routes'; +import { AuthClient, parseDomain, waitForAuthClientToRedirect } from './auth'; + +const AUTH0_DOMAIN = import.meta.env.VITE_AUTH0_DOMAIN || ''; +const AUTH0_CLIENT_ID = import.meta.env.VITE_AUTH0_CLIENT_ID || ''; +const AUTH0_AUDIENCE = import.meta.env.VITE_AUTH0_AUDIENCE; +const AUTH0_ISSUER = import.meta.env.VITE_AUTH0_ISSUER; + +// verify all AUTH0 env variables are set +if (!(AUTH0_DOMAIN && AUTH0_CLIENT_ID && AUTH0_AUDIENCE && AUTH0_ISSUER)) { + const message = 'Auth0 variables are not configured correctly.'; + Sentry.captureEvent({ + message, + level: 'fatal', + }); +} + +// Create the client as a module-scoped promise so all loaders will wait +// for this one single instance of client to resolve +let auth0ClientPromise: Promise; +async function getClient() { + if (!auth0ClientPromise) { + auth0ClientPromise = createAuth0Client({ + domain: AUTH0_DOMAIN, + clientId: AUTH0_CLIENT_ID, + issuer: AUTH0_ISSUER, + authorizationParams: { + audience: AUTH0_AUDIENCE, + }, + cacheLocation: 'localstorage', + useRefreshTokens: true, + // remove the subdomain from the cookie domain so that the ws server can access it + cookieDomain: parseDomain(window.location.host), + }); + } + const auth0Client = await auth0ClientPromise; + return auth0Client; +} + +type Auth0AuthClient = AuthClient; + +export const auth0Client: Auth0AuthClient = { + async isAuthenticated() { + const client = await getClient(); + const isAuthenticated = await client.isAuthenticated(); + return isAuthenticated; + }, + async user() { + const client = await getClient(); + const user = await client.getUser(); + return user; + }, + async login(redirectTo: string, isSignupFlow: boolean = false) { + const client = await getClient(); + await client.loginWithRedirect({ + authorizationParams: { + screen_hint: isSignupFlow ? 'signup' : 'login', + redirect_uri: + window.location.origin + + ROUTES.LOGIN_RESULT + + '?' + + new URLSearchParams([['redirectTo', redirectTo]]).toString(), + }, + }); + await waitForAuthClientToRedirect(); + }, + async handleSigninRedirect() { + const query = window.location.search; + if (query.includes('code=') && query.includes('state=')) { + const client = await getClient(); + await client.handleRedirectCallback(); + } + }, + async logout() { + const client = await getClient(); + await client.logout({ logoutParams: { returnTo: window.location.origin } }); + await waitForAuthClientToRedirect(); + }, + /** + * Tries to get a token for the current user from the auth0 client. + * If the token is still valid, it'll pull it from a cache. If it’s expired, + * it will fail and we will manually redirect the user to auth0 to re-authenticate + * and get a new token. + */ + async getTokenOrRedirect() { + const client = await getClient(); + try { + const token = await client.getTokenSilently(); + return token; + } catch (e) { + await this.login(new URL(window.location.href).pathname); + return ''; + } + }, +}; diff --git a/quadratic-client/src/dashboard/components/EducationDialog.tsx b/quadratic-client/src/dashboard/components/EducationDialog.tsx index bbceafa957..c35af2365c 100644 --- a/quadratic-client/src/dashboard/components/EducationDialog.tsx +++ b/quadratic-client/src/dashboard/components/EducationDialog.tsx @@ -1,4 +1,4 @@ -import { authClient } from '@/auth'; +import { authClient } from '@/auth/auth'; import { useDashboardRouteLoaderData } from '@/routes/_dashboard'; import { useRootRouteLoaderData } from '@/routes/_root'; import { SEARCH_PARAMS } from '@/shared/constants/routes'; diff --git a/quadratic-client/src/routes/_dashboard.tsx b/quadratic-client/src/routes/_dashboard.tsx index 780d393b59..94097ced91 100644 --- a/quadratic-client/src/routes/_dashboard.tsx +++ b/quadratic-client/src/routes/_dashboard.tsx @@ -1,4 +1,4 @@ -import { useCheckForAuthorizationTokenOnWindowFocus } from '@/auth'; +import { useCheckForAuthorizationTokenOnWindowFocus } from '@/auth/auth'; import { DashboardSidebar } from '@/dashboard/components/DashboardSidebar'; import { EducationDialog } from '@/dashboard/components/EducationDialog'; import { Empty } from '@/dashboard/components/Empty'; diff --git a/quadratic-client/src/routes/_root.tsx b/quadratic-client/src/routes/_root.tsx index 203d664bf8..0bf990b8c1 100644 --- a/quadratic-client/src/routes/_root.tsx +++ b/quadratic-client/src/routes/_root.tsx @@ -1,4 +1,4 @@ -import { authClient } from '@/auth'; +import { authClient } from '@/auth/auth'; import { Empty } from '@/dashboard/components/Empty'; import { GlobalSnackbarProvider } from '@/shared/components/GlobalSnackbarProvider'; import { Theme } from '@/shared/components/Theme'; diff --git a/quadratic-client/src/routes/file.$uuid.tsx b/quadratic-client/src/routes/file.$uuid.tsx index 8826dddac7..03e67c6685 100644 --- a/quadratic-client/src/routes/file.$uuid.tsx +++ b/quadratic-client/src/routes/file.$uuid.tsx @@ -8,7 +8,7 @@ import { VersionComparisonResult, compareVersions } from '@/app/schemas/compareV import { QuadraticApp } from '@/app/ui/QuadraticApp'; import { quadraticCore } from '@/app/web-workers/quadraticCore/quadraticCore'; import { initWorkers } from '@/app/web-workers/workers'; -import { authClient, useCheckForAuthorizationTokenOnWindowFocus } from '@/auth'; +import { authClient, useCheckForAuthorizationTokenOnWindowFocus } from '@/auth/auth'; import { apiClient } from '@/shared/api/apiClient'; import { ROUTES } from '@/shared/constants/routes'; import { CONTACT_URL } from '@/shared/constants/urls'; diff --git a/quadratic-client/src/routes/login-result.tsx b/quadratic-client/src/routes/login-result.tsx index b2c5ca1ba5..10678d1279 100644 --- a/quadratic-client/src/routes/login-result.tsx +++ b/quadratic-client/src/routes/login-result.tsx @@ -1,4 +1,4 @@ -import { authClient } from '@/auth'; +import { authClient } from '@/auth/auth'; import { apiClient } from '@/shared/api/apiClient'; import { redirect } from 'react-router-dom'; diff --git a/quadratic-client/src/routes/login.tsx b/quadratic-client/src/routes/login.tsx index f02db54da3..c2bbb771b3 100644 --- a/quadratic-client/src/routes/login.tsx +++ b/quadratic-client/src/routes/login.tsx @@ -1,4 +1,4 @@ -import { authClient } from '@/auth'; +import { authClient } from '@/auth/auth'; import { LoaderFunctionArgs, redirect } from 'react-router-dom'; export const loader = async ({ request }: LoaderFunctionArgs) => { diff --git a/quadratic-client/src/routes/logout.tsx b/quadratic-client/src/routes/logout.tsx index 1ff3102b61..3f0216b992 100644 --- a/quadratic-client/src/routes/logout.tsx +++ b/quadratic-client/src/routes/logout.tsx @@ -1,4 +1,4 @@ -import { authClient } from '@/auth'; +import { authClient } from '@/auth/auth'; import { redirect } from 'react-router-dom'; export const loader = async () => { diff --git a/quadratic-client/src/routes/teams.$teamUuid.files.create.tsx b/quadratic-client/src/routes/teams.$teamUuid.files.create.tsx index b2d0153b44..e37fde22c0 100644 --- a/quadratic-client/src/routes/teams.$teamUuid.files.create.tsx +++ b/quadratic-client/src/routes/teams.$teamUuid.files.create.tsx @@ -1,4 +1,4 @@ -import { authClient } from '@/auth'; +import { authClient } from '@/auth/auth'; import { apiClient } from '@/shared/api/apiClient'; import { snackbarMsgQueryParam, snackbarSeverityQueryParam } from '@/shared/components/GlobalSnackbarProvider'; import { ROUTES } from '@/shared/constants/routes'; diff --git a/quadratic-client/src/shared/api/connectionClient.ts b/quadratic-client/src/shared/api/connectionClient.ts index edd3b01d63..5fd0fbe97c 100644 --- a/quadratic-client/src/shared/api/connectionClient.ts +++ b/quadratic-client/src/shared/api/connectionClient.ts @@ -1,4 +1,4 @@ -import { authClient } from '@/auth'; +import { authClient } from '@/auth/auth'; import { ConnectionType, ConnectionTypeDetails } from 'quadratic-shared/typesAndSchemasConnections'; import z from 'zod'; const API_URL = import.meta.env.VITE_QUADRATIC_CONNECTION_URL; diff --git a/quadratic-client/src/shared/api/fetchFromApi.ts b/quadratic-client/src/shared/api/fetchFromApi.ts index 1d4420747a..7808b07f17 100644 --- a/quadratic-client/src/shared/api/fetchFromApi.ts +++ b/quadratic-client/src/shared/api/fetchFromApi.ts @@ -1,4 +1,4 @@ -import { authClient } from '@/auth'; +import { authClient } from '@/auth/auth'; import * as Sentry from '@sentry/react'; import z from 'zod'; import { apiClient } from './apiClient'; From b7aed3d0597a75a94de7fd67cba6fda6a854604e Mon Sep 17 00:00:00 2001 From: David Kircos Date: Mon, 12 Aug 2024 13:03:01 -0600 Subject: [PATCH 016/113] add env var to example files --- quadratic-files/.env.docker | 3 ++- quadratic-files/.env.example | 3 ++- quadratic-files/.env.test | 3 ++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/quadratic-files/.env.docker b/quadratic-files/.env.docker index 132746ccf1..1fdb5b0716 100644 --- a/quadratic-files/.env.docker +++ b/quadratic-files/.env.docker @@ -26,4 +26,5 @@ AWS_S3_ACCESS_KEY_ID= AWS_S3_SECRET_ACCESS_KEY= # Storage: file-system -STORAGE_DIR=./../docker/file-storage \ No newline at end of file +STORAGE_DIR=./../docker/file-storage +STORAGE_ENCRYPTION_KEYS=eb4758047f74bdb2603cce75c4370327ca2c3662c4786867659126da8e64dfcc \ No newline at end of file diff --git a/quadratic-files/.env.example b/quadratic-files/.env.example index 21d7b8c77c..8d18ad5335 100644 --- a/quadratic-files/.env.example +++ b/quadratic-files/.env.example @@ -26,4 +26,5 @@ AWS_S3_ACCESS_KEY_ID=test AWS_S3_SECRET_ACCESS_KEY=test # Storage: file-system -STORAGE_DIR=./../docker/file-storage \ No newline at end of file +STORAGE_DIR=./../docker/file-storage +STORAGE_ENCRYPTION_KEYS=eb4758047f74bdb2603cce75c4370327ca2c3662c4786867659126da8e64dfcc \ No newline at end of file diff --git a/quadratic-files/.env.test b/quadratic-files/.env.test index f6f8b48338..fd3e72165d 100644 --- a/quadratic-files/.env.test +++ b/quadratic-files/.env.test @@ -26,4 +26,5 @@ AWS_S3_ACCESS_KEY_ID= AWS_S3_SECRET_ACCESS_KEY= # Storage: file-system -STORAGE_DIR=./../docker/file-storage \ No newline at end of file +STORAGE_DIR=./../docker/file-storage +STORAGE_ENCRYPTION_KEYS=eb4758047f74bdb2603cce75c4370327ca2c3662c4786867659126da8e64dfcc \ No newline at end of file From 19f7d63eb4c6d01c48b625694489c77f3083811e Mon Sep 17 00:00:00 2001 From: David DiMaria Date: Tue, 13 Aug 2024 09:34:53 -0600 Subject: [PATCH 017/113] Full login/logout flow with server integration --- docker-compose.yml | 2 +- docker/ory-auth/config/kratos.yml | 51 +++++-- package-lock.json | 11 ++ package.json | 1 + quadratic-api/.env.test | 3 +- quadratic-api/package.json | 5 +- .../public/quadratic-local-jwks.json | 19 +++ quadratic-api/src/app.ts | 3 + quadratic-api/src/auth/auth.ts | 35 ++++- quadratic-api/src/auth/auth0.ts | 23 +++- quadratic-api/src/auth/ory.ts | 72 ++++++++++ quadratic-api/src/env-vars.ts | 27 ++-- .../src/middleware/validateAccessToken.ts | 17 +-- .../src/routes/v0/files.$uuid.invites.POST.ts | 4 +- .../src/routes/v0/teams.$uuid.invites.POST.ts | 4 +- quadratic-client/package.json | 1 + quadratic-client/public/.well-known/jwks.json | 26 ++++ .../multiplayerWebWorker/multiplayer.ts | 3 +- .../multiplayerClientMessages.ts | 2 +- .../worker/multiplayerServer.ts | 2 +- quadratic-client/src/auth/auth.ts | 14 +- quadratic-client/src/auth/ory.ts | 128 ++++++++++++++++++ quadratic-client/src/routes/_root.tsx | 3 +- .../src/shared/utils/analytics.ts | 4 +- quadratic-client/src/shared/utils/userUtil.ts | 4 +- 25 files changed, 398 insertions(+), 66 deletions(-) create mode 100644 quadratic-api/public/quadratic-local-jwks.json create mode 100644 quadratic-api/src/auth/ory.ts create mode 100644 quadratic-client/public/.well-known/jwks.json diff --git a/docker-compose.yml b/docker-compose.yml index 71d16f21ba..1264e7a6f3 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -204,7 +204,7 @@ services: PORT: 4455 SECURITY_MODE: KRATOS_PUBLIC_URL: http://host.docker.internal:4433/ - KRATOS_BROWSER_URL: http://127.0.0.1:4433/ + KRATOS_BROWSER_URL: http://localhost:4433/ COOKIE_SECRET: changeme CSRF_COOKIE_NAME: ory_csrf_ui CSRF_COOKIE_SECRET: changeme diff --git a/docker/ory-auth/config/kratos.yml b/docker/ory-auth/config/kratos.yml index 50c79ea997..5c7106a657 100644 --- a/docker/ory-auth/config/kratos.yml +++ b/docker/ory-auth/config/kratos.yml @@ -1,19 +1,37 @@ -version: v0.13.0 +# https://raw.githubusercontent.com/ory/kratos/v1.2.0/.schemastore/config.schema.json +version: v1.2.0 dsn: memory serve: public: - base_url: http://127.0.0.1:4433/ + base_url: http://localhost:4433/ cors: enabled: true + allowed_origins: + - http://localhost:3000 + allowed_methods: + - POST + - GET + - PUT + - PATCH + - DELETE + allowed_headers: + - Authorization + - Access-Control-Allow-Origin + - Cookie + - Content-Type + exposed_headers: + - Content-Type + - Set-Cookie admin: base_url: http://kratos:4434/ selfservice: - default_browser_return_url: http://127.0.0.1:4455/welcome + default_browser_return_url: http://localhost:3000 allowed_return_urls: - - http://127.0.0.1:4455 + - http://localhost:4455 + - http://localhost:3000 - http://localhost:19006/Callback - exp://localhost:8081/--/Callback @@ -33,42 +51,51 @@ selfservice: flows: error: - ui_url: http://127.0.0.1:4455/error + ui_url: http://localhost:4455/error settings: - ui_url: http://127.0.0.1:4455/settings + ui_url: http://localhost:4455/settings privileged_session_max_age: 15m required_aal: highest_available recovery: enabled: true - ui_url: http://127.0.0.1:4455/recovery + ui_url: http://localhost:4455/recovery use: code verification: enabled: true - ui_url: http://127.0.0.1:4455/verification + ui_url: http://localhost:4455/verification use: code after: - default_browser_return_url: http://127.0.0.1:4455/welcome + default_browser_return_url: http://localhost:3000 logout: after: - default_browser_return_url: http://127.0.0.1:4455/login + default_browser_return_url: http://localhost:4455/login login: - ui_url: http://127.0.0.1:4455/login + ui_url: http://localhost:4455/login lifespan: 10m registration: lifespan: 10m - ui_url: http://127.0.0.1:4455/registration + ui_url: http://localhost:4455/registration after: password: hooks: - hook: session - hook: show_verification_ui +session: + whoami: + tokenizer: + templates: + jwt_template: + jwks_url: http://host.docker.internal:3000/.well-known/jwks.json + # claims_mapper_url: base64://... # A JsonNet template for modifying the claims + ttl: 1m # 1 minute (defaults to 10 minutes) + log: level: debug format: text diff --git a/package-lock.json b/package-lock.json index f319072546..3745cc7e3d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -18,6 +18,7 @@ "quadratic-kernels/python-wasm" ], "dependencies": { + "@ory/kratos-client": "^1.2.1", "tsc": "^2.0.4", "vitest": "^1.5.0", "zod": "^3.22.4" @@ -6253,6 +6254,14 @@ "dev": true, "license": "MIT" }, + "node_modules/@ory/kratos-client": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/@ory/kratos-client/-/kratos-client-1.2.1.tgz", + "integrity": "sha512-HvipnVQotCKjEQC9I9DPjSlfBEww4pjDycMAKdUPj3g/0WkNSq6wbPDyqeclFz99rsOOsFMcpOO8qiCYHSgQeA==", + "dependencies": { + "axios": "^1.6.1" + } + }, "node_modules/@pixi/accessibility": { "version": "6.5.10", "license": "MIT", @@ -26660,6 +26669,7 @@ "@aws-sdk/client-s3": "^3.427.0", "@aws-sdk/client-secrets-manager": "^3.441.0", "@aws-sdk/s3-request-presigner": "^3.427.0", + "@ory/kratos-client": "^1.2.1", "@prisma/client": "^4.12.0", "@sendgrid/mail": "^8.1.0", "@sentry/node": "^7.50.0", @@ -26743,6 +26753,7 @@ "@monaco-editor/react": "^4.3.1", "@mui/icons-material": "^5.2.0", "@mui/material": "^5.2.2", + "@ory/kratos-client": "^1.2.1", "@radix-ui/react-alert-dialog": "^1.0.5", "@radix-ui/react-avatar": "^1.0.4", "@radix-ui/react-checkbox": "^1.0.4", diff --git a/package.json b/package.json index 8f6872eefd..4c0e467016 100644 --- a/package.json +++ b/package.json @@ -62,6 +62,7 @@ "gen:pyright:worker": "npm run gen:pyright:worker --workspace=quadratic-kernels/python-wasm" }, "dependencies": { + "@ory/kratos-client": "^1.2.1", "tsc": "^2.0.4", "vitest": "^1.5.0", "zod": "^3.22.4" diff --git a/quadratic-api/.env.test b/quadratic-api/.env.test index a0e6f64c1c..a85b211524 100644 --- a/quadratic-api/.env.test +++ b/quadratic-api/.env.test @@ -7,12 +7,13 @@ STRIPE_WEBHOOK_SECRET=STRIPE_WEBHOOK_SECRET ENCRYPTION_KEY=eb4758047f74bdb2603cce75c4370327ca2c3662c4786867659126da8e64dfcc # Auth -AUTH_TYPE=auth0 # auth0 or ? +AUTH_TYPE=auth0 # auth0 or ory AUTH0_JWKS_URI='https://dev.us.auth0.com/.well-known/jwks.json' AUTH0_ISSUER='https://auth-dev.quadratic.to/' AUTH0_CLIENT_ID="AUTH0_CLIENT_ID" AUTH0_CLIENT_SECRET="AUTH0_CLIENT_SECRET" AUTH0_DOMAIN="AUTH0_DOMAIN" +ORY_JWKS_URI='http://localhost:3000/.well-known/jwks.json' # Storage STORAGE_TYPE=s3 # s3 or file-system diff --git a/quadratic-api/package.json b/quadratic-api/package.json index 14493c8291..a70119d6a3 100644 --- a/quadratic-api/package.json +++ b/quadratic-api/package.json @@ -29,9 +29,10 @@ }, "author": "David Kircos", "dependencies": { - "@aws-sdk/client-secrets-manager": "^3.441.0", "@aws-sdk/client-s3": "^3.427.0", + "@aws-sdk/client-secrets-manager": "^3.441.0", "@aws-sdk/s3-request-presigner": "^3.427.0", + "@ory/kratos-client": "^1.2.1", "@prisma/client": "^4.12.0", "@sendgrid/mail": "^8.1.0", "@sentry/node": "^7.50.0", @@ -67,9 +68,9 @@ "@types/auth0": "^3.3.4", "@types/aws-sdk": "^2.7.0", "@types/cors": "^2.8.12", - "@types/pg": "^8.10.7", "@types/multer": "^1.4.8", "@types/multer-s3": "^3.0.1", + "@types/pg": "^8.10.7", "@typescript-eslint/eslint-plugin": "^6.13.1", "@typescript-eslint/parser": "^6.13.1", "dotenv-cli": "^7.1.0", diff --git a/quadratic-api/public/quadratic-local-jwks.json b/quadratic-api/public/quadratic-local-jwks.json new file mode 100644 index 0000000000..319a49cdec --- /dev/null +++ b/quadratic-api/public/quadratic-local-jwks.json @@ -0,0 +1,19 @@ +{ + "set": "quadratic-local", + "keys": [ + { + "alg": "RS256", + "d": "ZzQr3XjaRVCXFum_DwL-LiKFvAN9I15XJb-bzIGQDpLVolMmuJtOsvLWRa24dHRwGf6SPmZQP7lKIVRjPGHReoICdKNMAW0lYM0nvDMrMRC9TQsVg-0HxMHvkw29Ka4xPD_YtcPuzP4MKfeaE8D7pBiXFd4uoBBAVYKAvsBxLz7E4l3JIuy-S1kGa4if2q6IgHrgxnR95NhNynUNp7hJElNxWxKZoOboHaS31ZEe9Pqpp8MYDt6E6BrjYvvtsJjNLNEKMWw70mSgcSAAlgN3u0Bf_PazYYn-zGxYDpODAKSclXhPvyXxnJ2NC-wr6aLpSdxFqwz4JRAUTWCBFIWS4xDJPzgv2dc_JrQuYYfDvGwr5FAl7vGSFTU6w3e5swCZhKLlIjx9nLKaZbYejy01XrmCNm7VbDHX9iVjrxAww4RXLsqx2A3GR63GrE5r7yV8cDvyLe5qt9MfJm6M2y5rtBRu3XGtAKf6K2lP-qYixWW95QJ_3OIiYutFwCLxbI-LElaYgCRt6yCcfUeiY_wy2X-ts2RgYlAS8F6aQHQuX0Om6FqYiGkkh1snorUg5tkByoWxQeICGBRjpRl9sR3fzqI8dznB71d0kmKSyTYl3qhTKb0Btv56VqxJQsgMFUmfceQxkH0nnuXyEWIM7tF_4LsQtNYCFLHQore2-Qrxb2E", + "dp": "eibm6Jw6nIBChxwJlyjerBKwvuBP4LFP3rF-5kegPB8qrBnBxhOVZrMOl5bYYh73fIHe-rgE_gTzH-RzdzJsrsR2nzQcHQFhu5GHoMuYhSZJbQ32hERNUwZFRfIwtfhTU6-Un--eijMzKkCM-5VEipjTzj6gtHDAJsX1JL6ttGMqZpgLyQWp_Hvr_rGm3oQcSXZfrXmOtjtMaMPq56KF6fKRY8MviDFVN3GlRDr84s7cCdCwxnr2_dp9y9NajaRh76TR60gEEe5ZFs6fK_iZQUobhv6gVCvQB1jS6yC1rREj60XXSTK1BCafBlF6fQw0NsfL97Dj6sHuQlpv60UhHQ", + "dq": "j-aRMph1lctEupphXHosEGsjKOKAJw1QsJC5GQCvF39B-Bb91chekitSt_EOPt2i5gY4b4dI-oKCMh_BFr_BCV-RsGlDC3y3x_Ka1wF5GCN9JpcyGG2wOZOXYlglYG82ItyqRVgJZ7US8mqk_l1e8whReAs8rhNoQhLtigJgdC4cGeklZh45-be97ToTCfI3_R2WDcbEE7YQ8VkGsea3ZcOPAWBkzpwZIyOKi4Cyo5mwmQ_9pKkiFMWD1PlZGTCxltJW-DsQ-xZT9UKxkjxR2GyCCQfQ04_NjkIG6wR3H7aPqbo_xP-Cav30JaB-3_nxW3-yHvR7Me-6Zi1lgzmQOQ", + "e": "AQAB", + "kid": "438c4767-7651-45e6-8dfd-c19335a3ee1a", + "kty": "RSA", + "n": "6bKB0-vmuMAET-Xe7OUITo1hujQyg5z0hQWfIKavdMP6UFuBOjPBpJ_KuHoee3zq_MlJBHsLJTeC_IB5SxIB-_ZHWgrwPn5DvJYFSKU5cXv6iB1RdMxAnyjHl_SuEeWqt75gZHPgzVL2E2MDsQ5ypPLiUb3XeR4GPh5ynEnyKcjgm_vJBie-Z3vU8jXqVzM-CDPzWGkw7Ru1gwYRoGrbojWQbiQYTWspIqnFiQWTwJyDkYLtGy2EDD1z1ixoxKKFZq2JGxsPU1gbOTudVwUy4meW0uUqnmDFM0b_FpZHC_AN0SnQRB-jLr-slZoi5tnwJ_WF5NC0vqKotcKHhMqEKVv4egYNCbk_QHeta2MHavnAhuVw6l0c5G_N64T5pVcIyBvkxepgRTvXOqERHoPGnVwndCKZE3mgTaixIXadcp1afMENJFP-ZPUDZ0Y0yel8naNz7twYnUrzSKMXBIB4K6WEwNAUEVvC7bDZ-XGIXCJ-Ah29C5evnTPTpqrlqpuOl-Bfr5WJrvohv3jkTTH9sTRqnlerfaIUD8FNvtovz-yW3WtqFCQpoPKlePL1HRvWCzRU4blRFEb66MCdSgsXM6K7SIeqXyQ6T1scOFgnEAV1zQOJYB7qKIuc-wMNg0lJcULXhEIr2WVSKlxWWWd9i4rZSylHCP1Nvl2Ct6AyUac", + "p": "66m8M7DdkNHL76aoYTQePeV6rw472AmmgHR1j5S8Tyd-tLAWL1Yxi1CpIRBiGL5Xxq0GKz_bwHh3C8SkIfyfeNeC_UMr2mvgcjYDLsNQgX2diz8p_h9FhcrMMuEjImMneVZhc-QyDZuws5EvGfTB05PkBObWMq72LzjGq8K0fyes_vp8iK9XcxcMg7Zt_2OxbQGufSNYdOoev2Nl7eyZSXrwNPfADzL_5n1I6wGmue2DTGA9bX88IP0typ-KkMZYkGGt25H391YwqVki6VhJYqRpsavcwJe3rXizFisVCfvgrukBEBjTTXPxkWeRDt3H3R-JWTAuwEYfiHQDHiMXPw", + "q": "_d1YUX4aTGbAPDHyaqF2u2SWZJvUiozXBHRcqCEUCKvnRR4-rHIs4QYCIZYD3063WBVvSKNL7q-T-nkSiKhdEOwHoqGYGaItuHjq-OzH_ig9UXL4V2DmthPBt5kgcNOFfUQrGPD2NLT6-QtDU6eGxfpr7pWAgjdTGNZI--KpHPQkab8mO-jDfP84BmUfcc1yKVEHC5b2vBMiZ0_qy529qz7G5jrTA7TwjTfu9VyY_Gl2u62Zq69dWp96xP_CYgS_Gj964V53MNG0mBCkGVFBGyIJ5H1LUrSK4pCgS7dFhraZDgllZfo4nwz74hG2uPp2OJVwcpqxqD2N4bD0x1BTmQ", + "qi": "tstI7N-hHEjfrujyWgwaVtuChFx85JRiCtkPvLInLnaHXVKxFMWm7boIvLYJ8XHqd1RgWADRP-lofJfL3qtt3-tZimgrfduKcz1_vLYm_P_YYwtR6Fv7TNDjGiF3Pf-J1RSB8uYnKs2Hqc3s7XqIASeFEzxeV07Yiohmx4dIYwrqZePxr0siUaswGP4nsKJD1QcfeXvNt_mXnLcHX0LY740fCVzGDxu7Yiwuxtu1VU09gemexfC1XkPbYvi0IY8UDtl9qu5DM2cdcmuCnZP3QiJM33QBuvRM4FJGNUV_3nOrzvt0xdU3TkNmvkHlKDfWB0fY5KMpeQrB5DO1AhEPtQ", + "use": "sig" + } + ] +} diff --git a/quadratic-api/src/app.ts b/quadratic-api/src/app.ts index 9f988ea270..a856796ec9 100644 --- a/quadratic-api/src/app.ts +++ b/quadratic-api/src/app.ts @@ -61,6 +61,9 @@ app.use((req, res, next) => { return next(); }); +// static assets in the /public directory +app.use(express.static('public')); + // Health-check app.get('/', (req, res) => { res.status(200).json({ message: 'OK' }); diff --git a/quadratic-api/src/auth/auth.ts b/quadratic-api/src/auth/auth.ts index 83008b7ced..d369c29c17 100644 --- a/quadratic-api/src/auth/auth.ts +++ b/quadratic-api/src/auth/auth.ts @@ -1,15 +1,46 @@ import { AUTH_TYPE } from '../env-vars'; -import { getUsersFromAuth0 } from './auth0'; +import { getUsersFromAuth0, jwtConfigAuth0, lookupUsersFromAuth0ByEmail } from './auth0'; +import { getUsersFromOry, jwtConfigOry } from './ory'; export type UsersRequest = { id: number; auth0Id: string; }; -export const getUsers = async (users: UsersRequest[]) => { +export type User = { + id: number; + auth0Id: string; + email: string; + name?: string | undefined; + picture?: string | undefined; +}; + +export const getUsers = async (users: UsersRequest[]): Promise> => { switch (AUTH_TYPE) { case 'auth0': return await getUsersFromAuth0(users); + case 'ory': + return await getUsersFromOry(users); + default: + throw new Error(`Unsupported auth type in getUsers(): ${AUTH_TYPE}`); + } +}; + +export const getUsersByEmail = async (email: string) => { + switch (AUTH_TYPE) { + case 'auth0': + return await lookupUsersFromAuth0ByEmail(email); + default: + throw new Error(`Unsupported auth type in getUsers(): ${AUTH_TYPE}`); + } +}; + +export const jwtConfig = () => { + switch (AUTH_TYPE) { + case 'auth0': + return jwtConfigAuth0; + case 'ory': + return jwtConfigOry; default: throw new Error(`Unsupported auth type in getUsers(): ${AUTH_TYPE}`); } diff --git a/quadratic-api/src/auth/auth0.ts b/quadratic-api/src/auth/auth0.ts index 932b74e82d..0ca31e3056 100644 --- a/quadratic-api/src/auth/auth0.ts +++ b/quadratic-api/src/auth/auth0.ts @@ -1,6 +1,15 @@ import * as Sentry from '@sentry/node'; import { ManagementClient } from 'auth0'; -import { AUTH0_CLIENT_ID, AUTH0_CLIENT_SECRET, AUTH0_DOMAIN } from '../env-vars'; +import { Algorithm } from 'jsonwebtoken'; +import jwksRsa, { GetVerificationKey } from 'jwks-rsa'; +import { + AUTH0_AUDIENCE, + AUTH0_CLIENT_ID, + AUTH0_CLIENT_SECRET, + AUTH0_DOMAIN, + AUTH0_ISSUER, + AUTH0_JWKS_URI, +} from '../env-vars'; // Guide to Setting up on Auth0 // 1. Create an Auth0 Machine to Machine Application @@ -96,3 +105,15 @@ export const lookupUsersFromAuth0ByEmail = async (email: string) => { const auth0Users = await auth0.getUsersByEmail(email); return auth0Users; }; + +export const jwtConfigAuth0 = { + secret: jwksRsa.expressJwtSecret({ + cache: true, + rateLimit: true, + jwksRequestsPerMinute: 5, + jwksUri: AUTH0_JWKS_URI, + }) as GetVerificationKey, + audience: AUTH0_AUDIENCE, + issuer: AUTH0_ISSUER, + algorithms: ['RS256'] as Algorithm[], +}; diff --git a/quadratic-api/src/auth/ory.ts b/quadratic-api/src/auth/ory.ts new file mode 100644 index 0000000000..2e6137d739 --- /dev/null +++ b/quadratic-api/src/auth/ory.ts @@ -0,0 +1,72 @@ +import { Configuration, IdentityApi } from '@ory/kratos-client'; +import * as Sentry from '@sentry/node'; +import { Algorithm } from 'jsonwebtoken'; +import jwksRsa, { GetVerificationKey } from 'jwks-rsa'; +import { ORY_ADMIN_HOST, ORY_JWKS_URI } from '../env-vars'; +import { User } from './auth'; + +const config = new Configuration({ + basePath: ORY_ADMIN_HOST, + baseOptions: { + withCredentials: true, + }, +}); +const sdk = new IdentityApi(config); + +export const jwtConfigOry = { + secret: jwksRsa.expressJwtSecret({ + cache: true, + rateLimit: true, + jwksRequestsPerMinute: 5, + jwksUri: ORY_JWKS_URI, + }) as GetVerificationKey, + algorithms: ['RS256'] as Algorithm[], +}; + +export const getUsersFromOry = async (users: { id: number; auth0Id: string }[]) => { + // If we got nothing, we return an empty object + if (users.length === 0) return {}; + + const ids = users.map(({ auth0Id }) => auth0Id); + let identities; + + try { + identities = (await sdk.listIdentities({ ids })).data; + } catch (e) { + console.error(e); + return {}; + } + + // Map users by their Quadratic ID. If we didn't find a user, throw. + const usersById: Record = users.reduce((acc: Record, { id, auth0Id }) => { + const oryUser = identities.find(({ id }) => id === auth0Id); + + // If we're missing data we expect, log it to Sentry and skip this user + if (!oryUser || oryUser.traits.email === undefined) { + Sentry.captureException({ + message: 'Auth0 user returned without `email`', + level: 'error', + extra: { + auth0IdInOurDb: auth0Id, + oryUserResult: oryUser, + }, + }); + throw new Error('Failed to retrieve all user info from Ory'); + } + + const { email, name } = oryUser.traits; + + return { + ...acc, + [id]: { + id, + auth0Id, + email, + name: `${name.first} ${name.last}`, + picture: undefined, + }, + }; + }, {}); + + return usersById; +}; diff --git a/quadratic-api/src/env-vars.ts b/quadratic-api/src/env-vars.ts index 0532c9de7b..f9f64901a4 100644 --- a/quadratic-api/src/env-vars.ts +++ b/quadratic-api/src/env-vars.ts @@ -9,6 +9,14 @@ export const SENTRY_DSN = process.env.SENTRY_DSN; export const NODE_ENV = process.env.NODE_ENV || 'development'; export const PORT = process.env.PORT || 8000; export const ENVIRONMENT = process.env.ENVIRONMENT; +export const AUTH0_DOMAIN = process.env.AUTH0_DOMAIN as string; +export const AUTH0_CLIENT_ID = process.env.AUTH0_CLIENT_ID as string; +export const AUTH0_CLIENT_SECRET = process.env.AUTH0_CLIENT_SECRET as string; +export const AUTH0_JWKS_URI = process.env.AUTH0_JWKS_URI as string; +export const AUTH0_ISSUER = process.env.AUTH0_ISSUER as string; +export const AUTH0_AUDIENCE = process.env.AUTH0_AUDIENCE as string; +export const ORY_JWKS_URI = process.env.ORY_JWKS_URI as string; +export const ORY_ADMIN_HOST = process.env.ORY_ADMIN_HOST as string; export const QUADRATIC_FILE_URI = process.env.QUADRATIC_FILE_URI as string; export const AWS_S3_REGION = process.env.AWS_S3_REGION as string; export const AWS_S3_ACCESS_KEY_ID = process.env.AWS_S3_ACCESS_KEY_ID as string; @@ -16,28 +24,11 @@ export const AWS_S3_SECRET_ACCESS_KEY = process.env.AWS_S3_SECRET_ACCESS_KEY as export const AWS_S3_BUCKET_NAME = process.env.AWS_S3_BUCKET_NAME as string; // Required -export const AUTH0_DOMAIN = process.env.AUTH0_DOMAIN as string; -export const AUTH0_CLIENT_ID = process.env.AUTH0_CLIENT_ID as string; -export const AUTH0_CLIENT_SECRET = process.env.AUTH0_CLIENT_SECRET as string; -export const AUTH0_JWKS_URI = process.env.AUTH0_JWKS_URI as string; -export const AUTH0_ISSUER = process.env.AUTH0_ISSUER as string; -export const AUTH0_AUDIENCE = process.env.AUTH0_AUDIENCE as string; export const STRIPE_SECRET_KEY = process.env.STRIPE_SECRET_KEY as string; export const ENCRYPTION_KEY = process.env.ENCRYPTION_KEY as string; export const STORAGE_TYPE = process.env.STORAGE_TYPE as string; export const AUTH_TYPE = process.env.AUTH_TYPE as string; -[ - 'AUTH0_DOMAIN', - 'AUTH0_CLIENT_ID', - 'AUTH0_CLIENT_SECRET', - 'AUTH0_JWKS_URI', - 'AUTH0_ISSUER', - 'AUTH0_AUDIENCE', - 'STRIPE_SECRET_KEY', - 'ENCRYPTION_KEY', - 'STORAGE_TYPE', - 'AUTH_TYPE', -].forEach(ensureEnvVarExists); +['STRIPE_SECRET_KEY', 'ENCRYPTION_KEY', 'STORAGE_TYPE', 'AUTH_TYPE'].forEach(ensureEnvVarExists); // Required in prod, optional locally export const M2M_AUTH_TOKEN = process.env.M2M_AUTH_TOKEN; diff --git a/quadratic-api/src/middleware/validateAccessToken.ts b/quadratic-api/src/middleware/validateAccessToken.ts index b256bff7d9..4f1b023dcc 100644 --- a/quadratic-api/src/middleware/validateAccessToken.ts +++ b/quadratic-api/src/middleware/validateAccessToken.ts @@ -1,16 +1,5 @@ -import { GetVerificationKey, expressjwt } from 'express-jwt'; -import jwksRsa from 'jwks-rsa'; -import { AUTH0_AUDIENCE, AUTH0_ISSUER, AUTH0_JWKS_URI } from '../env-vars'; +import { expressjwt } from 'express-jwt'; +import { jwtConfig } from '../auth/auth'; // based on implementation from https://github.com/auth0-developer-hub/api_express_typescript_hello-world/blob/main/src/middleware/auth0.middleware.ts -export const validateAccessToken = expressjwt({ - secret: jwksRsa.expressJwtSecret({ - cache: true, - rateLimit: true, - jwksRequestsPerMinute: 5, - jwksUri: AUTH0_JWKS_URI, - }) as GetVerificationKey, - audience: AUTH0_AUDIENCE, - issuer: AUTH0_ISSUER, - algorithms: ['RS256'], -}); +export const validateAccessToken = expressjwt(jwtConfig() as any); diff --git a/quadratic-api/src/routes/v0/files.$uuid.invites.POST.ts b/quadratic-api/src/routes/v0/files.$uuid.invites.POST.ts index 7da048b755..022d4af70f 100644 --- a/quadratic-api/src/routes/v0/files.$uuid.invites.POST.ts +++ b/quadratic-api/src/routes/v0/files.$uuid.invites.POST.ts @@ -2,7 +2,7 @@ import * as Sentry from '@sentry/node'; import { Response } from 'express'; import { ApiSchemas, ApiTypes, FilePermissionSchema } from 'quadratic-shared/typesAndSchemas'; import { z } from 'zod'; -import { getUsers, lookupUsersFromAuth0ByEmail } from '../../auth/auth'; +import { getUsers, getUsersByEmail } from '../../auth/auth'; import dbClient from '../../dbClient'; import { sendEmail } from '../../email/sendEmail'; import { templates } from '../../email/templates'; @@ -92,7 +92,7 @@ async function handler(req: RequestWithUser, res: Response; user(): Promise; @@ -19,6 +29,8 @@ const getAuthClient = () => { switch (AUTH_TYPE) { case 'auth0': return auth0Client; + case 'ory': + return oryClient; default: throw new Error(`Unsupported auth type in getAuthClient(): ${AUTH_TYPE}`); } diff --git a/quadratic-client/src/auth/ory.ts b/quadratic-client/src/auth/ory.ts index e69de29bb2..aa9c53cf1a 100644 --- a/quadratic-client/src/auth/ory.ts +++ b/quadratic-client/src/auth/ory.ts @@ -0,0 +1,128 @@ +import { Configuration, FrontendApi, Session } from '@ory/kratos-client'; +import * as Sentry from '@sentry/react'; +import { AuthClient, User, waitForAuthClientToRedirect } from './auth'; + +const ORY_HOST = import.meta.env.VITE_ORY_HOST; + +// verify all Ory env variables are set +if (!ORY_HOST) { + const message = 'Ory variables are not configured correctly.'; + Sentry.captureEvent({ + message, + level: 'fatal', + }); +} + +const config = new Configuration({ + basePath: ORY_HOST, + baseOptions: { + withCredentials: true, + }, +}); + +const sdk = new FrontendApi(config); + +// singleton session +let session: Session | undefined; + +/** + * Get the current session from Ory. + * If the session is not cached or expired, fetch a new one. + * Return false if the session cannot be fetched. + */ +const getSession = async (): Promise => { + // if the session exists and is not expired, return it + if (session && session.expires_at && Date.parse(session.expires_at) > Date.now()) { + return session; + } + + try { + session = (await sdk.toSession({ tokenizeAs: 'jwt_template' })).data; + return session; + } catch (e) { + return false; + } +}; + +type OryAuthClient = AuthClient; + +export const oryClient: OryAuthClient = { + /** + * Retuen whether the user is authenticated and the session is valid. + */ + async isAuthenticated(): Promise { + return (await getSession()) !== false; + }, + + /** + * Get the current authenticated user from Ory. + */ + async user(): Promise { + const session = await getSession(); + + if (!session) return; + + const { first, last } = session.identity?.traits.name; + const data = { + name: `${first} ${last}`, + given_name: first, + family_name: last, + email: session.identity?.traits.email, + sub: session.identity?.id, + }; + + return data; + }, + + /** + * Login the user in Ory and create a new session. + * If `isSignupFlow` is true, the user will be redirected to the registration flow. + */ + async login(redirectTo: string, isSignupFlow: boolean = false) { + const sdkUrl = isSignupFlow ? await sdk.createBrowserRegistrationFlow() : await sdk.createBrowserLoginFlow(); + const url = new URL(sdkUrl.data.ui.action); + url.searchParams.set('return_to', redirectTo); + + // redirect to the login/signup flow + window.location.assign(url); + + await waitForAuthClientToRedirect(); + }, + + /** + * Currently this is a noop since state and code cannot both be present in the URL + */ + async handleSigninRedirect() {}, + + /** + * Logout the user in Ory and terminate the singleton session. + * Take the user back to the login page (as defined in the Ory config). + */ + async logout() { + const { data: flow } = await sdk.createBrowserLogoutFlow(); + + // clear the singleton session + session = undefined; + + window.location.assign(flow.logout_url); + + await waitForAuthClientToRedirect(); + }, + + /** + * Tries to get a token for the current user from the Ory client. + * If the token is still valid, it'll pull it from a cache. If it’s expired, + * it will fail and we will manually redirect the user to auth0 to re-authenticate + * and get a new token. + */ + async getTokenOrRedirect() { + const session = await getSession(); + + if (!session || !session.tokenized) { + await this.login('/'); + return ''; + } + + return session.tokenized; + }, +}; diff --git a/quadratic-client/src/routes/_root.tsx b/quadratic-client/src/routes/_root.tsx index 0bf990b8c1..12614776fb 100644 --- a/quadratic-client/src/routes/_root.tsx +++ b/quadratic-client/src/routes/_root.tsx @@ -1,10 +1,9 @@ -import { authClient } from '@/auth/auth'; +import { User, authClient } from '@/auth/auth'; import { Empty } from '@/dashboard/components/Empty'; import { GlobalSnackbarProvider } from '@/shared/components/GlobalSnackbarProvider'; import { Theme } from '@/shared/components/Theme'; import { ROUTE_LOADER_IDS } from '@/shared/constants/routes'; import { initializeAnalytics } from '@/shared/utils/analytics'; -import { User } from '@auth0/auth0-spa-js'; import { ExclamationTriangleIcon } from '@radix-ui/react-icons'; import * as Sentry from '@sentry/react'; import { LoaderFunctionArgs, Outlet, useRouteError, useRouteLoaderData } from 'react-router-dom'; diff --git a/quadratic-client/src/shared/utils/analytics.ts b/quadratic-client/src/shared/utils/analytics.ts index 09577633cf..9f60062fac 100644 --- a/quadratic-client/src/shared/utils/analytics.ts +++ b/quadratic-client/src/shared/utils/analytics.ts @@ -1,12 +1,12 @@ import { debugShow } from '@/app/debugFlags'; +import { User as AuthUser } from '@/auth/auth'; import * as amplitude from '@amplitude/analytics-browser'; -import { User as Auth0User } from '@auth0/auth0-spa-js'; import { setUser } from '@sentry/react'; import mixpanel from 'mixpanel-browser'; // Quadratic only shares analytics on the QuadraticHQ.com hosted version where the environment variables are set. -type User = Auth0User | undefined; +type User = AuthUser | undefined; export function googleAnalyticsAvailable(): boolean { return import.meta.env.VITE_GOOGLE_ANALYTICS_GTAG && import.meta.env.VITE_GOOGLE_ANALYTICS_GTAG !== 'none'; diff --git a/quadratic-client/src/shared/utils/userUtil.ts b/quadratic-client/src/shared/utils/userUtil.ts index 4267f29850..25d43a895f 100644 --- a/quadratic-client/src/shared/utils/userUtil.ts +++ b/quadratic-client/src/shared/utils/userUtil.ts @@ -1,5 +1,5 @@ import { MultiplayerUser } from '@/app/web-workers/multiplayerWebWorker/multiplayerTypes'; -import { User } from '@auth0/auth0-spa-js'; +import { User } from '@/auth/auth'; export const displayName = (user: User | MultiplayerUser | undefined, you: boolean): string => { let name = ''; @@ -37,5 +37,5 @@ export const displayInitials = (user: User | MultiplayerUser | undefined): strin return user.email[0]; } } - return user?.index !== undefined ? user.index + 1 : '0'; + return user?.index !== undefined ? String(user.index + 1) : '0'; }; From 1102a076bf6ab59beb3ad36be07dda051a328a86 Mon Sep 17 00:00:00 2001 From: David DiMaria Date: Tue, 13 Aug 2024 10:03:24 -0600 Subject: [PATCH 018/113] Add getUsersFromOryByEmail() in api + cleanup --- quadratic-api/.env.docker | 4 +++- quadratic-api/.env.example | 4 +++- quadratic-api/.env.test | 1 + quadratic-api/src/app.ts | 3 --- quadratic-api/src/auth/auth.ts | 10 ++++++++-- quadratic-api/src/auth/auth0.ts | 3 ++- quadratic-api/src/auth/ory.ts | 19 ++++++++++++++++--- .../src/middleware/validateAccessToken.ts | 4 ++-- quadratic-client/.env.docker | 3 ++- quadratic-client/.env.example | 3 ++- 10 files changed, 39 insertions(+), 15 deletions(-) diff --git a/quadratic-api/.env.docker b/quadratic-api/.env.docker index b08992efb7..32e31136e6 100644 --- a/quadratic-api/.env.docker +++ b/quadratic-api/.env.docker @@ -5,7 +5,9 @@ DATABASE_URL='postgresql://postgres:postgres@postgres:5432/postgres' ENCRYPTION_KEY=eb4758047f74bdb2603cce75c4370327ca2c3662c4786867659126da8e64dfcc # Auth -AUTH_TYPE=auth0 # auth0 or ? +AUTH_TYPE=ory # auth0 or ory +ORY_JWKS_URI='http://localhost:3000/.well-known/jwks.json' +ORY_ADMIN_HOST=http://0.0.0.0:4434 # Storage STORAGE_TYPE=s3 # s3 or file-system diff --git a/quadratic-api/.env.example b/quadratic-api/.env.example index 155365901c..7596637df3 100644 --- a/quadratic-api/.env.example +++ b/quadratic-api/.env.example @@ -16,13 +16,15 @@ STRIPE_WEBHOOK_SECRET=STRIPE_WEBHOOK_SECRET ENCRYPTION_KEY=eb4758047f74bdb2603cce75c4370327ca2c3662c4786867659126da8e64dfcc # Auth -AUTH_TYPE=auth0 # auth0 or ? +AUTH_TYPE=auth0 # auth0 or ory AUTH0_JWKS_URI=https://quadratic-community.us.auth0.com/.well-known/jwks.json AUTH0_ISSUER=https://quadratic-community.us.auth0.com/ AUTH0_DOMAIN=quadratic-community.us.auth0.com AUTH0_CLIENT_ID=DCPCvqyU5Q0bJD8Q3QmJEoV48x1zLH7W AUTH0_CLIENT_SECRET=94dp3PDcxlI9ZDqBSvkdjQHWgGdx0ZSeyTr5-Rn3Kcts-ZyTdj1FLlJjCyqrTXEG AUTH0_AUDIENCE=community-quadratic +ORY_JWKS_URI='http://localhost:3000/.well-known/jwks.json' +ORY_ADMIN_HOST=http://0.0.0.0:4434 # Storage STORAGE_TYPE=s3 # s3 or file-system diff --git a/quadratic-api/.env.test b/quadratic-api/.env.test index a85b211524..7ae496b973 100644 --- a/quadratic-api/.env.test +++ b/quadratic-api/.env.test @@ -14,6 +14,7 @@ AUTH0_CLIENT_ID="AUTH0_CLIENT_ID" AUTH0_CLIENT_SECRET="AUTH0_CLIENT_SECRET" AUTH0_DOMAIN="AUTH0_DOMAIN" ORY_JWKS_URI='http://localhost:3000/.well-known/jwks.json' +ORY_ADMIN_HOST=http://0.0.0.0:4434 # Storage STORAGE_TYPE=s3 # s3 or file-system diff --git a/quadratic-api/src/app.ts b/quadratic-api/src/app.ts index a856796ec9..9f988ea270 100644 --- a/quadratic-api/src/app.ts +++ b/quadratic-api/src/app.ts @@ -61,9 +61,6 @@ app.use((req, res, next) => { return next(); }); -// static assets in the /public directory -app.use(express.static('public')); - // Health-check app.get('/', (req, res) => { res.status(200).json({ message: 'OK' }); diff --git a/quadratic-api/src/auth/auth.ts b/quadratic-api/src/auth/auth.ts index d369c29c17..9dde968b8a 100644 --- a/quadratic-api/src/auth/auth.ts +++ b/quadratic-api/src/auth/auth.ts @@ -1,6 +1,6 @@ import { AUTH_TYPE } from '../env-vars'; import { getUsersFromAuth0, jwtConfigAuth0, lookupUsersFromAuth0ByEmail } from './auth0'; -import { getUsersFromOry, jwtConfigOry } from './ory'; +import { getUsersFromOry, getUsersFromOryByEmail, jwtConfigOry } from './ory'; export type UsersRequest = { id: number; @@ -15,6 +15,10 @@ export type User = { picture?: string | undefined; }; +export type ByEmailUser = { + user_id?: string; +}; + export const getUsers = async (users: UsersRequest[]): Promise> => { switch (AUTH_TYPE) { case 'auth0': @@ -26,10 +30,12 @@ export const getUsers = async (users: UsersRequest[]): Promise { +export const getUsersByEmail = async (email: string): Promise => { switch (AUTH_TYPE) { case 'auth0': return await lookupUsersFromAuth0ByEmail(email); + case 'ory': + return await getUsersFromOryByEmail(email); default: throw new Error(`Unsupported auth type in getUsers(): ${AUTH_TYPE}`); } diff --git a/quadratic-api/src/auth/auth0.ts b/quadratic-api/src/auth/auth0.ts index 0ca31e3056..f399c15674 100644 --- a/quadratic-api/src/auth/auth0.ts +++ b/quadratic-api/src/auth/auth0.ts @@ -10,6 +10,7 @@ import { AUTH0_ISSUER, AUTH0_JWKS_URI, } from '../env-vars'; +import { ByEmailUser } from './auth'; // Guide to Setting up on Auth0 // 1. Create an Auth0 Machine to Machine Application @@ -101,7 +102,7 @@ export const getUsersFromAuth0 = async (users: { id: number; auth0Id: string }[] return usersById; }; -export const lookupUsersFromAuth0ByEmail = async (email: string) => { +export const lookupUsersFromAuth0ByEmail = async (email: string): Promise => { const auth0Users = await auth0.getUsersByEmail(email); return auth0Users; }; diff --git a/quadratic-api/src/auth/ory.ts b/quadratic-api/src/auth/ory.ts index 2e6137d739..e818e76c97 100644 --- a/quadratic-api/src/auth/ory.ts +++ b/quadratic-api/src/auth/ory.ts @@ -3,7 +3,7 @@ import * as Sentry from '@sentry/node'; import { Algorithm } from 'jsonwebtoken'; import jwksRsa, { GetVerificationKey } from 'jwks-rsa'; import { ORY_ADMIN_HOST, ORY_JWKS_URI } from '../env-vars'; -import { User } from './auth'; +import { ByEmailUser, User } from './auth'; const config = new Configuration({ basePath: ORY_ADMIN_HOST, @@ -23,7 +23,7 @@ export const jwtConfigOry = { algorithms: ['RS256'] as Algorithm[], }; -export const getUsersFromOry = async (users: { id: number; auth0Id: string }[]) => { +export const getUsersFromOry = async (users: { id: number; auth0Id: string }[]): Promise> => { // If we got nothing, we return an empty object if (users.length === 0) return {}; @@ -44,7 +44,7 @@ export const getUsersFromOry = async (users: { id: number; auth0Id: string }[]) // If we're missing data we expect, log it to Sentry and skip this user if (!oryUser || oryUser.traits.email === undefined) { Sentry.captureException({ - message: 'Auth0 user returned without `email`', + message: 'Ory user returned without `email`', level: 'error', extra: { auth0IdInOurDb: auth0Id, @@ -70,3 +70,16 @@ export const getUsersFromOry = async (users: { id: number; auth0Id: string }[]) return usersById; }; + +export const getUsersFromOryByEmail = async (email: string): Promise => { + let identities; + + try { + identities = (await sdk.listIdentities({ credentialsIdentifier: email })).data; + } catch (e) { + console.error(e); + return []; + } + + return identities.map(({ id }) => ({ user_id: id })); +}; diff --git a/quadratic-api/src/middleware/validateAccessToken.ts b/quadratic-api/src/middleware/validateAccessToken.ts index 4f1b023dcc..e73dab1ecb 100644 --- a/quadratic-api/src/middleware/validateAccessToken.ts +++ b/quadratic-api/src/middleware/validateAccessToken.ts @@ -1,5 +1,5 @@ -import { expressjwt } from 'express-jwt'; +import { Params, expressjwt } from 'express-jwt'; import { jwtConfig } from '../auth/auth'; // based on implementation from https://github.com/auth0-developer-hub/api_express_typescript_hello-world/blob/main/src/middleware/auth0.middleware.ts -export const validateAccessToken = expressjwt(jwtConfig() as any); +export const validateAccessToken = expressjwt(jwtConfig() as Params); diff --git a/quadratic-client/.env.docker b/quadratic-client/.env.docker index c0ab043320..c1abc9b575 100644 --- a/quadratic-client/.env.docker +++ b/quadratic-client/.env.docker @@ -4,4 +4,5 @@ VITE_QUADRATIC_MULTIPLAYER_URL=ws://localhost:3001/ws VITE_QUADRATIC_CONNECTION_URL=http://localhost:3003 # Auth -VITE_AUTH_TYPE=auth0 # auth0 or ? +VITE_AUTH_TYPE=ory # auth0 or ory +VITE_ORY_HOST=http://localhost:4433 diff --git a/quadratic-client/.env.example b/quadratic-client/.env.example index 2e9d3df5cb..c433de836a 100644 --- a/quadratic-client/.env.example +++ b/quadratic-client/.env.example @@ -8,8 +8,9 @@ VITE_QUADRATIC_MULTIPLAYER_URL=ws://localhost:3001/ws VITE_QUADRATIC_CONNECTION_URL=http://localhost:3003 # Auth -VITE_AUTH_TYPE=auth0 # auth0 or ? +VITE_AUTH_TYPE=auth0 # auth0 or ory VITE_AUTH0_ISSUER=https://quadratic-community.us.auth0.com/ VITE_AUTH0_DOMAIN=quadratic-community.us.auth0.com VITE_AUTH0_CLIENT_ID=DCPCvqyU5Q0bJD8Q3QmJEoV48x1zLH7W VITE_AUTH0_AUDIENCE=community-quadratic +VITE_ORY_HOST=http://localhost:4433 From e33cc356a69f561135ce8b9d5e2b73bfdd96e36e Mon Sep 17 00:00:00 2001 From: David DiMaria Date: Tue, 13 Aug 2024 11:41:40 -0600 Subject: [PATCH 019/113] Remove unused --- .../public/quadratic-local-jwks.json | 19 ------------------- quadratic-api/src/auth/auth.ts | 4 ++-- 2 files changed, 2 insertions(+), 21 deletions(-) delete mode 100644 quadratic-api/public/quadratic-local-jwks.json diff --git a/quadratic-api/public/quadratic-local-jwks.json b/quadratic-api/public/quadratic-local-jwks.json deleted file mode 100644 index 319a49cdec..0000000000 --- a/quadratic-api/public/quadratic-local-jwks.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "set": "quadratic-local", - "keys": [ - { - "alg": "RS256", - "d": "ZzQr3XjaRVCXFum_DwL-LiKFvAN9I15XJb-bzIGQDpLVolMmuJtOsvLWRa24dHRwGf6SPmZQP7lKIVRjPGHReoICdKNMAW0lYM0nvDMrMRC9TQsVg-0HxMHvkw29Ka4xPD_YtcPuzP4MKfeaE8D7pBiXFd4uoBBAVYKAvsBxLz7E4l3JIuy-S1kGa4if2q6IgHrgxnR95NhNynUNp7hJElNxWxKZoOboHaS31ZEe9Pqpp8MYDt6E6BrjYvvtsJjNLNEKMWw70mSgcSAAlgN3u0Bf_PazYYn-zGxYDpODAKSclXhPvyXxnJ2NC-wr6aLpSdxFqwz4JRAUTWCBFIWS4xDJPzgv2dc_JrQuYYfDvGwr5FAl7vGSFTU6w3e5swCZhKLlIjx9nLKaZbYejy01XrmCNm7VbDHX9iVjrxAww4RXLsqx2A3GR63GrE5r7yV8cDvyLe5qt9MfJm6M2y5rtBRu3XGtAKf6K2lP-qYixWW95QJ_3OIiYutFwCLxbI-LElaYgCRt6yCcfUeiY_wy2X-ts2RgYlAS8F6aQHQuX0Om6FqYiGkkh1snorUg5tkByoWxQeICGBRjpRl9sR3fzqI8dznB71d0kmKSyTYl3qhTKb0Btv56VqxJQsgMFUmfceQxkH0nnuXyEWIM7tF_4LsQtNYCFLHQore2-Qrxb2E", - "dp": "eibm6Jw6nIBChxwJlyjerBKwvuBP4LFP3rF-5kegPB8qrBnBxhOVZrMOl5bYYh73fIHe-rgE_gTzH-RzdzJsrsR2nzQcHQFhu5GHoMuYhSZJbQ32hERNUwZFRfIwtfhTU6-Un--eijMzKkCM-5VEipjTzj6gtHDAJsX1JL6ttGMqZpgLyQWp_Hvr_rGm3oQcSXZfrXmOtjtMaMPq56KF6fKRY8MviDFVN3GlRDr84s7cCdCwxnr2_dp9y9NajaRh76TR60gEEe5ZFs6fK_iZQUobhv6gVCvQB1jS6yC1rREj60XXSTK1BCafBlF6fQw0NsfL97Dj6sHuQlpv60UhHQ", - "dq": "j-aRMph1lctEupphXHosEGsjKOKAJw1QsJC5GQCvF39B-Bb91chekitSt_EOPt2i5gY4b4dI-oKCMh_BFr_BCV-RsGlDC3y3x_Ka1wF5GCN9JpcyGG2wOZOXYlglYG82ItyqRVgJZ7US8mqk_l1e8whReAs8rhNoQhLtigJgdC4cGeklZh45-be97ToTCfI3_R2WDcbEE7YQ8VkGsea3ZcOPAWBkzpwZIyOKi4Cyo5mwmQ_9pKkiFMWD1PlZGTCxltJW-DsQ-xZT9UKxkjxR2GyCCQfQ04_NjkIG6wR3H7aPqbo_xP-Cav30JaB-3_nxW3-yHvR7Me-6Zi1lgzmQOQ", - "e": "AQAB", - "kid": "438c4767-7651-45e6-8dfd-c19335a3ee1a", - "kty": "RSA", - "n": "6bKB0-vmuMAET-Xe7OUITo1hujQyg5z0hQWfIKavdMP6UFuBOjPBpJ_KuHoee3zq_MlJBHsLJTeC_IB5SxIB-_ZHWgrwPn5DvJYFSKU5cXv6iB1RdMxAnyjHl_SuEeWqt75gZHPgzVL2E2MDsQ5ypPLiUb3XeR4GPh5ynEnyKcjgm_vJBie-Z3vU8jXqVzM-CDPzWGkw7Ru1gwYRoGrbojWQbiQYTWspIqnFiQWTwJyDkYLtGy2EDD1z1ixoxKKFZq2JGxsPU1gbOTudVwUy4meW0uUqnmDFM0b_FpZHC_AN0SnQRB-jLr-slZoi5tnwJ_WF5NC0vqKotcKHhMqEKVv4egYNCbk_QHeta2MHavnAhuVw6l0c5G_N64T5pVcIyBvkxepgRTvXOqERHoPGnVwndCKZE3mgTaixIXadcp1afMENJFP-ZPUDZ0Y0yel8naNz7twYnUrzSKMXBIB4K6WEwNAUEVvC7bDZ-XGIXCJ-Ah29C5evnTPTpqrlqpuOl-Bfr5WJrvohv3jkTTH9sTRqnlerfaIUD8FNvtovz-yW3WtqFCQpoPKlePL1HRvWCzRU4blRFEb66MCdSgsXM6K7SIeqXyQ6T1scOFgnEAV1zQOJYB7qKIuc-wMNg0lJcULXhEIr2WVSKlxWWWd9i4rZSylHCP1Nvl2Ct6AyUac", - "p": "66m8M7DdkNHL76aoYTQePeV6rw472AmmgHR1j5S8Tyd-tLAWL1Yxi1CpIRBiGL5Xxq0GKz_bwHh3C8SkIfyfeNeC_UMr2mvgcjYDLsNQgX2diz8p_h9FhcrMMuEjImMneVZhc-QyDZuws5EvGfTB05PkBObWMq72LzjGq8K0fyes_vp8iK9XcxcMg7Zt_2OxbQGufSNYdOoev2Nl7eyZSXrwNPfADzL_5n1I6wGmue2DTGA9bX88IP0typ-KkMZYkGGt25H391YwqVki6VhJYqRpsavcwJe3rXizFisVCfvgrukBEBjTTXPxkWeRDt3H3R-JWTAuwEYfiHQDHiMXPw", - "q": "_d1YUX4aTGbAPDHyaqF2u2SWZJvUiozXBHRcqCEUCKvnRR4-rHIs4QYCIZYD3063WBVvSKNL7q-T-nkSiKhdEOwHoqGYGaItuHjq-OzH_ig9UXL4V2DmthPBt5kgcNOFfUQrGPD2NLT6-QtDU6eGxfpr7pWAgjdTGNZI--KpHPQkab8mO-jDfP84BmUfcc1yKVEHC5b2vBMiZ0_qy529qz7G5jrTA7TwjTfu9VyY_Gl2u62Zq69dWp96xP_CYgS_Gj964V53MNG0mBCkGVFBGyIJ5H1LUrSK4pCgS7dFhraZDgllZfo4nwz74hG2uPp2OJVwcpqxqD2N4bD0x1BTmQ", - "qi": "tstI7N-hHEjfrujyWgwaVtuChFx85JRiCtkPvLInLnaHXVKxFMWm7boIvLYJ8XHqd1RgWADRP-lofJfL3qtt3-tZimgrfduKcz1_vLYm_P_YYwtR6Fv7TNDjGiF3Pf-J1RSB8uYnKs2Hqc3s7XqIASeFEzxeV07Yiohmx4dIYwrqZePxr0siUaswGP4nsKJD1QcfeXvNt_mXnLcHX0LY740fCVzGDxu7Yiwuxtu1VU09gemexfC1XkPbYvi0IY8UDtl9qu5DM2cdcmuCnZP3QiJM33QBuvRM4FJGNUV_3nOrzvt0xdU3TkNmvkHlKDfWB0fY5KMpeQrB5DO1AhEPtQ", - "use": "sig" - } - ] -} diff --git a/quadratic-api/src/auth/auth.ts b/quadratic-api/src/auth/auth.ts index 9dde968b8a..74df06ee67 100644 --- a/quadratic-api/src/auth/auth.ts +++ b/quadratic-api/src/auth/auth.ts @@ -37,7 +37,7 @@ export const getUsersByEmail = async (email: string): Promise => case 'ory': return await getUsersFromOryByEmail(email); default: - throw new Error(`Unsupported auth type in getUsers(): ${AUTH_TYPE}`); + throw new Error(`Unsupported auth type in getUsersByEmail(): ${AUTH_TYPE}`); } }; @@ -48,6 +48,6 @@ export const jwtConfig = () => { case 'ory': return jwtConfigOry; default: - throw new Error(`Unsupported auth type in getUsers(): ${AUTH_TYPE}`); + throw new Error(`Unsupported auth type in jwtConfig(): ${AUTH_TYPE}`); } }; From 30e7412fbb37e73281d7bb95a20bc7fc15e16819 Mon Sep 17 00:00:00 2001 From: David DiMaria Date: Thu, 15 Aug 2024 18:54:12 -0600 Subject: [PATCH 020/113] iMVP of self-hosting --- client.Dockerfile | 3 +- docker-compose.yml | 12 +- docker/ory-auth/config/kratos.yml | 2 +- quadratic-api/.env.docker | 19 +- quadratic-api/.env.example | 3 +- quadratic-api/.env.test | 3 +- quadratic-api/src/env-vars.ts | 1 + .../src/routes/v0/teams.$uuid.GET.ts | 22 +- quadratic-api/src/storage/fileSystem.ts | 18 +- quadratic-api/src/storage/storage.ts | 1 - quadratic-connection/.env.docker | 4 +- quadratic-files/.env.docker | 10 +- quadratic-files/src/storage.rs | 2 +- quadratic-multiplayer/.env.docker | 6 +- self-hosting/.gitignore | 9 + self-hosting/README.md | 2 + self-hosting/docker-compose.yml | 209 ++++++++++++++++++ self-hosting/docker/caddy/config/Caddyfile | 5 + .../ory-auth/config/identity.schema.json | 47 ++++ .../docker/ory-auth/config/kratos.yml | 130 +++++++++++ self-hosting/docker/postgres/.gitignore | 0 self-hosting/docker/postgres/scripts/init.sh | 18 ++ self-hosting/docker/redis/.gitignore | 0 self-hosting/start.sh | 32 +++ 24 files changed, 514 insertions(+), 44 deletions(-) create mode 100644 self-hosting/.gitignore create mode 100644 self-hosting/README.md create mode 100644 self-hosting/docker-compose.yml create mode 100644 self-hosting/docker/caddy/config/Caddyfile create mode 100644 self-hosting/docker/ory-auth/config/identity.schema.json create mode 100644 self-hosting/docker/ory-auth/config/kratos.yml create mode 100644 self-hosting/docker/postgres/.gitignore create mode 100755 self-hosting/docker/postgres/scripts/init.sh create mode 100644 self-hosting/docker/redis/.gitignore create mode 100755 self-hosting/start.sh diff --git a/client.Dockerfile b/client.Dockerfile index 0f55931755..834914b14a 100644 --- a/client.Dockerfile +++ b/client.Dockerfile @@ -47,6 +47,7 @@ RUN echo 'Building quadratic-rust-client...' && npm run build --workspace=quadra WORKDIR /app RUN echo 'Building front-end...' RUN npm ci +RUN npm install typescript RUN npx tsc ./quadratic-shared/*.ts RUN npm run build --workspace=quadratic-client @@ -56,6 +57,6 @@ RUN npm run build --workspace=quadratic-client FROM nginx:stable-alpine COPY --from=build /app/build /usr/share/nginx/html -EXPOSE 80 443 3000 +EXPOSE 3000 CMD ["nginx", "-g", "daemon off;"] \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index a7557a4791..7bd9f4b221 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -89,14 +89,13 @@ services: build: context: . dockerfile: client.Dockerfile - env_file: - - quadratic-client/.env.local - - quadratic-client/.env.docker - # override env vars here environment: + VITE_DEBUG: 1 VITE_QUADRATIC_API_URL: http://localhost:8000 - VITE_QUADRATIC_MULTIPLAYER_URL: ws://localhost:3001 - VITE_QUADRATIC_CONNECTION_URL: http://0.0.0.0:3003 + VITE_QUADRATIC_MULTIPLAYER_URL: ws://localhost:3001/ws + VITE_QUADRATIC_CONNECTION_URL: http://localhost:3003 + VITE_AUTH_TYPE: ory + VITE_ORY_HOST: http://localhost:4433 restart: "always" ports: # - "3000:3000" @@ -219,6 +218,7 @@ services: # condition: service_healthy # quadratic-api: # condition: service_started + profiles: - backend - quadratic-connection diff --git a/docker/ory-auth/config/kratos.yml b/docker/ory-auth/config/kratos.yml index 5c7106a657..ec26ebb261 100644 --- a/docker/ory-auth/config/kratos.yml +++ b/docker/ory-auth/config/kratos.yml @@ -94,7 +94,7 @@ session: jwt_template: jwks_url: http://host.docker.internal:3000/.well-known/jwks.json # claims_mapper_url: base64://... # A JsonNet template for modifying the claims - ttl: 1m # 1 minute (defaults to 10 minutes) + ttl: 24h # 24 hours (defaults to 10 minutes) log: level: debug diff --git a/quadratic-api/.env.docker b/quadratic-api/.env.docker index ae01c7b1b4..47bcf275b8 100644 --- a/quadratic-api/.env.docker +++ b/quadratic-api/.env.docker @@ -1,15 +1,20 @@ +CORS="*" +DATABASE_URL="postgresql://postgres:postgres@host.docker.internal:5432/postgres" ENVIRONMENT=docker -DATABASE_URL='postgresql://postgres:postgres@postgres:5432/postgres' -AWS_S3_ENDPOINT=http://localstack:4566 +STRIPE_SECRET_KEY=STRIPE_SECRET_KEY +STRIPE_WEBHOOK_SECRET=STRIPE_WEBHOOK_SECRET +OPENAI_API_KEY= +M2M_AUTH_TOKEN=M2M_AUTH_TOKEN # Hex string to be used as the key for enctyption, use npm run key:generate ENCRYPTION_KEY=eb4758047f74bdb2603cce75c4370327ca2c3662c4786867659126da8e64dfcc # Auth -AUTH_TYPE=ory # auth0 or ory -ORY_JWKS_URI='http://localhost:3000/.well-known/jwks.json' -ORY_ADMIN_HOST=http://0.0.0.0:4434 +AUTH_TYPE=ory +ORY_JWKS_URI="http://host.docker.internal/.well-known/jwks.json" +ORY_ADMIN_HOST=http://host.docker.internal:4434 # Storage -STORAGE_TYPE=s3 # s3 or file-system -QUADRATIC_FILE_URI=http://127.0.0.1:3002 \ No newline at end of file +STORAGE_TYPE=file-system +QUADRATIC_FILE_URI=http://host.docker.internal:3002 +QUADRATIC_FILE_URI_PUBLIC=http://localhost:3002 \ No newline at end of file diff --git a/quadratic-api/.env.example b/quadratic-api/.env.example index 7596637df3..4e0642d415 100644 --- a/quadratic-api/.env.example +++ b/quadratic-api/.env.example @@ -28,7 +28,8 @@ ORY_ADMIN_HOST=http://0.0.0.0:4434 # Storage STORAGE_TYPE=s3 # s3 or file-system -QUADRATIC_FILE_URI=http://127.0.0.1:3002 +QUADRATIC_FILE_URI=http://localhost:3002 +QUADRATIC_FILE_URI_PUBLIC=http://localhost:3002 AWS_S3_REGION=us-east-2 AWS_S3_ACCESS_KEY_ID=test AWS_S3_SECRET_ACCESS_KEY=test diff --git a/quadratic-api/.env.test b/quadratic-api/.env.test index 7ae496b973..a59f83027e 100644 --- a/quadratic-api/.env.test +++ b/quadratic-api/.env.test @@ -18,7 +18,8 @@ ORY_ADMIN_HOST=http://0.0.0.0:4434 # Storage STORAGE_TYPE=s3 # s3 or file-system -QUADRATIC_FILE_URI=http://127.0.0.1:3002 +QUADRATIC_FILE_URI=http://localhost:3002 +QUADRATIC_FILE_URI_PUBLIC=http://localhost:3002 AWS_S3_REGION=us-west-2 AWS_S3_BUCKET_NAME=AWS_S3_BUCKET_NAME AWS_S3_ACCESS_KEY_ID=AWS_S3_ACCESS_KEY_ID diff --git a/quadratic-api/src/env-vars.ts b/quadratic-api/src/env-vars.ts index 9b93acde0e..401fe680e1 100644 --- a/quadratic-api/src/env-vars.ts +++ b/quadratic-api/src/env-vars.ts @@ -19,6 +19,7 @@ export const AUTH0_AUDIENCE = process.env.AUTH0_AUDIENCE as string; export const ORY_JWKS_URI = process.env.ORY_JWKS_URI as string; export const ORY_ADMIN_HOST = process.env.ORY_ADMIN_HOST as string; export const QUADRATIC_FILE_URI = process.env.QUADRATIC_FILE_URI as string; +export const QUADRATIC_FILE_URI_PUBLIC = process.env.QUADRATIC_FILE_URI_PUBLIC as string; export const AWS_S3_REGION = process.env.AWS_S3_REGION as string; export const AWS_S3_ACCESS_KEY_ID = process.env.AWS_S3_ACCESS_KEY_ID as string; export const AWS_S3_SECRET_ACCESS_KEY = process.env.AWS_S3_SECRET_ACCESS_KEY as string; diff --git a/quadratic-api/src/routes/v0/teams.$uuid.GET.ts b/quadratic-api/src/routes/v0/teams.$uuid.GET.ts index 699c25fe1c..69439fa61a 100644 --- a/quadratic-api/src/routes/v0/teams.$uuid.GET.ts +++ b/quadratic-api/src/routes/v0/teams.$uuid.GET.ts @@ -114,16 +114,18 @@ async function handler(req: Request, res: Response { - const { email, name, picture } = auth0UsersById[id]; - return { - id, - email, - role, - name, - picture, - }; - }), + users: dbUsers + .filter(({ userId: id }) => auth0UsersById[id]) + .map(({ userId: id, role }) => { + const { email, name, picture } = auth0UsersById[id]; + return { + id, + email, + role, + name, + picture, + }; + }), invites: dbInvites.map(({ email, role, id }) => ({ email, role, id })), files: dbFiles .filter((file) => !file.ownerUserId) diff --git a/quadratic-api/src/storage/fileSystem.ts b/quadratic-api/src/storage/fileSystem.ts index 90f9c953bb..d048af333a 100644 --- a/quadratic-api/src/storage/fileSystem.ts +++ b/quadratic-api/src/storage/fileSystem.ts @@ -1,17 +1,21 @@ import { Request } from 'express'; import multer from 'multer'; import stream, { Readable } from 'node:stream'; -import { QUADRATIC_FILE_URI } from '../env-vars'; +import { QUADRATIC_FILE_URI, QUADRATIC_FILE_URI_PUBLIC } from '../env-vars'; import { UploadFile } from '../types/Request'; import { encryptFromEnv } from '../utils/crypto'; import { UploadFileResponse } from './storage'; -const generateUrl = (key: string): string => `${QUADRATIC_FILE_URI}/storage/${key}`; -const generatePresignedUrl = (key: string): string => generateUrl(`presigned/${key}`); +const generateUrl = (key: string, isPublic: boolean): string => { + const baseUrl = isPublic ? QUADRATIC_FILE_URI_PUBLIC : QUADRATIC_FILE_URI; + return `${baseUrl}/storage/${key}`; +}; + +const generatePresignedUrl = (key: string): string => generateUrl(`presigned/${key}`, true); // Get the URL for a given file (key) for the file service. export const getStorageUrl = (key: string): string => { - return generateUrl(key); + return generateUrl(key, true); }; // Get a presigned URL for a given file (key) for the file service. @@ -22,7 +26,11 @@ export const getPresignedStorageUrl = (key: string): string => { // Upload a file to the file service. export const upload = async (key: string, contents: string | Uint8Array, jwt: string): Promise => { - const url = generateUrl(key); + const url = generateUrl(key, false); + + if (typeof contents === 'string') { + contents = new Uint8Array(Buffer.from(contents, 'base64')); + } try { const response = await fetch(url, { diff --git a/quadratic-api/src/storage/storage.ts b/quadratic-api/src/storage/storage.ts index ba8791d769..1feaa15d6a 100644 --- a/quadratic-api/src/storage/storage.ts +++ b/quadratic-api/src/storage/storage.ts @@ -22,7 +22,6 @@ export const getFileUrl = async (key: string) => { // Get a presigned URL for a given file (key). export const getPresignedFileUrl = async (key: string) => { - console.warn('getPresignedFileUrl', getPresignedFileUrl); switch (STORAGE_TYPE) { case 's3': return await generatePresignedUrl(key); diff --git a/quadratic-connection/.env.docker b/quadratic-connection/.env.docker index 87620b829e..e47987bb11 100644 --- a/quadratic-connection/.env.docker +++ b/quadratic-connection/.env.docker @@ -2,8 +2,8 @@ HOST=0.0.0.0 PORT=3003 ENVIRONMENT=docker -AUTH0_JWKS_URI=https://dev-nje7dw8s.us.auth0.com/.well-known/jwks.json -QUADRATIC_API_URI=http://localhost:8000 +AUTH0_JWKS_URI=http://host.docker.internal/.well-known/jwks.json +QUADRATIC_API_URI=http://host.docker.internal:8000 M2M_AUTH_TOKEN=M2M_AUTH_TOKEN MAX_RESPONSE_BYTES=15728640 # 15MB STATIC_IPS=0.0.0.0,127.0.0.1 diff --git a/quadratic-files/.env.docker b/quadratic-files/.env.docker index 29d2724d36..ee04e9e338 100644 --- a/quadratic-files/.env.docker +++ b/quadratic-files/.env.docker @@ -6,18 +6,18 @@ TRUNCATE_FILE_CHECK_S=60 TRUNCATE_TRANSACTION_AGE_DAYS=5 # ENVIRONMENT=docker -AUTH0_JWKS_URI=https://dev-nje7dw8s.us.auth0.com/.well-known/jwks.json -QUADRATIC_API_URI=http://quadratic-api:8000 +AUTH0_JWKS_URI=http://host.docker.internal/.well-known/jwks.json +QUADRATIC_API_URI=http://host.docker.internal:8000 M2M_AUTH_TOKEN=M2M_AUTH_TOKEN -PUBSUB_HOST=redis +PUBSUB_HOST=host.docker.internal PUBSUB_PORT=6379 PUBSUB_PASSWORD= PUBSUB_ACTIVE_CHANNELS=active_channels PUBSUB_PROCESSED_TRANSACTIONS_CHANNEL=processed_transactions # Storage -STORAGE_TYPE=s3 # s3 or file-system +STORAGE_TYPE=file-system # s3 or file-system # Storage: s3 AWS_S3_REGION= @@ -26,5 +26,5 @@ AWS_S3_ACCESS_KEY_ID= AWS_S3_SECRET_ACCESS_KEY= # Storage: file-system -STORAGE_DIR=./../docker/file-storage +STORAGE_DIR=/file-storage STORAGE_ENCRYPTION_KEYS=eb4758047f74bdb2603cce75c4370327ca2c3662c4786867659126da8e64dfcc diff --git a/quadratic-files/src/storage.rs b/quadratic-files/src/storage.rs index 34a31c1344..296e1f70c7 100644 --- a/quadratic-files/src/storage.rs +++ b/quadratic-files/src/storage.rs @@ -52,7 +52,7 @@ pub(crate) async fn get_presigned_storage( Ok(file.into_response()) } _ => Err(FilesError::Storage( - "Presigned URLs supported in FileSystem storage options".to_string(), + "Presigned URLs only supported in FileSystem storage options".to_string(), )), } } diff --git a/quadratic-multiplayer/.env.docker b/quadratic-multiplayer/.env.docker index 2f33cf116d..97e42c0c50 100644 --- a/quadratic-multiplayer/.env.docker +++ b/quadratic-multiplayer/.env.docker @@ -2,14 +2,14 @@ HOST=0.0.0.0 PORT=3001 HEARTBEAT_CHECK_S=3 HEARTBEAT_TIMEOUT_S=600 -QUADRATIC_API_URI=http://quadratic-api:8000 +QUADRATIC_API_URI=http://host.docker.internal:8000 M2M_AUTH_TOKEN=M2M_AUTH_TOKEN ENVIRONMENT=docker -PUBSUB_HOST=redis +PUBSUB_HOST=host.docker.internal PUBSUB_PORT=6379 PUBSUB_PASSWORD= PUBSUB_ACTIVE_CHANNELS=active_channels -AUTH0_JWKS_URI=https://dev-nje7dw8s.us.auth0.com/.well-known/jwks.json +AUTH0_JWKS_URI=http://host.docker.internal/.well-known/jwks.json AUTHENTICATE_JWT=true \ No newline at end of file diff --git a/self-hosting/.gitignore b/self-hosting/.gitignore new file mode 100644 index 0000000000..1004414824 --- /dev/null +++ b/self-hosting/.gitignore @@ -0,0 +1,9 @@ +docker/localstack/data +docker/mysql/data +docker/postgres/data +docker/redis/data +docker/static/html +docker/postgres-connection/data +docker/mysql-connection/data +docker/file-storage +docker/caddy/quadratic-client \ No newline at end of file diff --git a/self-hosting/README.md b/self-hosting/README.md new file mode 100644 index 0000000000..1afcca9fa6 --- /dev/null +++ b/self-hosting/README.md @@ -0,0 +1,2 @@ +# Quadratic Self-Hosting + diff --git a/self-hosting/docker-compose.yml b/self-hosting/docker-compose.yml new file mode 100644 index 0000000000..b1e27233be --- /dev/null +++ b/self-hosting/docker-compose.yml @@ -0,0 +1,209 @@ +version: "3.8" + +services: + redis: + image: redis/redis-stack:latest + restart: always + ports: + - "6379:6379" + - "8001:8001" + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: "5s" + volumes: + - ./docker/redis/data:/data + profiles: + - base + + postgres: + image: postgres:15 + restart: always + container_name: postgres + ports: + - "5432:5432" + environment: + POSTGRES_USER: postgres + PGUSER: postgres + POSTGRES_PASSWORD: postgres + ADDITIONAL_DATABASES: kratos + healthcheck: + test: ["CMD-SHELL", "pg_isready -U postgres -d postgres"] + interval: 10s + timeout: 5s + retries: 5 + volumes: + - ./docker/postgres/data:/var/lib/postgresql/data + - ./docker/postgres/scripts:/docker-entrypoint-initdb.d + profiles: + - base + + caddy: + image: caddy:latest + container_name: caddy + ports: + - "80:80" + volumes: + - ./docker/caddy/config/Caddyfile:/etc/caddy/Caddyfile + - ./docker/caddy/quadratic-client:/srv + depends_on: + - frontend + profiles: + - caddy + + quadratic-api: + image: quadratic_quadratic-api + env_file: + - ../quadratic-api/.env.docker + restart: "always" + ports: + - "8000:8000" + command: "npm run start:prod --workspace=quadratic-api" + depends_on: + postgres: + condition: service_healthy + profiles: + - api + - frontend + + quadratic-multiplayer: + image: quadratic-multiplayer + env_file: + - ../quadratic-multiplayer/.env.docker + # override env vars here + environment: + RUST_LOG: info + restart: "always" + ports: + - "3001:3001" + depends_on: + postgres: + condition: service_healthy + redis: + condition: service_healthy + quadratic-api: + condition: service_started + profiles: + - backend + - multiplayer + networks: + - host + + quadratic-files: + image: quadratic-files + env_file: + - ../quadratic-files/.env.docker + # override env vars here + environment: + RUST_LOG: info + restart: "always" + ports: + - "3002:3002" + depends_on: + postgres: + condition: service_healthy + redis: + condition: service_healthy + quadratic-api: + condition: service_started + volumes: + - ./docker/file-storage:/file-storage + profiles: + - backend + - files + networks: + - host + + quadratic-connection: + image: quadratic-connection + env_file: + - ../quadratic-connection/.env.docker + # override env vars here + environment: + RUST_LOG: info + restart: "always" + ports: + - "3003:3003" + depends_on: + caddy: + condition: service_started + # postgres: + # condition: service_healthy + # quadratic-api: + # condition: service_started + profiles: + - backend + - connection + + # Auth Providers + + ory-auth: + image: oryd/kratos:v1.2.0 + ports: + - "4433:4433" # public + - "4434:4434" # admin + command: serve -c /etc/config/kratos/kratos.yml --dev --watch-courier + volumes: + - ./docker/ory-auth/config:/etc/config/kratos + environment: + DSN: postgresql://postgres:postgres@host.docker.internal:5432/kratos?sslmode=disable + LOG_LEVEL: trace + restart: unless-stopped + depends_on: + - postgres + - ory-auth-migrate + profiles: + - ory + networks: + - host + + ory-auth-migrate: + image: oryd/kratos:v1.2.0 + command: migrate -c /etc/config/kratos/kratos.yml sql -e --yes + volumes: + - ./docker/ory-auth/config:/etc/config/kratos + environment: + DSN: postgresql://postgres:postgres@host.docker.internal:5432/kratos?sslmode=disable + restart: on-failure + depends_on: + - postgres + profiles: + - ory + networks: + - host + + ory-auth-node: + image: oryd/kratos-selfservice-ui-node:v1.2.0 + ports: + - "4455:4455" + environment: + PORT: 4455 + SECURITY_MODE: + KRATOS_PUBLIC_URL: http://host.docker.internal:4433/ + KRATOS_BROWSER_URL: http://localhost:4433/ + COOKIE_SECRET: changeme + CSRF_COOKIE_NAME: ory_csrf_ui + CSRF_COOKIE_SECRET: changeme + restart: on-failure + profiles: + - ory + networks: + - host + + ory-auth-mail: + image: oryd/mailslurper:latest-smtps + ports: + - "1025:1025" + - "4436:4436" + - "4437:4437" + - "8080:8080" + profiles: + - ory + networks: + - host + +volumes: + docker: + name: docker + +networks: + host: diff --git a/self-hosting/docker/caddy/config/Caddyfile b/self-hosting/docker/caddy/config/Caddyfile new file mode 100644 index 0000000000..5e94cef443 --- /dev/null +++ b/self-hosting/docker/caddy/config/Caddyfile @@ -0,0 +1,5 @@ +:80 { + root * /srv + try_files {path} /index.html + file_server +} \ No newline at end of file diff --git a/self-hosting/docker/ory-auth/config/identity.schema.json b/self-hosting/docker/ory-auth/config/identity.schema.json new file mode 100644 index 0000000000..a953fc68ec --- /dev/null +++ b/self-hosting/docker/ory-auth/config/identity.schema.json @@ -0,0 +1,47 @@ +{ + "$id": "https://schemas.ory.sh/presets/kratos/quickstart/email-password/identity.schema.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Person", + "type": "object", + "properties": { + "traits": { + "type": "object", + "properties": { + "email": { + "type": "string", + "format": "email", + "title": "E-Mail", + "minLength": 3, + "ory.sh/kratos": { + "credentials": { + "password": { + "identifier": true + } + }, + "verification": { + "via": "email" + }, + "recovery": { + "via": "email" + } + } + }, + "name": { + "type": "object", + "properties": { + "first": { + "title": "First Name", + "type": "string" + }, + "last": { + "title": "Last Name", + "type": "string" + } + } + } + }, + "required": ["email"], + "additionalProperties": false + } + } +} diff --git a/self-hosting/docker/ory-auth/config/kratos.yml b/self-hosting/docker/ory-auth/config/kratos.yml new file mode 100644 index 0000000000..cf113a485d --- /dev/null +++ b/self-hosting/docker/ory-auth/config/kratos.yml @@ -0,0 +1,130 @@ +# https://raw.githubusercontent.com/ory/kratos/v1.2.0/.schemastore/config.schema.json +version: v1.2.0 + +dsn: memory + +serve: + public: + base_url: http://localhost:4433/ + cors: + enabled: true + allowed_origins: + - http://localhost + - http://localhost:3000 + allowed_methods: + - POST + - GET + - PUT + - PATCH + - DELETE + allowed_headers: + - Authorization + - Access-Control-Allow-Origin + - Cookie + - Content-Type + exposed_headers: + - Content-Type + - Set-Cookie + admin: + base_url: http://kratos:4434/ + +selfservice: + default_browser_return_url: http://localhost + allowed_return_urls: + - http://localhost:4455 + - http://localhost:3000 + - http://localhost:19006/Callback + - exp://localhost:8081/--/Callback + + methods: + password: + enabled: true + totp: + config: + issuer: Kratos + enabled: true + lookup_secret: + enabled: true + link: + enabled: true + code: + enabled: true + + flows: + error: + ui_url: http://localhost:4455/error + + settings: + ui_url: http://localhost:4455/settings + privileged_session_max_age: 15m + required_aal: highest_available + + recovery: + enabled: true + ui_url: http://localhost:4455/recovery + use: code + + verification: + enabled: true + ui_url: http://localhost:4455/verification + use: code + after: + default_browser_return_url: http://localhost + + logout: + after: + default_browser_return_url: http://localhost:4455/login + + login: + ui_url: http://localhost:4455/login + lifespan: 10m + + registration: + lifespan: 10m + ui_url: http://localhost:4455/registration + after: + password: + hooks: + - hook: session + - hook: show_verification_ui + +session: + whoami: + tokenizer: + templates: + jwt_template: + jwks_url: http://host.docker.internal/.well-known/jwks.json + # claims_mapper_url: base64://... # A JsonNet template for modifying the claims + ttl: 24h # 24 hours (defaults to 10 minutes) + +log: + level: debug + format: text + leak_sensitive_values: true + +secrets: + cookie: + - PLEASE-CHANGE-ME-I-AM-VERY-INSECURE + cipher: + - 32-LONG-SECRET-NOT-SECURE-AT-ALL + +ciphers: + algorithm: xchacha20-poly1305 + +hashers: + algorithm: bcrypt + bcrypt: + cost: 8 + +identity: + default_schema_id: default + schemas: + - id: default + url: file:///etc/config/kratos/identity.schema.json + +courier: + smtp: + connection_uri: smtps://test:test@host.docker.internal:1025/?skip_ssl_verify=true + +feature_flags: + use_continue_with_transitions: true \ No newline at end of file diff --git a/self-hosting/docker/postgres/.gitignore b/self-hosting/docker/postgres/.gitignore new file mode 100644 index 0000000000..e69de29bb2 diff --git a/self-hosting/docker/postgres/scripts/init.sh b/self-hosting/docker/postgres/scripts/init.sh new file mode 100755 index 0000000000..49102ba731 --- /dev/null +++ b/self-hosting/docker/postgres/scripts/init.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +set -e +set -u + +function create_user_and_database() { + local database=$1 + echo "Creating database '$database' with user '$POSTGRES_USER'" + psql -c "CREATE DATABASE $database;" || { echo "Failed to create database '$database'"; exit 1; } + echo "Database '$database' created" +} + +if [ -n "$ADDITIONAL_DATABASES" ]; then + for i in ${ADDITIONAL_DATABASES//,/ } + do + create_user_and_database $1 + done +fi diff --git a/self-hosting/docker/redis/.gitignore b/self-hosting/docker/redis/.gitignore new file mode 100644 index 0000000000..e69de29bb2 diff --git a/self-hosting/start.sh b/self-hosting/start.sh new file mode 100755 index 0000000000..3ed4813e40 --- /dev/null +++ b/self-hosting/start.sh @@ -0,0 +1,32 @@ +#!/bin/sh + +escape_for_sed() { + local input="$1" + printf '%s\n' "$input" | sed -e 's/[\/&]/\\&/g' +} + +replace_env_vars() { + TEMP=$'\r\n' GLOBIGNORE='*' command eval 'ENV_VARS=($(cat .env))' + + find "docker/caddy/quadratic-client" -type f -name "*.js" | while read file; do + echo "Replacing values in $file" + + for env_var in "${ENV_VARS[@]}"; do + var=${env_var%=*} + val=${env_var#*=} + escaped_val=$(escape_for_sed "$val") + + # echo "Replacing $var with ${val} in $file" + + sed -i '' "s/\($var:\"\)[^\"]*\"/\1$(echo "$escaped_val")\"/g" $file + done + done +} + +cd .. +rm -rf self-hosting/docker/caddy/quadratic-client/* +npm run build --workspace=quadratic-client +cp -r quadratic-client/build self-hosting/docker/caddy/quadratic-client + +cd self-hosting +# replace_env_vars \ No newline at end of file From 770de012019876f788dcb489c3c791147479dbd3 Mon Sep 17 00:00:00 2001 From: David DiMaria Date: Tue, 20 Aug 2024 17:52:37 -0600 Subject: [PATCH 021/113] Checkpoint after full setup, plus non-working TLS --- client.Dockerfile | 57 ++++++++++++++----- docker/ory-auth/config/kratos.yml | 24 ++++---- package-lock.json | 4 +- quadratic-api/.env.docker | 2 +- quadratic-api/.env.example | 2 +- quadratic-api/.env.test | 2 +- quadratic-client/package.json | 1 + quadratic-connection/.env.docker | 2 +- quadratic-files/.env.docker | 2 +- quadratic-multiplayer/.env.docker | 2 +- self-hosting/.gitignore | 9 +-- self-hosting/docker-compose.yml | 48 ++++++++++++++-- self-hosting/docker/caddy/config/Caddyfile | 22 +++++-- .../docker/client/config/default.conf | 17 ++++++ .../docker/client/scripts/replace_env_vars.sh | 26 +++++++++ .../docker/ory-auth/config/kratos.yml | 29 +++++----- self-hosting/start.sh | 14 ++--- 17 files changed, 194 insertions(+), 69 deletions(-) create mode 100644 self-hosting/docker/client/config/default.conf create mode 100755 self-hosting/docker/client/scripts/replace_env_vars.sh diff --git a/client.Dockerfile b/client.Dockerfile index 834914b14a..604bb7c3e7 100644 --- a/client.Dockerfile +++ b/client.Dockerfile @@ -8,33 +8,45 @@ ENV PATH="/root/.cargo/bin:${PATH}" # Install wasm-pack RUN echo 'Installing wasm-pack...' && curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh -# Install python -RUN apt-get update || : && apt-get install python-is-python3 -y && apt install python3-pip -y +# Install wasm32-unknown-unknown target +RUN rustup target add wasm32-unknown-unknown -# Install binaryen -RUN apt install binaryen -y +# Install python, binaryen & clean up +RUN apt-get update && apt-get install -y python-is-python3 python3-pip binaryen && apt-get clean && rm -rf /var/lib/apt/lists/* -# Copy the rest of the application code +# Install npm dependencies WORKDIR /app - COPY package.json . COPY package-lock.json . +COPY ./quadratic-kernels/python-wasm/package*.json ./quadratic-kernels/python-wasm/ +COPY ./quadratic-core/package*.json ./quadratic-core/ +COPY ./quadratic-rust-client/package*.json ./quadratic-rust-client/ +COPY ./quadratic-shared/package*.json ./quadratic-shared/ +COPY ./quadratic-client/package*.json ./quadratic-client/ +RUN npm install + +# Install typescript +RUN npm install -D typescript + +# Copy the rest of the application +WORKDIR /app COPY updateAlertVersion.json . -COPY ./quadratic-client/. ./quadratic-client/ -COPY ./quadratic-core/. ./quadratic-core/ COPY ./quadratic-kernels/python-wasm/. ./quadratic-kernels/python-wasm/ +COPY ./quadratic-core/. ./quadratic-core/ COPY ./quadratic-rust-client/. ./quadratic-rust-client/ COPY ./quadratic-shared/. ./quadratic-shared/ +COPY ./quadratic-client/. ./quadratic-client/ # Run the packaging script for quadratic_py +WORKDIR /app RUN ./quadratic-kernels/python-wasm/package.sh --no-poetry # Build wasm WORKDIR /app/quadratic-core -RUN rustup target add wasm32-unknown-unknown -RUN echo 'Building wasm...' && wasm-pack build --target web --out-dir ../quadratic-client/src/app/quadratic-core --weak-refs +RUN echo 'Building wasm...' && wasm-pack build --target web --out-dir ../quadratic-client/src/app/quadratic-core --weak-refs # Export TS/Rust types +WORKDIR /app/quadratic-core RUN echo 'Exporting TS/Rust types...' && cargo run --bin export_types # Build the quadratic-rust-client @@ -43,12 +55,20 @@ ARG GIT_COMMIT ENV GIT_COMMIT=$GIT_COMMIT RUN echo 'Building quadratic-rust-client...' && npm run build --workspace=quadratic-rust-client +# Build the quadratic-shared +WORKDIR /app +RUN echo 'Building quadratic-shared...' && npx tsc ./quadratic-shared/*.ts + # Build the front-end WORKDIR /app RUN echo 'Building front-end...' -RUN npm ci -RUN npm install typescript -RUN npx tsc ./quadratic-shared/*.ts +ENV VITE_DEBUG=0 +ENV VITE_QUADRATIC_API_URL=http://localhost:8000 +ENV VITE_QUADRATIC_MULTIPLAYER_URL=ws://localhost:3001/ws +ENV VITE_QUADRATIC_CONNECTION_URL=http://localhost:3003 +ENV VITE_AUTH_TYPE=ory +ENV VITE_ORY_HOST=http://localhost:4433 +ENV VITE_ENVIRONMENT=production RUN npm run build --workspace=quadratic-client # The default command to run the application @@ -57,6 +77,13 @@ RUN npm run build --workspace=quadratic-client FROM nginx:stable-alpine COPY --from=build /app/build /usr/share/nginx/html -EXPOSE 3000 +EXPOSE 80 443 3000 + +CMD ["nginx", "-g", "daemon off;"] + + + + + + -CMD ["nginx", "-g", "daemon off;"] \ No newline at end of file diff --git a/docker/ory-auth/config/kratos.yml b/docker/ory-auth/config/kratos.yml index ec26ebb261..6bcc17d1bb 100644 --- a/docker/ory-auth/config/kratos.yml +++ b/docker/ory-auth/config/kratos.yml @@ -5,10 +5,11 @@ dsn: memory serve: public: - base_url: http://localhost:4433/ + base_url: https://kratos.localhost/ cors: enabled: true allowed_origins: + - https://localhost - http://localhost:3000 allowed_methods: - POST @@ -28,9 +29,12 @@ serve: base_url: http://kratos:4434/ selfservice: - default_browser_return_url: http://localhost:3000 + default_browser_return_url: https://localhost allowed_return_urls: - - http://localhost:4455 + - http://localhost + - https://localhost + - https://kratos.localhost + - https://kratos-client.localhost - http://localhost:3000 - http://localhost:19006/Callback - exp://localhost:8081/--/Callback @@ -51,36 +55,36 @@ selfservice: flows: error: - ui_url: http://localhost:4455/error + ui_url: https://kratos-client.localhost/error settings: - ui_url: http://localhost:4455/settings + ui_url: https://kratos-client.localhost/settings privileged_session_max_age: 15m required_aal: highest_available recovery: enabled: true - ui_url: http://localhost:4455/recovery + ui_url: https://kratos-client.localhost/recovery use: code verification: enabled: true - ui_url: http://localhost:4455/verification + ui_url: https://kratos-client.localhost/verification use: code after: default_browser_return_url: http://localhost:3000 logout: after: - default_browser_return_url: http://localhost:4455/login + default_browser_return_url: https://kratos-client.localhost/login login: - ui_url: http://localhost:4455/login + ui_url: https://kratos-client.localhost/login lifespan: 10m registration: lifespan: 10m - ui_url: http://localhost:4455/registration + ui_url: https://kratos-client.localhost/registration after: password: hooks: diff --git a/package-lock.json b/package-lock.json index 8cda13357f..6f80d80f89 100644 --- a/package-lock.json +++ b/package-lock.json @@ -14341,7 +14341,8 @@ }, "node_modules/events": { "version": "3.3.0", - "license": "MIT", + "resolved": "https://registry.npmjs.org/events/-/events-3.3.0.tgz", + "integrity": "sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==", "engines": { "node": ">=0.8.x" } @@ -26550,6 +26551,7 @@ "color": "^4.2.3", "esbuild-wasm": "^0.20.2", "eventemitter3": "^5.0.1", + "events": "^3.3.0", "fontfaceobserver": "^2.3.0", "fuzzysort": "^2.0.4", "localforage": "^1.10.0", diff --git a/quadratic-api/.env.docker b/quadratic-api/.env.docker index 47bcf275b8..dfad586888 100644 --- a/quadratic-api/.env.docker +++ b/quadratic-api/.env.docker @@ -11,7 +11,7 @@ ENCRYPTION_KEY=eb4758047f74bdb2603cce75c4370327ca2c3662c4786867659126da8e64dfcc # Auth AUTH_TYPE=ory -ORY_JWKS_URI="http://host.docker.internal/.well-known/jwks.json" +ORY_JWKS_URI="http://host.docker.internal:3000/.well-known/jwks.json" ORY_ADMIN_HOST=http://host.docker.internal:4434 # Storage diff --git a/quadratic-api/.env.example b/quadratic-api/.env.example index 4e0642d415..da8c07f9d8 100644 --- a/quadratic-api/.env.example +++ b/quadratic-api/.env.example @@ -23,7 +23,7 @@ AUTH0_DOMAIN=quadratic-community.us.auth0.com AUTH0_CLIENT_ID=DCPCvqyU5Q0bJD8Q3QmJEoV48x1zLH7W AUTH0_CLIENT_SECRET=94dp3PDcxlI9ZDqBSvkdjQHWgGdx0ZSeyTr5-Rn3Kcts-ZyTdj1FLlJjCyqrTXEG AUTH0_AUDIENCE=community-quadratic -ORY_JWKS_URI='http://localhost:3000/.well-known/jwks.json' +ORY_JWKS_URI='http://host.docker.internal:3000/.well-known/jwks.json' ORY_ADMIN_HOST=http://0.0.0.0:4434 # Storage diff --git a/quadratic-api/.env.test b/quadratic-api/.env.test index a59f83027e..edc931a607 100644 --- a/quadratic-api/.env.test +++ b/quadratic-api/.env.test @@ -13,7 +13,7 @@ AUTH0_ISSUER='https://auth-dev.quadratic.to/' AUTH0_CLIENT_ID="AUTH0_CLIENT_ID" AUTH0_CLIENT_SECRET="AUTH0_CLIENT_SECRET" AUTH0_DOMAIN="AUTH0_DOMAIN" -ORY_JWKS_URI='http://localhost:3000/.well-known/jwks.json' +ORY_JWKS_URI='http://host.docker.internal:3000/.well-known/jwks.json' ORY_ADMIN_HOST=http://0.0.0.0:4434 # Storage diff --git a/quadratic-client/package.json b/quadratic-client/package.json index 5b606f4a4c..e2c58f53b0 100644 --- a/quadratic-client/package.json +++ b/quadratic-client/package.json @@ -54,6 +54,7 @@ "color": "^4.2.3", "esbuild-wasm": "^0.20.2", "eventemitter3": "^5.0.1", + "events": "^3.3.0", "fontfaceobserver": "^2.3.0", "fuzzysort": "^2.0.4", "localforage": "^1.10.0", diff --git a/quadratic-connection/.env.docker b/quadratic-connection/.env.docker index e47987bb11..522914b032 100644 --- a/quadratic-connection/.env.docker +++ b/quadratic-connection/.env.docker @@ -2,7 +2,7 @@ HOST=0.0.0.0 PORT=3003 ENVIRONMENT=docker -AUTH0_JWKS_URI=http://host.docker.internal/.well-known/jwks.json +AUTH0_JWKS_URI=http://host.docker.internal:3000/.well-known/jwks.json QUADRATIC_API_URI=http://host.docker.internal:8000 M2M_AUTH_TOKEN=M2M_AUTH_TOKEN MAX_RESPONSE_BYTES=15728640 # 15MB diff --git a/quadratic-files/.env.docker b/quadratic-files/.env.docker index ee04e9e338..1691e5bfd9 100644 --- a/quadratic-files/.env.docker +++ b/quadratic-files/.env.docker @@ -6,7 +6,7 @@ TRUNCATE_FILE_CHECK_S=60 TRUNCATE_TRANSACTION_AGE_DAYS=5 # ENVIRONMENT=docker -AUTH0_JWKS_URI=http://host.docker.internal/.well-known/jwks.json +AUTH0_JWKS_URI=http://host.docker.internal:3000/.well-known/jwks.json QUADRATIC_API_URI=http://host.docker.internal:8000 M2M_AUTH_TOKEN=M2M_AUTH_TOKEN diff --git a/quadratic-multiplayer/.env.docker b/quadratic-multiplayer/.env.docker index 97e42c0c50..a6dfb7fa0f 100644 --- a/quadratic-multiplayer/.env.docker +++ b/quadratic-multiplayer/.env.docker @@ -11,5 +11,5 @@ PUBSUB_PORT=6379 PUBSUB_PASSWORD= PUBSUB_ACTIVE_CHANNELS=active_channels -AUTH0_JWKS_URI=http://host.docker.internal/.well-known/jwks.json +AUTH0_JWKS_URI=http://host.docker.internal:3000/.well-known/jwks.json AUTHENTICATE_JWT=true \ No newline at end of file diff --git a/self-hosting/.gitignore b/self-hosting/.gitignore index 1004414824..b4c7785e98 100644 --- a/self-hosting/.gitignore +++ b/self-hosting/.gitignore @@ -1,9 +1,6 @@ +docker/caddy/quadratic-client +docker/file-storage docker/localstack/data docker/mysql/data docker/postgres/data -docker/redis/data -docker/static/html -docker/postgres-connection/data -docker/mysql-connection/data -docker/file-storage -docker/caddy/quadratic-client \ No newline at end of file +docker/redis/data \ No newline at end of file diff --git a/self-hosting/docker-compose.yml b/self-hosting/docker-compose.yml index b1e27233be..6120546f88 100644 --- a/self-hosting/docker-compose.yml +++ b/self-hosting/docker-compose.yml @@ -42,13 +42,39 @@ services: container_name: caddy ports: - "80:80" + - "443:443" volumes: - ./docker/caddy/config/Caddyfile:/etc/caddy/Caddyfile - - ./docker/caddy/quadratic-client:/srv - depends_on: - - frontend + # - ./docker/caddy/quadratic-client/html:/srv profiles: - caddy + - frontend + networks: + - host + + quadratic-client: + image: quadratic_quadratic-client:latest + env_file: + - ./docker/client/config/.env + restart: "always" + ports: + - "3000:80" + command: > + sh -c "/client/scripts/replace_env_vars.sh && + nginx -g \"daemon off;\"" + healthcheck: + test: ["CMD-SHELL", "curl -f http://host.docker.internal:3000/ || exit 1"] + interval: 10s + timeout: 5s + volumes: + - ./docker/client:/client + - ./docker/client/config/default.conf:/etc/nginx/conf.d/default.conf + # - ./docker/client/build:/usr/share/nginx/html + profiles: + - client + - frontend + networks: + - host quadratic-api: image: quadratic_quadratic-api @@ -64,6 +90,8 @@ services: profiles: - api - frontend + networks: + - host quadratic-multiplayer: image: quadratic-multiplayer @@ -82,6 +110,8 @@ services: condition: service_healthy quadratic-api: condition: service_started + quadratic-client: + condition: service_healthy profiles: - backend - multiplayer @@ -105,6 +135,8 @@ services: condition: service_healthy quadratic-api: condition: service_started + quadratic-client: + condition: service_healthy volumes: - ./docker/file-storage:/file-storage profiles: @@ -126,6 +158,8 @@ services: depends_on: caddy: condition: service_started + quadratic-client: + condition: service_healthy # postgres: # condition: service_healthy # quadratic-api: @@ -133,6 +167,8 @@ services: profiles: - backend - connection + networks: + - host # Auth Providers @@ -145,7 +181,7 @@ services: volumes: - ./docker/ory-auth/config:/etc/config/kratos environment: - DSN: postgresql://postgres:postgres@host.docker.internal:5432/kratos?sslmode=disable + DSN: postgresql://postgres:postgres@host.docker.internal:5432/kratos LOG_LEVEL: trace restart: unless-stopped depends_on: @@ -162,7 +198,7 @@ services: volumes: - ./docker/ory-auth/config:/etc/config/kratos environment: - DSN: postgresql://postgres:postgres@host.docker.internal:5432/kratos?sslmode=disable + DSN: postgresql://postgres:postgres@host.docker.internal:5432/kratos restart: on-failure depends_on: - postgres @@ -179,7 +215,7 @@ services: PORT: 4455 SECURITY_MODE: KRATOS_PUBLIC_URL: http://host.docker.internal:4433/ - KRATOS_BROWSER_URL: http://localhost:4433/ + KRATOS_BROWSER_URL: https://kratos.localhost/ COOKIE_SECRET: changeme CSRF_COOKIE_NAME: ory_csrf_ui CSRF_COOKIE_SECRET: changeme diff --git a/self-hosting/docker/caddy/config/Caddyfile b/self-hosting/docker/caddy/config/Caddyfile index 5e94cef443..b6adc6d17f 100644 --- a/self-hosting/docker/caddy/config/Caddyfile +++ b/self-hosting/docker/caddy/config/Caddyfile @@ -1,5 +1,19 @@ -:80 { - root * /srv - try_files {path} /index.html - file_server +localhost { + tls internal + reverse_proxy http://host.docker.internal:3000 +} + +api.localhost { + tls internal + reverse_proxy http://host.docker.internal:8000 +} + +kratos.localhost { + tls internal + reverse_proxy http://host.docker.internal:4433 +} + +kratos-client.localhost { + tls internal + reverse_proxy http://host.docker.internal:4455 } \ No newline at end of file diff --git a/self-hosting/docker/client/config/default.conf b/self-hosting/docker/client/config/default.conf new file mode 100644 index 0000000000..1d92083399 --- /dev/null +++ b/self-hosting/docker/client/config/default.conf @@ -0,0 +1,17 @@ +server { + listen 80; + server_name localhost; + + root /usr/share/nginx/html; + index index.html; + + location / { + try_files $uri $uri/ /index.html =404; + } + + location ~* \.(?:css|js|json|gif|png|jpg|jpeg|svg|ico)$ { + expires 1y; + access_log off; + add_header Cache-Control "public, no-transform"; + } +} \ No newline at end of file diff --git a/self-hosting/docker/client/scripts/replace_env_vars.sh b/self-hosting/docker/client/scripts/replace_env_vars.sh new file mode 100755 index 0000000000..c8cac40de8 --- /dev/null +++ b/self-hosting/docker/client/scripts/replace_env_vars.sh @@ -0,0 +1,26 @@ +#!/bin/sh + +ENV_PATH="/client/config/.env" + +escape_for_sed() { + input="$1" + printf '%s\n' "$input" | sed -e 's/[\/&]/\\&/g' +} + +replace_env_vars() { + ENV_VARS=$(cat $ENV_PATH) + + find "/usr/share/nginx/html/assets" -type f -name "*.js" | xargs grep -l "VITE_" | while read file; do + + echo "$ENV_VARS" | while read env_var; do + var=$(echo "$env_var" | cut -d'=' -f1) + val=$(echo "$env_var" | cut -d'=' -f2-) + escaped_val=$(escape_for_sed "$val") + + # echo "Replacing $var with $val in $file" + sed -i "s/\($var:\"\)[^\"]*\"/\1$(echo "$escaped_val")\"/g" "$file" + done + done +} + +echo "Replacing .env values in $ENV_PATH" diff --git a/self-hosting/docker/ory-auth/config/kratos.yml b/self-hosting/docker/ory-auth/config/kratos.yml index cf113a485d..8226b03200 100644 --- a/self-hosting/docker/ory-auth/config/kratos.yml +++ b/self-hosting/docker/ory-auth/config/kratos.yml @@ -5,11 +5,11 @@ dsn: memory serve: public: - base_url: http://localhost:4433/ + base_url: https://kratos.localhost/ cors: enabled: true allowed_origins: - - http://localhost + - https://localhost - http://localhost:3000 allowed_methods: - POST @@ -31,7 +31,7 @@ serve: selfservice: default_browser_return_url: http://localhost allowed_return_urls: - - http://localhost:4455 + - https://kratos-client.localhost - http://localhost:3000 - http://localhost:19006/Callback - exp://localhost:8081/--/Callback @@ -52,36 +52,36 @@ selfservice: flows: error: - ui_url: http://localhost:4455/error + ui_url: https://kratos-client.localhost/error settings: - ui_url: http://localhost:4455/settings + ui_url: https://kratos-client.localhost/settings privileged_session_max_age: 15m required_aal: highest_available recovery: enabled: true - ui_url: http://localhost:4455/recovery + ui_url: https://kratos-client.localhost/recovery use: code verification: enabled: true - ui_url: http://localhost:4455/verification + ui_url: https://kratos-client.localhost/verification use: code after: default_browser_return_url: http://localhost logout: after: - default_browser_return_url: http://localhost:4455/login + default_browser_return_url: https://kratos-client.localhost/login login: - ui_url: http://localhost:4455/login + ui_url: https://kratos-client.localhost/login lifespan: 10m registration: lifespan: 10m - ui_url: http://localhost:4455/registration + ui_url: https://kratos-client.localhost/registration after: password: hooks: @@ -93,14 +93,15 @@ session: tokenizer: templates: jwt_template: - jwks_url: http://host.docker.internal/.well-known/jwks.json + jwks_url: http://host.docker.internal:3000/.well-known/jwks.json # claims_mapper_url: base64://... # A JsonNet template for modifying the claims ttl: 24h # 24 hours (defaults to 10 minutes) log: - level: debug - format: text - leak_sensitive_values: true + level: warning + format: json + redaction_text: "" + leak_sensitive_values: false secrets: cookie: diff --git a/self-hosting/start.sh b/self-hosting/start.sh index 3ed4813e40..1b47357b3d 100755 --- a/self-hosting/start.sh +++ b/self-hosting/start.sh @@ -8,7 +8,7 @@ escape_for_sed() { replace_env_vars() { TEMP=$'\r\n' GLOBIGNORE='*' command eval 'ENV_VARS=($(cat .env))' - find "docker/caddy/quadratic-client" -type f -name "*.js" | while read file; do + find "/usr/share/nginx/html" -type f -name "*.js" | while read file; do echo "Replacing values in $file" for env_var in "${ENV_VARS[@]}"; do @@ -23,10 +23,10 @@ replace_env_vars() { done } -cd .. -rm -rf self-hosting/docker/caddy/quadratic-client/* -npm run build --workspace=quadratic-client -cp -r quadratic-client/build self-hosting/docker/caddy/quadratic-client +# cd .. +# rm -rf self-hosting/docker/caddy/quadratic-client/* +# npm run build --workspace=quadratic-client +# cp -r quadratic-client/build self-hosting/docker/caddy/quadratic-client -cd self-hosting -# replace_env_vars \ No newline at end of file +# cd self-hosting +replace_env_vars \ No newline at end of file From 3f8976ab0bca14b2fac8dff25b08ec80503ad1fb Mon Sep 17 00:00:00 2001 From: David DiMaria Date: Wed, 21 Aug 2024 12:02:55 -0600 Subject: [PATCH 022/113] Route the client in self-hosting through caddy --- client.Dockerfile | 13 +++++---- docker/ory-auth/config/kratos.yml | 23 +++++++--------- self-hosting/.gitignore | 2 +- self-hosting/docker-compose.yml | 10 +++---- self-hosting/docker/caddy/config/Caddyfile | 18 +------------ .../docker/client/scripts/replace_env_vars.sh | 11 +++++--- .../docker/ory-auth/config/kratos.yml | 27 +++++++++++-------- 7 files changed, 46 insertions(+), 58 deletions(-) diff --git a/client.Dockerfile b/client.Dockerfile index 604bb7c3e7..dc4369d120 100644 --- a/client.Dockerfile +++ b/client.Dockerfile @@ -62,13 +62,12 @@ RUN echo 'Building quadratic-shared...' && npx tsc ./quadratic-shared/*.ts # Build the front-end WORKDIR /app RUN echo 'Building front-end...' -ENV VITE_DEBUG=0 -ENV VITE_QUADRATIC_API_URL=http://localhost:8000 -ENV VITE_QUADRATIC_MULTIPLAYER_URL=ws://localhost:3001/ws -ENV VITE_QUADRATIC_CONNECTION_URL=http://localhost:3003 -ENV VITE_AUTH_TYPE=ory -ENV VITE_ORY_HOST=http://localhost:4433 -ENV VITE_ENVIRONMENT=production +ENV VITE_DEBUG=VITE_DEBUG_VAL +ENV VITE_QUADRATIC_API_URL=VITE_QUADRATIC_API_URL_VAL +ENV VITE_QUADRATIC_MULTIPLAYER_URL=VITE_QUADRATIC_MULTIPLAYER_URL_VAL +ENV VITE_QUADRATIC_CONNECTION_URL=VITE_QUADRATIC_CONNECTION_URL_VAL +ENV VITE_AUTH_TYPE=VITE_AUTH_TYPE_VAL +ENV VITE_ORY_HOST=VITE_ORY_HOST_VAL RUN npm run build --workspace=quadratic-client # The default command to run the application diff --git a/docker/ory-auth/config/kratos.yml b/docker/ory-auth/config/kratos.yml index 6bcc17d1bb..347b900d1b 100644 --- a/docker/ory-auth/config/kratos.yml +++ b/docker/ory-auth/config/kratos.yml @@ -5,11 +5,10 @@ dsn: memory serve: public: - base_url: https://kratos.localhost/ + base_url: http://localhost:4433/ cors: enabled: true allowed_origins: - - https://localhost - http://localhost:3000 allowed_methods: - POST @@ -29,12 +28,10 @@ serve: base_url: http://kratos:4434/ selfservice: - default_browser_return_url: https://localhost + default_browser_return_url: http://localhost:3000 allowed_return_urls: - http://localhost - - https://localhost - - https://kratos.localhost - - https://kratos-client.localhost + - http://localhost:4455 - http://localhost:3000 - http://localhost:19006/Callback - exp://localhost:8081/--/Callback @@ -55,36 +52,36 @@ selfservice: flows: error: - ui_url: https://kratos-client.localhost/error + ui_url: http://localhost:4455/error settings: - ui_url: https://kratos-client.localhost/settings + ui_url: http://localhost:4455/settings privileged_session_max_age: 15m required_aal: highest_available recovery: enabled: true - ui_url: https://kratos-client.localhost/recovery + ui_url: http://localhost:4455/recovery use: code verification: enabled: true - ui_url: https://kratos-client.localhost/verification + ui_url: http://localhost:4455/verification use: code after: default_browser_return_url: http://localhost:3000 logout: after: - default_browser_return_url: https://kratos-client.localhost/login + default_browser_return_url: http://localhost:4455/login login: - ui_url: https://kratos-client.localhost/login + ui_url: http://localhost:4455/login lifespan: 10m registration: lifespan: 10m - ui_url: https://kratos-client.localhost/registration + ui_url: http://localhost:4455/registration after: password: hooks: diff --git a/self-hosting/.gitignore b/self-hosting/.gitignore index b4c7785e98..74d7674042 100644 --- a/self-hosting/.gitignore +++ b/self-hosting/.gitignore @@ -1,4 +1,4 @@ -docker/caddy/quadratic-client +docker/caddy/certs docker/file-storage docker/localstack/data docker/mysql/data diff --git a/self-hosting/docker-compose.yml b/self-hosting/docker-compose.yml index 6120546f88..8480d3ac36 100644 --- a/self-hosting/docker-compose.yml +++ b/self-hosting/docker-compose.yml @@ -45,6 +45,7 @@ services: - "443:443" volumes: - ./docker/caddy/config/Caddyfile:/etc/caddy/Caddyfile + - ./docker/caddy/certs:/data/caddy/pki/authorities/local # - ./docker/caddy/quadratic-client/html:/srv profiles: - caddy @@ -181,7 +182,7 @@ services: volumes: - ./docker/ory-auth/config:/etc/config/kratos environment: - DSN: postgresql://postgres:postgres@host.docker.internal:5432/kratos + DSN: postgresql://postgres:postgres@host.docker.internal:5432/kratos?sslmode=disable LOG_LEVEL: trace restart: unless-stopped depends_on: @@ -198,7 +199,7 @@ services: volumes: - ./docker/ory-auth/config:/etc/config/kratos environment: - DSN: postgresql://postgres:postgres@host.docker.internal:5432/kratos + DSN: postgresql://postgres:postgres@host.docker.internal:5432/kratos?sslmode=disable restart: on-failure depends_on: - postgres @@ -213,11 +214,10 @@ services: - "4455:4455" environment: PORT: 4455 - SECURITY_MODE: KRATOS_PUBLIC_URL: http://host.docker.internal:4433/ - KRATOS_BROWSER_URL: https://kratos.localhost/ + KRATOS_BROWSER_URL: http://localhost:4433/ COOKIE_SECRET: changeme - CSRF_COOKIE_NAME: ory_csrf_ui + CSRF_COOKIE_NAME: __HOST-localhost-x-csrf-token CSRF_COOKIE_SECRET: changeme restart: on-failure profiles: diff --git a/self-hosting/docker/caddy/config/Caddyfile b/self-hosting/docker/caddy/config/Caddyfile index b6adc6d17f..979148ec01 100644 --- a/self-hosting/docker/caddy/config/Caddyfile +++ b/self-hosting/docker/caddy/config/Caddyfile @@ -1,19 +1,3 @@ -localhost { - tls internal +:80 { reverse_proxy http://host.docker.internal:3000 } - -api.localhost { - tls internal - reverse_proxy http://host.docker.internal:8000 -} - -kratos.localhost { - tls internal - reverse_proxy http://host.docker.internal:4433 -} - -kratos-client.localhost { - tls internal - reverse_proxy http://host.docker.internal:4455 -} \ No newline at end of file diff --git a/self-hosting/docker/client/scripts/replace_env_vars.sh b/self-hosting/docker/client/scripts/replace_env_vars.sh index c8cac40de8..21cf3057e7 100755 --- a/self-hosting/docker/client/scripts/replace_env_vars.sh +++ b/self-hosting/docker/client/scripts/replace_env_vars.sh @@ -13,14 +13,17 @@ replace_env_vars() { find "/usr/share/nginx/html/assets" -type f -name "*.js" | xargs grep -l "VITE_" | while read file; do echo "$ENV_VARS" | while read env_var; do - var=$(echo "$env_var" | cut -d'=' -f1) - val=$(echo "$env_var" | cut -d'=' -f2-) + echo "env_var: $env_var" + var="$(echo "$env_var" | cut -d'=' -f1)" + val="$(echo "$env_var" | cut -d'=' -f2-)" + appended_var="${var}_VAL" escaped_val=$(escape_for_sed "$val") - # echo "Replacing $var with $val in $file" - sed -i "s/\($var:\"\)[^\"]*\"/\1$(echo "$escaped_val")\"/g" "$file" + echo "Replacing $appended_var with $escaped_val in $file" + sed -i "s/${appended_var}/${escaped_val}/g" "$file" done done } echo "Replacing .env values in $ENV_PATH" +replace_env_vars diff --git a/self-hosting/docker/ory-auth/config/kratos.yml b/self-hosting/docker/ory-auth/config/kratos.yml index 8226b03200..06ab817464 100644 --- a/self-hosting/docker/ory-auth/config/kratos.yml +++ b/self-hosting/docker/ory-auth/config/kratos.yml @@ -5,11 +5,11 @@ dsn: memory serve: public: - base_url: https://kratos.localhost/ + base_url: http://localhost:4433/ cors: enabled: true allowed_origins: - - https://localhost + - http://localhost - http://localhost:3000 allowed_methods: - POST @@ -31,7 +31,8 @@ serve: selfservice: default_browser_return_url: http://localhost allowed_return_urls: - - https://kratos-client.localhost + - http://localhost + - http://localhost:4455 - http://localhost:3000 - http://localhost:19006/Callback - exp://localhost:8081/--/Callback @@ -52,36 +53,36 @@ selfservice: flows: error: - ui_url: https://kratos-client.localhost/error + ui_url: http://localhost:4455/error settings: - ui_url: https://kratos-client.localhost/settings + ui_url: http://localhost:4455/settings privileged_session_max_age: 15m required_aal: highest_available recovery: enabled: true - ui_url: https://kratos-client.localhost/recovery + ui_url: http://localhost:4455/recovery use: code verification: enabled: true - ui_url: https://kratos-client.localhost/verification + ui_url: http://localhost:4455/verification use: code after: default_browser_return_url: http://localhost logout: after: - default_browser_return_url: https://kratos-client.localhost/login + default_browser_return_url: http://localhost:4455/login login: - ui_url: https://kratos-client.localhost/login + ui_url: http://localhost:4455/login lifespan: 10m registration: lifespan: 10m - ui_url: https://kratos-client.localhost/registration + ui_url: http://localhost:4455/registration after: password: hooks: @@ -96,7 +97,11 @@ session: jwks_url: http://host.docker.internal:3000/.well-known/jwks.json # claims_mapper_url: base64://... # A JsonNet template for modifying the claims ttl: 24h # 24 hours (defaults to 10 minutes) - +cookies: + domain: localhost + path: / + same_site: Lax + log: level: warning format: json From 1d1d43c2cf9188e088d98663c539b67d60c2d362 Mon Sep 17 00:00:00 2001 From: David DiMaria Date: Wed, 21 Aug 2024 12:33:11 -0600 Subject: [PATCH 023/113] Embed env vars in docker-compose --- self-hosting/docker-compose.yml | 101 +++++++++++++++--- .../docker/client/scripts/replace_env_vars.sh | 15 ++- self-hosting/start.sh | 33 +----- 3 files changed, 97 insertions(+), 52 deletions(-) diff --git a/self-hosting/docker-compose.yml b/self-hosting/docker-compose.yml index 8480d3ac36..8ed7a9295b 100644 --- a/self-hosting/docker-compose.yml +++ b/self-hosting/docker-compose.yml @@ -55,9 +55,14 @@ services: quadratic-client: image: quadratic_quadratic-client:latest - env_file: - - ./docker/client/config/.env restart: "always" + environment: + VITE_DEBUG: 1 + VITE_QUADRATIC_API_URL: http://localhost:8000 + VITE_QUADRATIC_MULTIPLAYER_URL: ws://localhost:3001/ws + VITE_QUADRATIC_CONNECTION_URL: http://localhost:3003 + VITE_AUTH_TYPE: ory + VITE_ORY_HOST: http://localhost:4433 ports: - "3000:80" command: > @@ -79,8 +84,27 @@ services: quadratic-api: image: quadratic_quadratic-api - env_file: - - ../quadratic-api/.env.docker + environment: + CORS: "*" + DATABASE_URL: "postgresql://postgres:postgres@host.docker.internal:5432/postgres" + ENVIRONMENT: docker + STRIPE_SECRET_KEY: STRIPE_SECRET_KEY + STRIPE_WEBHOOK_SECRET: STRIPE_WEBHOOK_SECRET + OPENAI_API_KEY: + M2M_AUTH_TOKEN: M2M_AUTH_TOKEN + + # Hex string to be used as the key for enctyption, use npm run key:generate + ENCRYPTION_KEY: eb4758047f74bdb2603cce75c4370327ca2c3662c4786867659126da8e64dfcc + + # Auth + AUTH_TYPE: ory + ORY_JWKS_URI: "http://host.docker.internal:3000/.well-known/jwks.json" + ORY_ADMIN_HOST: http://host.docker.internal:4434 + + # Storage + STORAGE_TYPE: file-system + QUADRATIC_FILE_URI: http://host.docker.internal:3002 + QUADRATIC_FILE_URI_PUBLIC: http://localhost:3002 restart: "always" ports: - "8000:8000" @@ -96,11 +120,23 @@ services: quadratic-multiplayer: image: quadratic-multiplayer - env_file: - - ../quadratic-multiplayer/.env.docker - # override env vars here environment: RUST_LOG: info + HOST: 0.0.0.0 + PORT: 3001 + HEARTBEAT_CHECK_S: 3 + HEARTBEAT_TIMEOUT_S: 600 + QUADRATIC_API_URI: http://host.docker.internal:8000 + M2M_AUTH_TOKEN: M2M_AUTH_TOKEN + ENVIRONMENT: docker + + PUBSUB_HOST: host.docker.internal + PUBSUB_PORT: 6379 + PUBSUB_PASSWORD: + PUBSUB_ACTIVE_CHANNELS: active_channels + + AUTH0_JWKS_URI: http://host.docker.internal:3000/.well-known/jwks.json + AUTHENTICATE_JWT: true restart: "always" ports: - "3001:3001" @@ -121,11 +157,39 @@ services: quadratic-files: image: quadratic-files - env_file: - - ../quadratic-files/.env.docker - # override env vars here environment: RUST_LOG: info + HOST: 0.0.0.0 + PORT: 3002 + FILE_CHECK_S: 5 + FILES_PER_CHECK: 1000 + TRUNCATE_FILE_CHECK_S: 60 + TRUNCATE_TRANSACTION_AGE_DAYS: 5 # + ENVIRONMENT: docker + + AUTH0_JWKS_URI: http://host.docker.internal:3000/.well-known/jwks.json + QUADRATIC_API_URI: http://host.docker.internal:8000 + M2M_AUTH_TOKEN: M2M_AUTH_TOKEN + + PUBSUB_HOST: host.docker.internal + PUBSUB_PORT: 6379 + PUBSUB_PASSWORD: + PUBSUB_ACTIVE_CHANNELS: active_channels + PUBSUB_PROCESSED_TRANSACTIONS_CHANNEL: processed_transactions + + # Storage + STORAGE_TYPE: file-system # s3 or file-system + + # Storage: s3 + AWS_S3_REGION: + AWS_S3_BUCKET_NAME: quadratic-api-docker + AWS_S3_ACCESS_KEY_ID: + AWS_S3_SECRET_ACCESS_KEY: + + # Storage: file-system + STORAGE_DIR: /file-storage + STORAGE_ENCRYPTION_KEYS: eb4758047f74bdb2603cce75c4370327ca2c3662c4786867659126da8e64dfcc + restart: "always" ports: - "3002:3002" @@ -148,11 +212,18 @@ services: quadratic-connection: image: quadratic-connection - env_file: - - ../quadratic-connection/.env.docker - # override env vars here environment: RUST_LOG: info + HOST: 0.0.0.0 + PORT: 3003 + ENVIRONMENT: docker + + AUTH0_JWKS_URI: http://host.docker.internal:3000/.well-known/jwks.json + QUADRATIC_API_URI: http://host.docker.internal:8000 + M2M_AUTH_TOKEN: M2M_AUTH_TOKEN + MAX_RESPONSE_BYTES: 15728640 # 15MB + STATIC_IPS: 0.0.0.0,127.0.0.1 + restart: "always" ports: - "3003:3003" @@ -161,10 +232,6 @@ services: condition: service_started quadratic-client: condition: service_healthy - # postgres: - # condition: service_healthy - # quadratic-api: - # condition: service_started profiles: - backend - connection diff --git a/self-hosting/docker/client/scripts/replace_env_vars.sh b/self-hosting/docker/client/scripts/replace_env_vars.sh index 21cf3057e7..fa268115f0 100755 --- a/self-hosting/docker/client/scripts/replace_env_vars.sh +++ b/self-hosting/docker/client/scripts/replace_env_vars.sh @@ -1,19 +1,24 @@ #!/bin/sh -ENV_PATH="/client/config/.env" - escape_for_sed() { input="$1" printf '%s\n' "$input" | sed -e 's/[\/&]/\\&/g' } replace_env_vars() { - ENV_VARS=$(cat $ENV_PATH) + vite_vars="" + + for env_var in $(env); do + case "$env_var" in + VITE_*) + vite_vars="$vite_vars $env_var" + ;; + esac + done find "/usr/share/nginx/html/assets" -type f -name "*.js" | xargs grep -l "VITE_" | while read file; do - echo "$ENV_VARS" | while read env_var; do - echo "env_var: $env_var" + for env_var in $vite_vars; do var="$(echo "$env_var" | cut -d'=' -f1)" val="$(echo "$env_var" | cut -d'=' -f2-)" appended_var="${var}_VAL" diff --git a/self-hosting/start.sh b/self-hosting/start.sh index 1b47357b3d..7382f41a11 100755 --- a/self-hosting/start.sh +++ b/self-hosting/start.sh @@ -1,32 +1,5 @@ #!/bin/sh -escape_for_sed() { - local input="$1" - printf '%s\n' "$input" | sed -e 's/[\/&]/\\&/g' -} - -replace_env_vars() { - TEMP=$'\r\n' GLOBIGNORE='*' command eval 'ENV_VARS=($(cat .env))' - - find "/usr/share/nginx/html" -type f -name "*.js" | while read file; do - echo "Replacing values in $file" - - for env_var in "${ENV_VARS[@]}"; do - var=${env_var%=*} - val=${env_var#*=} - escaped_val=$(escape_for_sed "$val") - - # echo "Replacing $var with ${val} in $file" - - sed -i '' "s/\($var:\"\)[^\"]*\"/\1$(echo "$escaped_val")\"/g" $file - done - done -} - -# cd .. -# rm -rf self-hosting/docker/caddy/quadratic-client/* -# npm run build --workspace=quadratic-client -# cp -r quadratic-client/build self-hosting/docker/caddy/quadratic-client - -# cd self-hosting -replace_env_vars \ No newline at end of file +docker compose --profile "*" down +yes | docker compose rm quadratic-client +docker compose --profile "*" up \ No newline at end of file From 298bc07d5635241eaac40688562fc3eaa9e7f8c8 Mon Sep 17 00:00:00 2001 From: David DiMaria Date: Wed, 21 Aug 2024 13:52:50 -0600 Subject: [PATCH 024/113] Create script that downloads self-hosting directory and starts docker --- .../docker/client/scripts/replace_env_vars.sh | 2 +- self-hosting/start.sh | 24 ++++++++++++++++--- 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/self-hosting/docker/client/scripts/replace_env_vars.sh b/self-hosting/docker/client/scripts/replace_env_vars.sh index fa268115f0..60160a8ccf 100755 --- a/self-hosting/docker/client/scripts/replace_env_vars.sh +++ b/self-hosting/docker/client/scripts/replace_env_vars.sh @@ -24,7 +24,7 @@ replace_env_vars() { appended_var="${var}_VAL" escaped_val=$(escape_for_sed "$val") - echo "Replacing $appended_var with $escaped_val in $file" + # echo "Replacing $appended_var with $escaped_val in $file" sed -i "s/${appended_var}/${escaped_val}/g" "$file" done done diff --git a/self-hosting/start.sh b/self-hosting/start.sh index 7382f41a11..780b414147 100755 --- a/self-hosting/start.sh +++ b/self-hosting/start.sh @@ -1,5 +1,23 @@ #!/bin/sh -docker compose --profile "*" down -yes | docker compose rm quadratic-client -docker compose --profile "*" up \ No newline at end of file +REPO="https://github.com/quadratichq/quadratic.git" +BRANCH="self-hosting-setup" +DIR="self-hosting" + +checkout() { + git clone -b $BRANCH --filter=blob:none --no-checkout --depth 1 --sparse $REPO + cd quadratic + git sparse-checkout set ${DIR}/ + git checkout + cd $DIR + +} + +start() { + docker compose --profile "*" down + yes | docker compose rm quadratic-client + docker compose --profile "*" up +} + +checkout +start \ No newline at end of file From 8584061565599149e65ad41d0a64474ef4f37c88 Mon Sep 17 00:00:00 2001 From: David DiMaria Date: Fri, 23 Aug 2024 17:12:51 -0600 Subject: [PATCH 025/113] Enforce self-hosting license --- quadratic-api/.env.docker | 6 ++- quadratic-api/.env.example | 4 ++ quadratic-api/.env.test | 4 +- quadratic-api/src/env-vars.ts | 6 ++- quadratic-api/src/licenseClient.ts | 21 ++++++++ .../src/routes/v0/teams.$uuid.GET.ts | 41 ++++++++++------ .../src/app/quadratic-core-types/index.d.ts | 2 +- .../src/shared/components/ShareDialog.tsx | 23 +++++++++ quadratic-shared/typesAndSchemas.ts | 8 ++++ self-hosting/docker-compose.yml | 16 +++++++ self-hosting/docker/admin/config/openapi.yaml | 48 +++++++++++++++++++ 11 files changed, 160 insertions(+), 19 deletions(-) create mode 100644 quadratic-api/src/licenseClient.ts create mode 100644 self-hosting/docker/admin/config/openapi.yaml diff --git a/quadratic-api/.env.docker b/quadratic-api/.env.docker index dfad586888..6f404d6180 100644 --- a/quadratic-api/.env.docker +++ b/quadratic-api/.env.docker @@ -17,4 +17,8 @@ ORY_ADMIN_HOST=http://host.docker.internal:4434 # Storage STORAGE_TYPE=file-system QUADRATIC_FILE_URI=http://host.docker.internal:3002 -QUADRATIC_FILE_URI_PUBLIC=http://localhost:3002 \ No newline at end of file +QUADRATIC_FILE_URI_PUBLIC=http://localhost:3002 + +# Admin +LICENSE_KEY=LICENSE_KEY +LICENSE_API_URI=https://selfhost.quadratic-preview.com diff --git a/quadratic-api/.env.example b/quadratic-api/.env.example index da8c07f9d8..9cff5fe189 100644 --- a/quadratic-api/.env.example +++ b/quadratic-api/.env.example @@ -34,3 +34,7 @@ AWS_S3_REGION=us-east-2 AWS_S3_ACCESS_KEY_ID=test AWS_S3_SECRET_ACCESS_KEY=test AWS_S3_BUCKET_NAME=quadratic-api-docker + +# Admin +LICENSE_KEY=LICENSE_KEY +LICENSE_API_URI=https://selfhost.quadratic-preview.com diff --git a/quadratic-api/.env.test b/quadratic-api/.env.test index edc931a607..426cfeefc7 100644 --- a/quadratic-api/.env.test +++ b/quadratic-api/.env.test @@ -26,4 +26,6 @@ AWS_S3_ACCESS_KEY_ID=AWS_S3_ACCESS_KEY_ID AWS_S3_SECRET_ACCESS_KEY=AWS_S3_SECRET_ACCESS_KEY AWS_S3_ENDPOINT=http://0.0.0.0:4566 - +# Admin +LICENSE_KEY=LICENSE_KEY +LICENSE_API_URI=https://selfhost.quadratic-preview.com diff --git a/quadratic-api/src/env-vars.ts b/quadratic-api/src/env-vars.ts index 401fe680e1..ecb2662239 100644 --- a/quadratic-api/src/env-vars.ts +++ b/quadratic-api/src/env-vars.ts @@ -30,7 +30,11 @@ export const STRIPE_SECRET_KEY = process.env.STRIPE_SECRET_KEY as string; export const ENCRYPTION_KEY = process.env.ENCRYPTION_KEY as string; export const STORAGE_TYPE = process.env.STORAGE_TYPE as string; export const AUTH_TYPE = process.env.AUTH_TYPE as string; -['STRIPE_SECRET_KEY', 'ENCRYPTION_KEY', 'STORAGE_TYPE', 'AUTH_TYPE'].forEach(ensureEnvVarExists); +export const LICENSE_KEY = process.env.LICENSE_KEY as string; +export const LICENSE_API_URI = process.env.LICENSE_API_URI as string; +['STRIPE_SECRET_KEY', 'ENCRYPTION_KEY', 'STORAGE_TYPE', 'AUTH_TYPE', 'LICENSE_KEY', 'LICENSE_API_URI'].forEach( + ensureEnvVarExists +); // Required in prod, optional locally export const M2M_AUTH_TOKEN = process.env.M2M_AUTH_TOKEN; diff --git a/quadratic-api/src/licenseClient.ts b/quadratic-api/src/licenseClient.ts new file mode 100644 index 0000000000..85b38507c8 --- /dev/null +++ b/quadratic-api/src/licenseClient.ts @@ -0,0 +1,21 @@ +import axios from 'axios'; +import { LicenseSchema } from 'quadratic-shared/typesAndSchemas'; +import z from 'zod'; +import { LICENSE_API_URI, LICENSE_KEY } from './env-vars'; + +type LicenseResponse = z.infer; + +export const licenseClient = { + license: { + post: async (seats: number): Promise => { + try { + const body = { stats: { seats } }; + const response = await axios.post(`${LICENSE_API_URI}/api/license/${LICENSE_KEY}`, body); + return LicenseSchema.parse(response.data) as LicenseResponse; + } catch (err) { + console.error('Failed to get the license info from the license service', err); + return null; + } + }, + }, +}; diff --git a/quadratic-api/src/routes/v0/teams.$uuid.GET.ts b/quadratic-api/src/routes/v0/teams.$uuid.GET.ts index 69439fa61a..375a4f2ba4 100644 --- a/quadratic-api/src/routes/v0/teams.$uuid.GET.ts +++ b/quadratic-api/src/routes/v0/teams.$uuid.GET.ts @@ -3,12 +3,14 @@ import { ApiTypes } from 'quadratic-shared/typesAndSchemas'; import { z } from 'zod'; import { getUsers } from '../../auth/auth'; import dbClient from '../../dbClient'; +import { licenseClient } from '../../licenseClient'; import { getTeam } from '../../middleware/getTeam'; import { userMiddleware } from '../../middleware/user'; import { validateAccessToken } from '../../middleware/validateAccessToken'; import { parseRequest } from '../../middleware/validateRequestSchema'; import { getPresignedFileUrl } from '../../storage/storage'; import { RequestWithUser } from '../../types/Request'; +import { ResponseError } from '../../types/Response'; import { getFilePermissions } from '../../utils/permissions'; export default [validateAccessToken, userMiddleware, handler]; @@ -19,7 +21,7 @@ const schema = z.object({ }), }); -async function handler(req: Request, res: Response) { +async function handler(req: Request, res: Response) { const { params: { uuid }, } = parseRequest(req, schema); @@ -88,6 +90,27 @@ async function handler(req: Request, res: Response user)); + // IDEA: (enhancement) we could put this in /sharing and just return the userCount + // then require the data for the team share modal to be a seaparte network request + const users = dbUsers + .filter(({ userId: id }) => auth0UsersById[id]) + .map(({ userId: id, role }) => { + const { email, name, picture } = auth0UsersById[id]; + return { + id, + email, + role, + name, + picture, + }; + }); + + const license = await licenseClient.license.post(users.length); + + if (!license) { + return res.status(500).json({ error: { message: 'Unable to retrieve license' } }); + } + // Get signed thumbnail URLs await Promise.all( dbFiles.map(async (file) => { @@ -112,20 +135,7 @@ async function handler(req: Request, res: Response auth0UsersById[id]) - .map(({ userId: id, role }) => { - const { email, name, picture } = auth0UsersById[id]; - return { - id, - email, - role, - name, - picture, - }; - }), + users, invites: dbInvites.map(({ email, role, id }) => ({ email, role, id })), files: dbFiles .filter((file) => !file.ownerUserId) @@ -170,6 +180,7 @@ async function handler(req: Request, res: Response user.role === 'OWNER').length; @@ -116,6 +117,28 @@ export function ShareTeamDialog({ data }: { data: ApiTypes['/v0/teams/:uuid.GET. /> )} + {license.status === 'exceeded' && ( +
+
+ Over the user limit! +
+ + You are over your user limit of {license.limits.seats}. Please contact Quadratic Support to increase this. + +
+ )} + + {license.status === 'revoked' && ( +
+
+ License Revoked! +
+ + Your license has been revoked. Please contact Quadratic Support to increase this. + +
+ )} + {users.map((user) => { const isLoggedInUser = userMakingRequest.id === user.id; const canDelete = isLoggedInUser diff --git a/quadratic-shared/typesAndSchemas.ts b/quadratic-shared/typesAndSchemas.ts index a816c8b083..1a0477ccaf 100644 --- a/quadratic-shared/typesAndSchemas.ts +++ b/quadratic-shared/typesAndSchemas.ts @@ -103,6 +103,13 @@ const TeamFilesSchema = z.array( }) ); +export const LicenseSchema = z.object({ + limits: z.object({ + seats: z.number(), + }), + status: z.string(), +}); + // Zod schemas for API endpoints export const ApiSchemas = { /** @@ -314,6 +321,7 @@ export const ApiSchemas = { filesPrivate: TeamFilesSchema, users: z.array(TeamUserSchema), invites: z.array(z.object({ email: emailSchema, role: UserTeamRoleSchema, id: z.number() })), + license: LicenseSchema, }), '/v0/teams/:uuid.PATCH.request': TeamSchema.pick({ name: true }), '/v0/teams/:uuid.PATCH.response': TeamSchema.pick({ name: true }), diff --git a/self-hosting/docker-compose.yml b/self-hosting/docker-compose.yml index 8ed7a9295b..2e8546b458 100644 --- a/self-hosting/docker-compose.yml +++ b/self-hosting/docker-compose.yml @@ -105,6 +105,10 @@ services: STORAGE_TYPE: file-system QUADRATIC_FILE_URI: http://host.docker.internal:3002 QUADRATIC_FILE_URI_PUBLIC: http://localhost:3002 + + # Admin + LICENSE_KEY: LICENSE_KEY + LICENSE_API_URI: http://localhost:4000 restart: "always" ports: - "8000:8000" @@ -304,6 +308,18 @@ services: networks: - host + admin: + image: stoplight/prism:latest + command: "mock -p 4000 -h 0.0.0.0 /tmp/openapi.yaml" + ports: + - 4000:4000 + volumes: + - ./docker/admin/config/openapi.yaml:/tmp/openapi.yaml:ro + profiles: + - admin + networks: + - host + volumes: docker: name: docker diff --git a/self-hosting/docker/admin/config/openapi.yaml b/self-hosting/docker/admin/config/openapi.yaml new file mode 100644 index 0000000000..ff6fd4fe92 --- /dev/null +++ b/self-hosting/docker/admin/config/openapi.yaml @@ -0,0 +1,48 @@ +openapi: 3.0.3 +info: + title: My API + version: '1.0' + x-logo: + url: '' +paths: + /license/{licenseKey}: + post: + tags: [] + operationId: license + parameters: + - name: licenseKey + in: path + required: true + deprecated: false + example: sdaf + schema: + type: string + x-last-modified: 1724376657454 + responses: + '200': + content: + application/json: + schema: + type: object + properties: + limits: + type: object + properties: + seats: + type: number + example: 5 + x-last-modified: 1724376729775 + description: '' + headers: {} + links: {} + x-last-modified: 1724376538356 +components: + securitySchemes: {} + schemas: {} + headers: {} + responses: {} + parameters: {} +tags: [] +servers: + - url: https://api.example.io +security: [] From 73909aa406521bc812bbce3cbc535b4f88c4e484 Mon Sep 17 00:00:00 2001 From: David DiMaria Date: Mon, 26 Aug 2024 14:13:13 -0600 Subject: [PATCH 026/113] Handle licenses --- package-lock.json | 17 +++++- quadratic-api/src/licenseClient.ts | 55 +++++++++++++++---- .../src/routes/v0/files.$uuid.GET.ts | 10 ++++ .../src/routes/v0/teams.$uuid.GET.ts | 10 ++-- quadratic-api/src/storage/fileSystem.ts | 18 +++--- quadratic-client/src/routes/_dashboard.tsx | 26 ++++++++- quadratic-client/src/routes/file.$uuid.tsx | 28 +++++++++- quadratic-client/src/shared/api/apiClient.ts | 26 ++++++++- .../src/shared/api/fetchFromApi.ts | 1 + .../src/shared/components/ShareDialog.tsx | 4 +- quadratic-client/src/shared/constants/urls.ts | 1 + quadratic-shared/package.json | 6 +- quadratic-shared/typesAndSchemas.ts | 3 +- 13 files changed, 168 insertions(+), 37 deletions(-) diff --git a/package-lock.json b/package-lock.json index 6f80d80f89..b0a2007398 100644 --- a/package-lock.json +++ b/package-lock.json @@ -26790,7 +26790,22 @@ }, "quadratic-shared": { "version": "1.0.0", - "license": "ISC" + "license": "ISC", + "dependencies": { + "typescript": "^5.5.4" + } + }, + "quadratic-shared/node_modules/typescript": { + "version": "5.5.4", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.5.4.tgz", + "integrity": "sha512-Mtq29sKDAEYP7aljRgtPOpTvOfbwRWlS6dPRzwjdE+C0R4brX/GUyhHSecbHMFLNBLcJIPt9nl9yG5TZ1weH+Q==", + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } } } } diff --git a/quadratic-api/src/licenseClient.ts b/quadratic-api/src/licenseClient.ts index 85b38507c8..9f5f0592b1 100644 --- a/quadratic-api/src/licenseClient.ts +++ b/quadratic-api/src/licenseClient.ts @@ -1,21 +1,54 @@ import axios from 'axios'; import { LicenseSchema } from 'quadratic-shared/typesAndSchemas'; import z from 'zod'; +import dbClient from './dbClient'; import { LICENSE_API_URI, LICENSE_KEY } from './env-vars'; type LicenseResponse = z.infer; +// const StatusEnum = LicenseSchema.shape.status; +// type StatusType = z.infer; + +let cachedResult: LicenseResponse | null = null; +let lastCheckedTime: number | null = null; +// const cacheDuration = 15 * 60 * 1000; // 15 minutes in milliseconds +const cacheDuration = 0; // TODO(ddimaria): remove export const licenseClient = { - license: { - post: async (seats: number): Promise => { - try { - const body = { stats: { seats } }; - const response = await axios.post(`${LICENSE_API_URI}/api/license/${LICENSE_KEY}`, body); - return LicenseSchema.parse(response.data) as LicenseResponse; - } catch (err) { - console.error('Failed to get the license info from the license service', err); - return null; - } - }, + post: async (seats: number): Promise => { + try { + const body = { stats: { seats } }; + const response = await axios.post(`${LICENSE_API_URI}/api/license/${LICENSE_KEY}`, body); + return LicenseSchema.parse(response.data) as LicenseResponse; + } catch (err) { + console.error('Failed to get the license info from the license service', err); + return null; + } + }, + checkFromServer: async (): Promise => { + const userCount = await dbClient.user.count(); + const license = await licenseClient.post(userCount); + + if (!license) { + return null; + } + + return license; + }, + check: async (): Promise => { + const currentTime = Date.now(); + + if (cachedResult && lastCheckedTime && currentTime - lastCheckedTime < cacheDuration) { + // Use cached result if within the cache duration + return cachedResult; + } + + // Otherwise, perform the check + const result = await licenseClient.checkFromServer(); + + // Cache the result and update the last checked time + cachedResult = result; + lastCheckedTime = currentTime; + + return result; }, }; diff --git a/quadratic-api/src/routes/v0/files.$uuid.GET.ts b/quadratic-api/src/routes/v0/files.$uuid.GET.ts index d7b9c514e4..c208bd98f4 100644 --- a/quadratic-api/src/routes/v0/files.$uuid.GET.ts +++ b/quadratic-api/src/routes/v0/files.$uuid.GET.ts @@ -2,6 +2,7 @@ import { Response } from 'express'; import { ApiTypes } from 'quadratic-shared/typesAndSchemas'; import z from 'zod'; import dbClient from '../../dbClient'; +import { licenseClient } from '../../licenseClient'; import { getFile } from '../../middleware/getFile'; import { userOptionalMiddleware } from '../../middleware/user'; import { validateOptionalAccessToken } from '../../middleware/validateOptionalAccessToken'; @@ -61,6 +62,14 @@ async function handler( fileTeamPrivacy = 'PUBLIC_TO_TEAM'; } + const license = await licenseClient.check(); + + if (license === null) { + return res.status(500).json({ error: { message: 'Unable to retrieve license' } }); + } + + console.log('license', license); + const data = { file: { uuid, @@ -83,6 +92,7 @@ async function handler( teamRole, teamPermissions, }, + license, }; return res.status(200).json(data); diff --git a/quadratic-api/src/routes/v0/teams.$uuid.GET.ts b/quadratic-api/src/routes/v0/teams.$uuid.GET.ts index 375a4f2ba4..1727debae9 100644 --- a/quadratic-api/src/routes/v0/teams.$uuid.GET.ts +++ b/quadratic-api/src/routes/v0/teams.$uuid.GET.ts @@ -87,15 +87,15 @@ async function handler(req: Request, res: Response user)); + // Get user info from auth + const authUsersById = await getUsers(dbUsers.map(({ user }) => user)); // IDEA: (enhancement) we could put this in /sharing and just return the userCount // then require the data for the team share modal to be a seaparte network request const users = dbUsers - .filter(({ userId: id }) => auth0UsersById[id]) + .filter(({ userId: id }) => authUsersById[id]) .map(({ userId: id, role }) => { - const { email, name, picture } = auth0UsersById[id]; + const { email, name, picture } = authUsersById[id]; return { id, email, @@ -105,7 +105,7 @@ async function handler(req: Request, res: Response res.json()); + const response = await axios + .post(url, { + body: contents, + headers: { + 'Content-Type': 'text/plain', + Authorization: `${jwt}`, + }, + }) + .then((res) => res.data); return response; } catch (e) { diff --git a/quadratic-client/src/routes/_dashboard.tsx b/quadratic-client/src/routes/_dashboard.tsx index 94097ced91..0c30270881 100644 --- a/quadratic-client/src/routes/_dashboard.tsx +++ b/quadratic-client/src/routes/_dashboard.tsx @@ -5,7 +5,7 @@ import { Empty } from '@/dashboard/components/Empty'; import { useRootRouteLoaderData } from '@/routes/_root'; import { apiClient } from '@/shared/api/apiClient'; import { ROUTES, ROUTE_LOADER_IDS, SEARCH_PARAMS } from '@/shared/constants/routes'; -import { CONTACT_URL } from '@/shared/constants/urls'; +import { CONTACT_URL, SCHEDULE_MEETING } from '@/shared/constants/urls'; import { useTheme } from '@/shared/hooks/useTheme'; import { Button } from '@/shared/shadcn/ui/button'; import { Sheet, SheetContent, SheetTrigger } from '@/shared/shadcn/ui/sheet'; @@ -234,7 +234,31 @@ export const ErrorBoundary = () => { ); + const actionsLicenseRevoked = ( + + ); + if (isRouteErrorResponse(error)) { + if (error.status === 402) + return ( + + ); if (error.status === 403) return ( { export const ErrorBoundary = () => { const error = useRouteError(); - const actions = ( + const actionsDefault = ( ); + const actionsLicenseRevoked = ( + + ); + if (isRouteErrorResponse(error)) { let title = ''; let description: string = ''; + let actions = actionsDefault; if (error.status === 404) { title = 'File not found'; @@ -144,6 +162,10 @@ export const ErrorBoundary = () => { } else if (error.status === 400) { title = 'Bad file request'; description = 'Check the URL and try again.'; + } else if (error.status === 402) { + title = 'License Revoked'; + description = 'Your license has been revoked. Please contact Quadratic Support to increase this.'; + actions = actionsLicenseRevoked; } else if (error.status === 403) { title = 'Permission denied'; description = 'You do not have permission to view this file. Try reaching out to the file owner.'; @@ -169,7 +191,7 @@ export const ErrorBoundary = () => { title="Unexpected error" description="Something went wrong loading this file. If the error continues, contact us." Icon={ExclamationTriangleIcon} - actions={actions} + actions={actionsDefault} severity="error" /> ); diff --git a/quadratic-client/src/shared/api/apiClient.ts b/quadratic-client/src/shared/api/apiClient.ts index 92f4dc9a6f..8fe08bf52d 100644 --- a/quadratic-client/src/shared/api/apiClient.ts +++ b/quadratic-client/src/shared/api/apiClient.ts @@ -3,7 +3,7 @@ import * as Sentry from '@sentry/react'; import { Buffer } from 'buffer'; import mixpanel from 'mixpanel-browser'; import { ApiSchemas, ApiTypes } from 'quadratic-shared/typesAndSchemas'; -import { fetchFromApi } from './fetchFromApi'; +import { ApiError, fetchFromApi } from './fetchFromApi'; // TODO(ddimaria): make this dynamic const CURRENT_FILE_VERSION = '1.6'; @@ -14,7 +14,17 @@ export const apiClient = { return fetchFromApi(`/v0/teams`, { method: 'GET' }, ApiSchemas['/v0/teams.GET.response']); }, async get(uuid: string) { - return fetchFromApi(`/v0/teams/${uuid}`, { method: 'GET' }, ApiSchemas['/v0/teams/:uuid.GET.response']); + const response = await fetchFromApi( + `/v0/teams/${uuid}`, + { method: 'GET' }, + ApiSchemas['/v0/teams/:uuid.GET.response'] + ); + + if (response.license.status === 'revoked') { + throw new ApiError('License Revoked', 402, undefined); + } + + return response; }, async update(uuid: string, body: ApiTypes['/v0/teams/:uuid.PATCH.request']) { return fetchFromApi( @@ -91,7 +101,17 @@ export const apiClient = { return fetchFromApi(url, { method: 'GET' }, ApiSchemas['/v0/files.GET.response']); }, async get(uuid: string) { - return fetchFromApi(`/v0/files/${uuid}`, { method: 'GET' }, ApiSchemas['/v0/files/:uuid.GET.response']); + let response = await fetchFromApi( + `/v0/files/${uuid}`, + { method: 'GET' }, + ApiSchemas['/v0/files/:uuid.GET.response'] + ); + + if (response.license.status === 'revoked') { + throw new ApiError('License Revoked', 402, undefined); + } + + return response; }, async create({ file, diff --git a/quadratic-client/src/shared/api/fetchFromApi.ts b/quadratic-client/src/shared/api/fetchFromApi.ts index 7808b07f17..9dd4fbedb4 100644 --- a/quadratic-client/src/shared/api/fetchFromApi.ts +++ b/quadratic-client/src/shared/api/fetchFromApi.ts @@ -61,6 +61,7 @@ export async function fetchFromApi( // Compare the response to the expected schema const result = schema.safeParse(json); + if (!result.success) { console.error(`Zod schema validation failed at: ${path}`, JSON.stringify(result.error, null, 2)); diff --git a/quadratic-client/src/shared/components/ShareDialog.tsx b/quadratic-client/src/shared/components/ShareDialog.tsx index 48c1f70792..3834b38437 100644 --- a/quadratic-client/src/shared/components/ShareDialog.tsx +++ b/quadratic-client/src/shared/components/ShareDialog.tsx @@ -133,9 +133,7 @@ export function ShareTeamDialog({ data }: { data: ApiTypes['/v0/teams/:uuid.GET.
License Revoked!
- - Your license has been revoked. Please contact Quadratic Support to increase this. - + Your license has been revoked. Please contact Quadratic Support. )} diff --git a/quadratic-client/src/shared/constants/urls.ts b/quadratic-client/src/shared/constants/urls.ts index db68b2dc6f..0411616575 100644 --- a/quadratic-client/src/shared/constants/urls.ts +++ b/quadratic-client/src/shared/constants/urls.ts @@ -17,3 +17,4 @@ export const CONTACT_URL = 'https://quadratichq.com/contact'; export const WEBSITE_CONNECTIONS = 'https://www.quadratichq.com/connections'; export const WEBSITE_EXAMPLES = 'https://www.quadratichq.com/examples'; export const WEBSITE_CHANGELOG = 'https://www.quadratichq.com/changelog'; +export const SCHEDULE_MEETING = 'https://calendly.com/d/ckz9-g8t-stb/quadratic-demo'; diff --git a/quadratic-shared/package.json b/quadratic-shared/package.json index 31b7296f6b..b8e3dc12ff 100644 --- a/quadratic-shared/package.json +++ b/quadratic-shared/package.json @@ -4,8 +4,12 @@ "description": "Code shared between quadratic-api and quadratic-client (each app has to compile this code itself)", "main": "index.js", "scripts": { + "compile": "tsc ./*.ts", "test": "echo \"Error: no test specified\" && exit 1" }, "author": "", - "license": "ISC" + "license": "ISC", + "dependencies": { + "typescript": "^5.5.4" + } } diff --git a/quadratic-shared/typesAndSchemas.ts b/quadratic-shared/typesAndSchemas.ts index 1a0477ccaf..823a45961a 100644 --- a/quadratic-shared/typesAndSchemas.ts +++ b/quadratic-shared/typesAndSchemas.ts @@ -107,7 +107,7 @@ export const LicenseSchema = z.object({ limits: z.object({ seats: z.number(), }), - status: z.string(), + status: z.enum(['active', 'exceeded', 'revoked']), }); // Zod schemas for API endpoints @@ -160,6 +160,7 @@ export const ApiSchemas = { teamPermissions: z.array(TeamPermissionSchema).optional(), teamRole: UserTeamRoleSchema.optional(), }), + license: LicenseSchema, }), '/v0/files/:uuid.DELETE.response': z.object({ message: z.string(), From d8c11bf7e519be41a3f4ebd3e669dbde2a71cd0b Mon Sep 17 00:00:00 2001 From: David DiMaria Date: Mon, 26 Aug 2024 14:31:09 -0600 Subject: [PATCH 027/113] Update copy of revoked license on the file view --- quadratic-client/src/routes/file.$uuid.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/quadratic-client/src/routes/file.$uuid.tsx b/quadratic-client/src/routes/file.$uuid.tsx index 43c29d6e2d..7699116f8d 100644 --- a/quadratic-client/src/routes/file.$uuid.tsx +++ b/quadratic-client/src/routes/file.$uuid.tsx @@ -164,7 +164,7 @@ export const ErrorBoundary = () => { description = 'Check the URL and try again.'; } else if (error.status === 402) { title = 'License Revoked'; - description = 'Your license has been revoked. Please contact Quadratic Support to increase this.'; + description = 'Your license has been revoked. Please contact Quadratic Support.'; actions = actionsLicenseRevoked; } else if (error.status === 403) { title = 'Permission denied'; From 459995626566437a96a750c877b1a9d40522aaa0 Mon Sep 17 00:00:00 2001 From: David DiMaria Date: Mon, 26 Aug 2024 15:29:43 -0600 Subject: [PATCH 028/113] Ask user for license key in self-hosting and add value to docker-compose --- self-hosting/docker-compose.yml | 2 +- self-hosting/start.sh | 23 +++++++++++++++++++++-- 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/self-hosting/docker-compose.yml b/self-hosting/docker-compose.yml index 2e8546b458..fb41729b02 100644 --- a/self-hosting/docker-compose.yml +++ b/self-hosting/docker-compose.yml @@ -107,7 +107,7 @@ services: QUADRATIC_FILE_URI_PUBLIC: http://localhost:3002 # Admin - LICENSE_KEY: LICENSE_KEY + LICENSE_KEY: "LICENSE_KEY" LICENSE_API_URI: http://localhost:4000 restart: "always" ports: diff --git a/self-hosting/start.sh b/self-hosting/start.sh index 780b414147..cd5f51e7f0 100755 --- a/self-hosting/start.sh +++ b/self-hosting/start.sh @@ -3,6 +3,18 @@ REPO="https://github.com/quadratichq/quadratic.git" BRANCH="self-hosting-setup" DIR="self-hosting" +INVALID_LICENSE_KEY="Invalid license key." + +get_license_key() { + read -p "Please enter your license key: " user_input + + if [[ $user_input =~ ^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$ ]]; then + echo $user_input + else + echo $INVALID_LICENSE_KEY + return 1 + fi +} checkout() { git clone -b $BRANCH --filter=blob:none --no-checkout --depth 1 --sparse $REPO @@ -19,5 +31,12 @@ start() { docker compose --profile "*" up } -checkout -start \ No newline at end of file +LICENSE_KEY=$(get_license_key) + +if [ "$LICENSE_KEY" = "$INVALID_LICENSE_KEY" ]; then + echo $INVALID_LICENSE_KEY +else + checkout + start + sed -i '' "s/\"LICENSE_KEY\"/\"$LICENSE_KEY\"/g" "docker-compose.yml" +fi From a84179b7579fa4de380ce1a12f7f87cfce201084 Mon Sep 17 00:00:00 2001 From: David DiMaria Date: Mon, 26 Aug 2024 15:55:16 -0600 Subject: [PATCH 029/113] Treat invalid licenses like revoked licenses --- quadratic-api/src/licenseClient.ts | 17 ++++---- quadratic-client/src/shared/api/apiClient.ts | 42 ++++++++++++-------- 2 files changed, 36 insertions(+), 23 deletions(-) diff --git a/quadratic-api/src/licenseClient.ts b/quadratic-api/src/licenseClient.ts index 9f5f0592b1..865b5a8dc3 100644 --- a/quadratic-api/src/licenseClient.ts +++ b/quadratic-api/src/licenseClient.ts @@ -18,21 +18,20 @@ export const licenseClient = { try { const body = { stats: { seats } }; const response = await axios.post(`${LICENSE_API_URI}/api/license/${LICENSE_KEY}`, body); + return LicenseSchema.parse(response.data) as LicenseResponse; } catch (err) { - console.error('Failed to get the license info from the license service', err); + if (err instanceof Error) { + console.error('Failed to get the license info from the license service', err.message); + } + return null; } }, checkFromServer: async (): Promise => { const userCount = await dbClient.user.count(); - const license = await licenseClient.post(userCount); - - if (!license) { - return null; - } - return license; + return licenseClient.post(userCount); }, check: async (): Promise => { const currentTime = Date.now(); @@ -45,6 +44,10 @@ export const licenseClient = { // Otherwise, perform the check const result = await licenseClient.checkFromServer(); + if (!result) { + return null; + } + // Cache the result and update the last checked time cachedResult = result; lastCheckedTime = currentTime; diff --git a/quadratic-client/src/shared/api/apiClient.ts b/quadratic-client/src/shared/api/apiClient.ts index 8fe08bf52d..34f91f779b 100644 --- a/quadratic-client/src/shared/api/apiClient.ts +++ b/quadratic-client/src/shared/api/apiClient.ts @@ -14,17 +14,22 @@ export const apiClient = { return fetchFromApi(`/v0/teams`, { method: 'GET' }, ApiSchemas['/v0/teams.GET.response']); }, async get(uuid: string) { - const response = await fetchFromApi( - `/v0/teams/${uuid}`, - { method: 'GET' }, - ApiSchemas['/v0/teams/:uuid.GET.response'] - ); + try { + const response = await fetchFromApi( + `/v0/teams/${uuid}`, + { method: 'GET' }, + ApiSchemas['/v0/teams/:uuid.GET.response'] + ); + + if (response.license.status === 'revoked') { + throw new ApiError('License Revoked', 402, undefined); + } - if (response.license.status === 'revoked') { + return response; + } catch (err) { + console.error('Error retrieving license key', err); throw new ApiError('License Revoked', 402, undefined); } - - return response; }, async update(uuid: string, body: ApiTypes['/v0/teams/:uuid.PATCH.request']) { return fetchFromApi( @@ -101,17 +106,22 @@ export const apiClient = { return fetchFromApi(url, { method: 'GET' }, ApiSchemas['/v0/files.GET.response']); }, async get(uuid: string) { - let response = await fetchFromApi( - `/v0/files/${uuid}`, - { method: 'GET' }, - ApiSchemas['/v0/files/:uuid.GET.response'] - ); + try { + let response = await fetchFromApi( + `/v0/files/${uuid}`, + { method: 'GET' }, + ApiSchemas['/v0/files/:uuid.GET.response'] + ); + + if (response.license.status === 'revoked') { + throw new ApiError('License Revoked', 402, undefined); + } - if (response.license.status === 'revoked') { + return response; + } catch (err) { + console.error('Error retrieving license key', err); throw new ApiError('License Revoked', 402, undefined); } - - return response; }, async create({ file, From 3e7c74307168e952e051880c3816243b29755c5f Mon Sep 17 00:00:00 2001 From: David DiMaria Date: Mon, 26 Aug 2024 15:56:54 -0600 Subject: [PATCH 030/113] Don't cache errors or non-active licenses --- quadratic-api/src/licenseClient.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/quadratic-api/src/licenseClient.ts b/quadratic-api/src/licenseClient.ts index 865b5a8dc3..319ab3cd17 100644 --- a/quadratic-api/src/licenseClient.ts +++ b/quadratic-api/src/licenseClient.ts @@ -44,7 +44,8 @@ export const licenseClient = { // Otherwise, perform the check const result = await licenseClient.checkFromServer(); - if (!result) { + // don't cache errors or non-active licenses + if (!result || result.status !== 'active') { return null; } From 71457ed52d90731ca1d066de85e44084239b015a Mon Sep 17 00:00:00 2001 From: David DiMaria Date: Mon, 26 Aug 2024 15:58:53 -0600 Subject: [PATCH 031/113] Update copy on share dialog --- quadratic-client/src/shared/components/ShareDialog.tsx | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/quadratic-client/src/shared/components/ShareDialog.tsx b/quadratic-client/src/shared/components/ShareDialog.tsx index 3834b38437..a0577c6182 100644 --- a/quadratic-client/src/shared/components/ShareDialog.tsx +++ b/quadratic-client/src/shared/components/ShareDialog.tsx @@ -123,7 +123,8 @@ export function ShareTeamDialog({ data }: { data: ApiTypes['/v0/teams/:uuid.GET. Over the user limit! - You are over your user limit of {license.limits.seats}. Please contact Quadratic Support to increase this. + You are over your user limit of {license.limits.seats}. Please contact Quadratic Support to increase your + limit. )} From 8bd7c9d6b16fad557a906ad47ef881f6fd26d3ab Mon Sep 17 00:00:00 2001 From: David DiMaria Date: Mon, 26 Aug 2024 16:00:32 -0600 Subject: [PATCH 032/113] Replace docker compose license key before starting docker-compose --- self-hosting/start.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/self-hosting/start.sh b/self-hosting/start.sh index cd5f51e7f0..45adfd582a 100755 --- a/self-hosting/start.sh +++ b/self-hosting/start.sh @@ -37,6 +37,8 @@ if [ "$LICENSE_KEY" = "$INVALID_LICENSE_KEY" ]; then echo $INVALID_LICENSE_KEY else checkout - start + sed -i '' "s/\"LICENSE_KEY\"/\"$LICENSE_KEY\"/g" "docker-compose.yml" + + start fi From 2b5308771a311b98918d13866a3cb4c5fe651736 Mon Sep 17 00:00:00 2001 From: David DiMaria Date: Mon, 26 Aug 2024 16:03:45 -0600 Subject: [PATCH 033/113] Separate apart init.sh and start.sh --- self-hosting/init.sh | 38 ++++++++++++++++++++++++++++++++++++++ self-hosting/start.sh | 37 +------------------------------------ 2 files changed, 39 insertions(+), 36 deletions(-) create mode 100755 self-hosting/init.sh diff --git a/self-hosting/init.sh b/self-hosting/init.sh new file mode 100755 index 0000000000..fb174052b0 --- /dev/null +++ b/self-hosting/init.sh @@ -0,0 +1,38 @@ +#!/bin/sh + +REPO="https://github.com/quadratichq/quadratic.git" +BRANCH="self-hosting-setup" +DIR="self-hosting" +INVALID_LICENSE_KEY="Invalid license key." + +get_license_key() { + read -p "Please enter your license key: " user_input + + if [[ $user_input =~ ^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$ ]]; then + echo $user_input + else + echo $INVALID_LICENSE_KEY + return 1 + fi +} + +checkout() { + git clone -b $BRANCH --filter=blob:none --no-checkout --depth 1 --sparse $REPO + cd quadratic + git sparse-checkout set ${DIR}/ + git checkout + cd $DIR + +} + +LICENSE_KEY=$(get_license_key) + +if [ "$LICENSE_KEY" = "$INVALID_LICENSE_KEY" ]; then + echo $INVALID_LICENSE_KEY +else + checkout + + sed -i '' "s/\"LICENSE_KEY\"/\"$LICENSE_KEY\"/g" "docker-compose.yml" + + sh start.sh +fi diff --git a/self-hosting/start.sh b/self-hosting/start.sh index 45adfd582a..a11d58e1ca 100755 --- a/self-hosting/start.sh +++ b/self-hosting/start.sh @@ -1,44 +1,9 @@ #!/bin/sh -REPO="https://github.com/quadratichq/quadratic.git" -BRANCH="self-hosting-setup" -DIR="self-hosting" -INVALID_LICENSE_KEY="Invalid license key." - -get_license_key() { - read -p "Please enter your license key: " user_input - - if [[ $user_input =~ ^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$ ]]; then - echo $user_input - else - echo $INVALID_LICENSE_KEY - return 1 - fi -} - -checkout() { - git clone -b $BRANCH --filter=blob:none --no-checkout --depth 1 --sparse $REPO - cd quadratic - git sparse-checkout set ${DIR}/ - git checkout - cd $DIR - -} - start() { docker compose --profile "*" down yes | docker compose rm quadratic-client docker compose --profile "*" up } -LICENSE_KEY=$(get_license_key) - -if [ "$LICENSE_KEY" = "$INVALID_LICENSE_KEY" ]; then - echo $INVALID_LICENSE_KEY -else - checkout - - sed -i '' "s/\"LICENSE_KEY\"/\"$LICENSE_KEY\"/g" "docker-compose.yml" - - start -fi +start \ No newline at end of file From f1ef337fba4bcd97af738c124f6a6be389876c5c Mon Sep 17 00:00:00 2001 From: David DiMaria Date: Mon, 26 Aug 2024 16:42:09 -0600 Subject: [PATCH 034/113] Include the url to the self-hosting admin in inti.sh --- self-hosting/README.md | 3 +++ self-hosting/init.sh | 3 ++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/self-hosting/README.md b/self-hosting/README.md index 1afcca9fa6..b6acbb1320 100644 --- a/self-hosting/README.md +++ b/self-hosting/README.md @@ -1,2 +1,5 @@ # Quadratic Self-Hosting +```shell +curl -sSf https://raw.githubusercontent.com/quadratichq/quadratic/2b5308771a311b98918d13866a3cb4c5fe651736/self-hosting/init.sh -o init.sh && sh -i init.sh +``` \ No newline at end of file diff --git a/self-hosting/init.sh b/self-hosting/init.sh index fb174052b0..471bc7219b 100755 --- a/self-hosting/init.sh +++ b/self-hosting/init.sh @@ -3,10 +3,11 @@ REPO="https://github.com/quadratichq/quadratic.git" BRANCH="self-hosting-setup" DIR="self-hosting" +SELF_HOSTING_URI="https://selfhost.quadratic-preview.com" INVALID_LICENSE_KEY="Invalid license key." get_license_key() { - read -p "Please enter your license key: " user_input + read -p "Enter your license key (Get one for free instantly at $SELF_HOSTING_URI): " user_input if [[ $user_input =~ ^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$ ]]; then echo $user_input From bddbe9833b601cc4bf2c26739262eef228e32af4 Mon Sep 17 00:00:00 2001 From: David DiMaria Date: Mon, 26 Aug 2024 16:47:17 -0600 Subject: [PATCH 035/113] Re-enable the cache --- quadratic-api/src/licenseClient.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/quadratic-api/src/licenseClient.ts b/quadratic-api/src/licenseClient.ts index 319ab3cd17..a1ca469bed 100644 --- a/quadratic-api/src/licenseClient.ts +++ b/quadratic-api/src/licenseClient.ts @@ -10,8 +10,8 @@ type LicenseResponse = z.infer; let cachedResult: LicenseResponse | null = null; let lastCheckedTime: number | null = null; -// const cacheDuration = 15 * 60 * 1000; // 15 minutes in milliseconds -const cacheDuration = 0; // TODO(ddimaria): remove +const cacheDuration = 15 * 60 * 1000; // 15 minutes in milliseconds +// const cacheDuration = 0; // disable the cache for testing export const licenseClient = { post: async (seats: number): Promise => { From 4dc570a7d0fc8c86a2f8a1e025abd8385669c836 Mon Sep 17 00:00:00 2001 From: David Kircos Date: Mon, 26 Aug 2024 17:08:49 -0600 Subject: [PATCH 036/113] test image push --- .../workflows/production-publish-images.yml | 43 +++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 .github/workflows/production-publish-images.yml diff --git a/.github/workflows/production-publish-images.yml b/.github/workflows/production-publish-images.yml new file mode 100644 index 0000000000..625b682a36 --- /dev/null +++ b/.github/workflows/production-publish-images.yml @@ -0,0 +1,43 @@ +name: Build and Publish Images to ECR + +on: + push: + branches: + - self-hosting-setup + +concurrency: + group: production-publish-images + +jobs: + publish_images: + runs-on: ubuntu-latest-8-cores + strategy: + matrix: + service: [multiplayer, files, connection, client, api] + steps: + - uses: actions/checkout@v4 + + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ secrets.AWS_REGION }} + + - name: Login to Amazon ECR + id: login-ecr + uses: aws-actions/amazon-ecr-login@v2 + + - name: Create ECR Repository if not exists + run: | + aws ecr describe-repositories --repository-names quadratic-${{ matrix.service }}-production || \ + aws ecr create-repository --repository-name quadratic-${{ matrix.service }}-production + + - name: Build, Tag, and Push Image to Amazon ECR + env: + ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} + ECR_REPOSITORY: quadratic-${{ matrix.service }}-production + IMAGE_TAG: 0.1.0 + run: | + docker build -t $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG -f quadratic-${{ matrix.service }}/Dockerfile . + docker push $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG \ No newline at end of file From 8b6d8edaca6ad570c4f5f95839ddcad2fa4e240a Mon Sep 17 00:00:00 2001 From: David Kircos Date: Mon, 26 Aug 2024 17:15:22 -0600 Subject: [PATCH 037/113] point towards dev --- .github/workflows/production-publish-images.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/production-publish-images.yml b/.github/workflows/production-publish-images.yml index 625b682a36..8edaef79e4 100644 --- a/.github/workflows/production-publish-images.yml +++ b/.github/workflows/production-publish-images.yml @@ -20,8 +20,8 @@ jobs: - name: Configure AWS Credentials uses: aws-actions/configure-aws-credentials@v4 with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID_DEVELOPMENT }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY_DEVELOPMENT }} aws-region: ${{ secrets.AWS_REGION }} - name: Login to Amazon ECR From 3d19a2b69b7f0ed91968e68423108724fd64baaa Mon Sep 17 00:00:00 2001 From: David Kircos Date: Mon, 26 Aug 2024 17:24:37 -0600 Subject: [PATCH 038/113] public --- .github/workflows/production-publish-images.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/production-publish-images.yml b/.github/workflows/production-publish-images.yml index 8edaef79e4..c0d5990413 100644 --- a/.github/workflows/production-publish-images.yml +++ b/.github/workflows/production-publish-images.yml @@ -28,16 +28,16 @@ jobs: id: login-ecr uses: aws-actions/amazon-ecr-login@v2 - - name: Create ECR Repository if not exists + - name: Create Public ECR Repository if not exists run: | - aws ecr describe-repositories --repository-names quadratic-${{ matrix.service }}-production || \ - aws ecr create-repository --repository-name quadratic-${{ matrix.service }}-production + aws ecr-public describe-repositories --repository-names quadratic-${{ matrix.service }}-production || \ + aws ecr-public create-repository --repository-name quadratic-${{ matrix.service }}-production - - name: Build, Tag, and Push Image to Amazon ECR + - name: Build, Tag, and Push Image to Amazon ECR Public env: ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} ECR_REPOSITORY: quadratic-${{ matrix.service }}-production IMAGE_TAG: 0.1.0 run: | - docker build -t $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG -f quadratic-${{ matrix.service }}/Dockerfile . - docker push $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG \ No newline at end of file + docker build -t public.ecr.aws/$ECR_REPOSITORY:$IMAGE_TAG -f quadratic-${{ matrix.service }}/Dockerfile . + docker push public.ecr.aws/$ECR_REPOSITORY:$IMAGE_TAG \ No newline at end of file From d442f4eae7c88f8bf13373b2716acb9f6454b358 Mon Sep 17 00:00:00 2001 From: David Kircos Date: Mon, 26 Aug 2024 17:26:02 -0600 Subject: [PATCH 039/113] create in us-east-1 --- .github/workflows/production-publish-images.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/production-publish-images.yml b/.github/workflows/production-publish-images.yml index c0d5990413..c14f91db15 100644 --- a/.github/workflows/production-publish-images.yml +++ b/.github/workflows/production-publish-images.yml @@ -22,7 +22,7 @@ jobs: with: aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID_DEVELOPMENT }} aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY_DEVELOPMENT }} - aws-region: ${{ secrets.AWS_REGION }} + aws-region: us-east-1 - name: Login to Amazon ECR id: login-ecr From d695e852a824efad37bf31bb40157bb3de499029 Mon Sep 17 00:00:00 2001 From: David Kircos Date: Mon, 26 Aug 2024 18:58:37 -0600 Subject: [PATCH 040/113] use correct URL --- .github/workflows/production-publish-images.yml | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/.github/workflows/production-publish-images.yml b/.github/workflows/production-publish-images.yml index c14f91db15..192891166f 100644 --- a/.github/workflows/production-publish-images.yml +++ b/.github/workflows/production-publish-images.yml @@ -29,15 +29,17 @@ jobs: uses: aws-actions/amazon-ecr-login@v2 - name: Create Public ECR Repository if not exists + id: create-ecr run: | - aws ecr-public describe-repositories --repository-names quadratic-${{ matrix.service }}-production || \ - aws ecr-public create-repository --repository-name quadratic-${{ matrix.service }}-production + REPO_INFO=$(aws ecr-public describe-repositories --repository-names quadratic-${{ matrix.service }}-production || \ + aws ecr-public create-repository --repository-name quadratic-${{ matrix.service }}-production) + ECR_URL=$(echo $REPO_INFO | jq -r '.repositories[0].repositoryUri') + echo "ECR_URL=$ECR_URL" >> $GITHUB_OUTPUT - name: Build, Tag, and Push Image to Amazon ECR Public env: - ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} - ECR_REPOSITORY: quadratic-${{ matrix.service }}-production + ECR_URL: ${{ steps.create-ecr.outputs.ECR_URL }} IMAGE_TAG: 0.1.0 run: | - docker build -t public.ecr.aws/$ECR_REPOSITORY:$IMAGE_TAG -f quadratic-${{ matrix.service }}/Dockerfile . - docker push public.ecr.aws/$ECR_REPOSITORY:$IMAGE_TAG \ No newline at end of file + docker build -t $ECR_URL:$IMAGE_TAG -f quadratic-${{ matrix.service }}/Dockerfile . + docker push $ECR_URL:$IMAGE_TAG \ No newline at end of file From 9a078dc2bae713c022b844dbae0e12b766337fbc Mon Sep 17 00:00:00 2001 From: David Kircos Date: Mon, 26 Aug 2024 19:22:48 -0600 Subject: [PATCH 041/113] use public --- .github/workflows/production-publish-images.yml | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/.github/workflows/production-publish-images.yml b/.github/workflows/production-publish-images.yml index 192891166f..64ef3e68ae 100644 --- a/.github/workflows/production-publish-images.yml +++ b/.github/workflows/production-publish-images.yml @@ -24,15 +24,24 @@ jobs: aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY_DEVELOPMENT }} aws-region: us-east-1 - - name: Login to Amazon ECR + - name: Login to Amazon ECR Public id: login-ecr uses: aws-actions/amazon-ecr-login@v2 + with: + registry-type: public + + - name: Define repository name + id: repo-name + run: | + echo "REPO_NAME=quadratic-${{ matrix.service }}-production" >> $GITHUB_OUTPUT - name: Create Public ECR Repository if not exists id: create-ecr + env: + REPO_NAME: ${{ steps.repo-name.outputs.REPO_NAME }} run: | - REPO_INFO=$(aws ecr-public describe-repositories --repository-names quadratic-${{ matrix.service }}-production || \ - aws ecr-public create-repository --repository-name quadratic-${{ matrix.service }}-production) + REPO_INFO=$(aws ecr-public describe-repositories --repository-names $REPO_NAME || \ + aws ecr-public create-repository --repository-name $REPO_NAME) ECR_URL=$(echo $REPO_INFO | jq -r '.repositories[0].repositoryUri') echo "ECR_URL=$ECR_URL" >> $GITHUB_OUTPUT From 016874f1e7e4c7c54e0529831d27d7b52aded8c7 Mon Sep 17 00:00:00 2001 From: David Kircos Date: Mon, 26 Aug 2024 19:30:07 -0600 Subject: [PATCH 042/113] always tag latest as well --- .github/workflows/production-publish-images.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/production-publish-images.yml b/.github/workflows/production-publish-images.yml index 64ef3e68ae..c606a42968 100644 --- a/.github/workflows/production-publish-images.yml +++ b/.github/workflows/production-publish-images.yml @@ -51,4 +51,5 @@ jobs: IMAGE_TAG: 0.1.0 run: | docker build -t $ECR_URL:$IMAGE_TAG -f quadratic-${{ matrix.service }}/Dockerfile . - docker push $ECR_URL:$IMAGE_TAG \ No newline at end of file + docker push $ECR_URL:$IMAGE_TAG + docker push $ECR_URL:latest \ No newline at end of file From 03d8f4072f3d4df3c01f909662d261a92048dad3 Mon Sep 17 00:00:00 2001 From: David Kircos Date: Mon, 26 Aug 2024 19:30:18 -0600 Subject: [PATCH 043/113] change repo to staging --- .github/workflows/production-publish-images.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/production-publish-images.yml b/.github/workflows/production-publish-images.yml index c606a42968..be63aa666c 100644 --- a/.github/workflows/production-publish-images.yml +++ b/.github/workflows/production-publish-images.yml @@ -33,7 +33,7 @@ jobs: - name: Define repository name id: repo-name run: | - echo "REPO_NAME=quadratic-${{ matrix.service }}-production" >> $GITHUB_OUTPUT + echo "REPO_NAME=quadratic-${{ matrix.service }}-staging" >> $GITHUB_OUTPUT - name: Create Public ECR Repository if not exists id: create-ecr From 661bc4e9c15f53779cece96006fbe3aa19a14bb9 Mon Sep 17 00:00:00 2001 From: David Kircos Date: Mon, 26 Aug 2024 19:35:37 -0600 Subject: [PATCH 044/113] create repo then describe it --- .github/workflows/production-publish-images.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/production-publish-images.yml b/.github/workflows/production-publish-images.yml index be63aa666c..b0453c3391 100644 --- a/.github/workflows/production-publish-images.yml +++ b/.github/workflows/production-publish-images.yml @@ -40,11 +40,11 @@ jobs: env: REPO_NAME: ${{ steps.repo-name.outputs.REPO_NAME }} run: | - REPO_INFO=$(aws ecr-public describe-repositories --repository-names $REPO_NAME || \ - aws ecr-public create-repository --repository-name $REPO_NAME) + aws ecr-public create-repository --repository-name $REPO_NAME || true + REPO_INFO=$(aws ecr-public describe-repositories --repository-names $REPO_NAME) ECR_URL=$(echo $REPO_INFO | jq -r '.repositories[0].repositoryUri') echo "ECR_URL=$ECR_URL" >> $GITHUB_OUTPUT - + - name: Build, Tag, and Push Image to Amazon ECR Public env: ECR_URL: ${{ steps.create-ecr.outputs.ECR_URL }} From 85b5f5e795874ee052861ad6df1b7122c7367d4f Mon Sep 17 00:00:00 2001 From: David Kircos Date: Mon, 26 Aug 2024 19:52:51 -0600 Subject: [PATCH 045/113] add second tag --- .github/workflows/production-publish-images.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/production-publish-images.yml b/.github/workflows/production-publish-images.yml index b0453c3391..25a1d567ef 100644 --- a/.github/workflows/production-publish-images.yml +++ b/.github/workflows/production-publish-images.yml @@ -50,6 +50,6 @@ jobs: ECR_URL: ${{ steps.create-ecr.outputs.ECR_URL }} IMAGE_TAG: 0.1.0 run: | - docker build -t $ECR_URL:$IMAGE_TAG -f quadratic-${{ matrix.service }}/Dockerfile . + docker build -t $ECR_URL:$IMAGE_TAG -t $ECR_URL:latest -f quadratic-${{ matrix.service }}/Dockerfile . docker push $ECR_URL:$IMAGE_TAG docker push $ECR_URL:latest \ No newline at end of file From 27714636745e6c68e5d0412e2d0eafa16167aa30 Mon Sep 17 00:00:00 2001 From: David DiMaria Date: Wed, 28 Aug 2024 10:22:00 -0600 Subject: [PATCH 046/113] Use ecr images in self-hosting docker-compose --- self-hosting/docker-compose.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/self-hosting/docker-compose.yml b/self-hosting/docker-compose.yml index fb41729b02..176f0b1c86 100644 --- a/self-hosting/docker-compose.yml +++ b/self-hosting/docker-compose.yml @@ -54,7 +54,7 @@ services: - host quadratic-client: - image: quadratic_quadratic-client:latest + image: public.ecr.aws/l3i4i9z2/quadratic-client-staging:latest restart: "always" environment: VITE_DEBUG: 1 @@ -83,7 +83,7 @@ services: - host quadratic-api: - image: quadratic_quadratic-api + image: public.ecr.aws/l3i4i9z2/quadratic-api-staging:latest environment: CORS: "*" DATABASE_URL: "postgresql://postgres:postgres@host.docker.internal:5432/postgres" @@ -123,7 +123,7 @@ services: - host quadratic-multiplayer: - image: quadratic-multiplayer + image: public.ecr.aws/l3i4i9z2/quadratic-multiplayer-staging:latest environment: RUST_LOG: info HOST: 0.0.0.0 @@ -160,7 +160,7 @@ services: - host quadratic-files: - image: quadratic-files + image: public.ecr.aws/l3i4i9z2/quadratic-files-staging:latest environment: RUST_LOG: info HOST: 0.0.0.0 @@ -215,7 +215,7 @@ services: - host quadratic-connection: - image: quadratic-connection + image: public.ecr.aws/l3i4i9z2/quadratic-connection-staging:latest environment: RUST_LOG: info HOST: 0.0.0.0 From 19454acedd34662f40c53027625e4754d53489af Mon Sep 17 00:00:00 2001 From: David DiMaria Date: Wed, 28 Aug 2024 11:24:40 -0600 Subject: [PATCH 047/113] Update client Dockerfile --- quadratic-client/Dockerfile | 82 +++++++++++++++++++++++++++++++++---- 1 file changed, 75 insertions(+), 7 deletions(-) diff --git a/quadratic-client/Dockerfile b/quadratic-client/Dockerfile index e14d164e17..1633f99787 100644 --- a/quadratic-client/Dockerfile +++ b/quadratic-client/Dockerfile @@ -1,13 +1,81 @@ -FROM node:18-alpine AS builder +# Use an official node image as a parent image +FROM node:18 AS build + +# Install rustup +RUN echo 'Installing rustup...' && curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y +ENV PATH="/root/.cargo/bin:${PATH}" + +# Install wasm-pack +RUN echo 'Installing wasm-pack...' && curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh + +# Install wasm32-unknown-unknown target +RUN rustup target add wasm32-unknown-unknown + +# Install python, binaryen & clean up +RUN apt-get update && apt-get install -y python-is-python3 python3-pip binaryen && apt-get clean && rm -rf /var/lib/apt/lists/* + +# Install npm dependencies WORKDIR /app COPY package.json . COPY package-lock.json . -COPY updateAlertVersion.json . -COPY quadratic-client ./quadratic-client -COPY quadratic-shared ./quadratic-shared +COPY ./quadratic-kernels/python-wasm/package*.json ./quadratic-kernels/python-wasm/ +COPY ./quadratic-core/package*.json ./quadratic-core/ +COPY ./quadratic-rust-client/package*.json ./quadratic-rust-client/ +COPY ./quadratic-shared/package*.json ./quadratic-shared/ +COPY ./quadratic-client/package*.json ./quadratic-client/ RUN npm install -FROM node:18-slim AS runtime +# Install typescript +RUN npm install -D typescript + +# Copy the rest of the application +WORKDIR /app +COPY updateAlertVersion.json . +COPY ./quadratic-kernels/python-wasm/. ./quadratic-kernels/python-wasm/ +COPY ./quadratic-core/. ./quadratic-core/ +COPY ./quadratic-rust-client/. ./quadratic-rust-client/ +COPY ./quadratic-shared/. ./quadratic-shared/ +COPY ./quadratic-client/. ./quadratic-client/ + +# Run the packaging script for quadratic_py +WORKDIR /app +RUN ./quadratic-kernels/python-wasm/package.sh --no-poetry + +# Build wasm +WORKDIR /app/quadratic-core +RUN echo 'Building wasm...' && wasm-pack build --target web --out-dir ../quadratic-client/src/app/quadratic-core --weak-refs + +# Export TS/Rust types +WORKDIR /app/quadratic-core +RUN echo 'Exporting TS/Rust types...' && cargo run --bin export_types + +# Build the quadratic-rust-client WORKDIR /app -COPY --from=builder /app . -CMD ["npm", "start"] +ARG GIT_COMMIT +ENV GIT_COMMIT=$GIT_COMMIT +RUN echo 'Building quadratic-rust-client...' && npm run build --workspace=quadratic-rust-client + +# Build the quadratic-shared +WORKDIR /app +RUN echo 'Building quadratic-shared...' && npx tsc ./quadratic-shared/*.ts + +# Build the front-end +WORKDIR /app +RUN echo 'Building front-end...' +ENV VITE_DEBUG=VITE_DEBUG_VAL +ENV VITE_QUADRATIC_API_URL=VITE_QUADRATIC_API_URL_VAL +ENV VITE_QUADRATIC_MULTIPLAYER_URL=VITE_QUADRATIC_MULTIPLAYER_URL_VAL +ENV VITE_QUADRATIC_CONNECTION_URL=VITE_QUADRATIC_CONNECTION_URL_VAL +ENV VITE_AUTH_TYPE=VITE_AUTH_TYPE_VAL +ENV VITE_ORY_HOST=VITE_ORY_HOST_VAL +RUN npm run build --workspace=quadratic-client + +# The default command to run the application +# CMD ["npm", "run", "start:production"] + +FROM nginx:stable-alpine +COPY --from=build /app/build /usr/share/nginx/html + +EXPOSE 80 443 3000 + +CMD ["nginx", "-g", "daemon off;"] From e48f821199ec5db4a8d1510db83cd9e4dc05b0d8 Mon Sep 17 00:00:00 2001 From: David DiMaria Date: Wed, 28 Aug 2024 13:33:17 -0600 Subject: [PATCH 048/113] Add self-hosting documentation, fix quadratic-api to conditionally load --- quadratic-api/src/auth/auth.ts | 16 +++++++++-- quadratic-api/src/storage/storage.ts | 16 +++++++++-- self-hosting/README.md | 42 ++++++++++++++++++++++++++-- self-hosting/docker-compose.yml | 17 +++++++---- 4 files changed, 80 insertions(+), 11 deletions(-) diff --git a/quadratic-api/src/auth/auth.ts b/quadratic-api/src/auth/auth.ts index 74df06ee67..f00b88425b 100644 --- a/quadratic-api/src/auth/auth.ts +++ b/quadratic-api/src/auth/auth.ts @@ -1,6 +1,18 @@ import { AUTH_TYPE } from '../env-vars'; -import { getUsersFromAuth0, jwtConfigAuth0, lookupUsersFromAuth0ByEmail } from './auth0'; -import { getUsersFromOry, getUsersFromOryByEmail, jwtConfigOry } from './ory'; + +let { getUsersFromAuth0, jwtConfigAuth0, lookupUsersFromAuth0ByEmail } = {} as any; +let { getUsersFromOry, getUsersFromOryByEmail, jwtConfigOry } = {} as any; + +switch (AUTH_TYPE) { + case 'auth0': + ({ getUsersFromAuth0, jwtConfigAuth0, lookupUsersFromAuth0ByEmail } = require('./auth0')); + break; + case 'ory': + ({ getUsersFromOry, getUsersFromOryByEmail, jwtConfigOry } = require('./ory')); + break; + default: + throw new Error(`Unsupported auth type in auth.ts: ${AUTH_TYPE}`); +} export type UsersRequest = { id: number; diff --git a/quadratic-api/src/storage/storage.ts b/quadratic-api/src/storage/storage.ts index 1feaa15d6a..88444e98c7 100644 --- a/quadratic-api/src/storage/storage.ts +++ b/quadratic-api/src/storage/storage.ts @@ -1,7 +1,19 @@ import multer from 'multer'; import { STORAGE_TYPE } from '../env-vars'; -import { getPresignedStorageUrl, getStorageUrl, multerFileSystemStorage, upload } from './fileSystem'; -import { generatePresignedUrl, multerS3Storage, uploadStringAsFileS3 } from './s3'; + +let { getPresignedStorageUrl, getStorageUrl, multerFileSystemStorage, upload } = {} as any; +let { generatePresignedUrl, multerS3Storage, uploadStringAsFileS3 } = {} as any; + +switch (STORAGE_TYPE) { + case 's3': + ({ generatePresignedUrl, multerS3Storage, uploadStringAsFileS3 } = require('./s3')); + break; + case 'file-system': + ({ getPresignedStorageUrl, getStorageUrl, multerFileSystemStorage, upload } = require('./fileSystem')); + break; + default: + throw new Error(`Unsupported storage type in storage.ts: ${STORAGE_TYPE}`); +} export type UploadFileResponse = { bucket: string; diff --git a/self-hosting/README.md b/self-hosting/README.md index b6acbb1320..48502968ca 100644 --- a/self-hosting/README.md +++ b/self-hosting/README.md @@ -1,5 +1,43 @@ # Quadratic Self-Hosting +Implement the entire Quadratic stack outside of Quadratic. The use cases we currently support: + +- [x] Localhost +- [x] EC2 (using your own load balancer) +- [ ] EC2 (using [Caddy's load balancer](https://caddyserver.com/docs/caddyfile/directives/reverse_proxy)) +- [ ] Multiple Docker instance setup (for any cloud provider) +- [ ] Kubernetes + +## Dependencies + +* [Git](https://github.com/git-guides/install-git) +* [Docker](https://docs.docker.com/engine/install/) + +## Requirements + +* MacOS or Linux (not tested on Windows) +* License Key (available at https://selfhost.quadratic-preview.com) + +## Installation + +> **NOTE:** _Before installing, please create a license and copy the key at https://selfhost.quadratic-preview.com._ + +Quadratic can be installed via a single command: + +```shell +curl -sSf https://raw.githubusercontent.com/quadratichq/quadratic/27714636745e6c68e5d0412e2d0eafa16167aa30/self-hosting/init.sh -o init.sh && sh -i init.sh +``` + +This will download the initialization script, which will prompt for a license key in order to register Quadratic. + +Additionally, the docker compose network will start (see [Starting](#Starting)). Please allow several minutes for the docker images to downloaded. + +Refer to the [Closing](#Closing) section. + +## Starting + +Once the Quadratic is initialized, a single command is needed to start all of the images: + ```shell -curl -sSf https://raw.githubusercontent.com/quadratichq/quadratic/2b5308771a311b98918d13866a3cb4c5fe651736/self-hosting/init.sh -o init.sh && sh -i init.sh -``` \ No newline at end of file +./start.sh +``` diff --git a/self-hosting/docker-compose.yml b/self-hosting/docker-compose.yml index 176f0b1c86..9647f9cf47 100644 --- a/self-hosting/docker-compose.yml +++ b/self-hosting/docker-compose.yml @@ -83,7 +83,7 @@ services: - host quadratic-api: - image: public.ecr.aws/l3i4i9z2/quadratic-api-staging:latest + image: quadratic-api environment: CORS: "*" DATABASE_URL: "postgresql://postgres:postgres@host.docker.internal:5432/postgres" @@ -100,6 +100,12 @@ services: AUTH_TYPE: ory ORY_JWKS_URI: "http://host.docker.internal:3000/.well-known/jwks.json" ORY_ADMIN_HOST: http://host.docker.internal:4434 + AUTH0_JWKS_URI: AUTH0_JWKS_URI + AUTH0_ISSUER: AUTH0_ISSUER + AUTH0_CLIENT_ID: AUTH0_CLIENT_ID + AUTH0_CLIENT_SECRET: AUTH0_CLIENT_SECRET + AUTH0_DOMAIN: AUTH0_DOMAIN + AUTH0_AUDIENCE: AUTH0_AUDIENCE # Storage STORAGE_TYPE: file-system @@ -112,10 +118,11 @@ services: restart: "always" ports: - "8000:8000" - command: "npm run start:prod --workspace=quadratic-api" - depends_on: - postgres: - condition: service_healthy + # command: "npm run start:prod --workspace=quadratic-api" + command: "node quadratic-api/dist/src/server.js --max-old-space-size=8192" + # depends_on: + # postgres: + # condition: service_healthy profiles: - api - frontend From 18e2d12cf4173e7c39f83ef43fe56e3f205a00f8 Mon Sep 17 00:00:00 2001 From: David DiMaria Date: Wed, 28 Aug 2024 13:33:52 -0600 Subject: [PATCH 049/113] Revert docker-compose --- self-hosting/docker-compose.yml | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/self-hosting/docker-compose.yml b/self-hosting/docker-compose.yml index 9647f9cf47..176f0b1c86 100644 --- a/self-hosting/docker-compose.yml +++ b/self-hosting/docker-compose.yml @@ -83,7 +83,7 @@ services: - host quadratic-api: - image: quadratic-api + image: public.ecr.aws/l3i4i9z2/quadratic-api-staging:latest environment: CORS: "*" DATABASE_URL: "postgresql://postgres:postgres@host.docker.internal:5432/postgres" @@ -100,12 +100,6 @@ services: AUTH_TYPE: ory ORY_JWKS_URI: "http://host.docker.internal:3000/.well-known/jwks.json" ORY_ADMIN_HOST: http://host.docker.internal:4434 - AUTH0_JWKS_URI: AUTH0_JWKS_URI - AUTH0_ISSUER: AUTH0_ISSUER - AUTH0_CLIENT_ID: AUTH0_CLIENT_ID - AUTH0_CLIENT_SECRET: AUTH0_CLIENT_SECRET - AUTH0_DOMAIN: AUTH0_DOMAIN - AUTH0_AUDIENCE: AUTH0_AUDIENCE # Storage STORAGE_TYPE: file-system @@ -118,11 +112,10 @@ services: restart: "always" ports: - "8000:8000" - # command: "npm run start:prod --workspace=quadratic-api" - command: "node quadratic-api/dist/src/server.js --max-old-space-size=8192" - # depends_on: - # postgres: - # condition: service_healthy + command: "npm run start:prod --workspace=quadratic-api" + depends_on: + postgres: + condition: service_healthy profiles: - api - frontend From 11e73b6996d141770b08ef91b5ae1205d72a0031 Mon Sep 17 00:00:00 2001 From: David DiMaria Date: Wed, 28 Aug 2024 13:56:57 -0600 Subject: [PATCH 050/113] Add prisma target debian-openssl-3.0.x and add empty passwords --- quadratic-api/prisma/schema.prisma | 2 +- self-hosting/docker-compose.yml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/quadratic-api/prisma/schema.prisma b/quadratic-api/prisma/schema.prisma index f59991e81f..44c585f4f3 100644 --- a/quadratic-api/prisma/schema.prisma +++ b/quadratic-api/prisma/schema.prisma @@ -1,6 +1,6 @@ generator client { provider = "prisma-client-js" - binaryTargets = ["native", "linux-arm64-openssl-3.0.x"] + binaryTargets = ["native", "linux-arm64-openssl-3.0.x", "debian-openssl-3.0.x"] } datasource db { diff --git a/self-hosting/docker-compose.yml b/self-hosting/docker-compose.yml index 176f0b1c86..b532b396df 100644 --- a/self-hosting/docker-compose.yml +++ b/self-hosting/docker-compose.yml @@ -136,7 +136,7 @@ services: PUBSUB_HOST: host.docker.internal PUBSUB_PORT: 6379 - PUBSUB_PASSWORD: + PUBSUB_PASSWORD: "" PUBSUB_ACTIVE_CHANNELS: active_channels AUTH0_JWKS_URI: http://host.docker.internal:3000/.well-known/jwks.json @@ -177,7 +177,7 @@ services: PUBSUB_HOST: host.docker.internal PUBSUB_PORT: 6379 - PUBSUB_PASSWORD: + PUBSUB_PASSWORD: "" PUBSUB_ACTIVE_CHANNELS: active_channels PUBSUB_PROCESSED_TRANSACTIONS_CHANNEL: processed_transactions From 22a8e9ed1af148e0a758b693c810cd5c86172244 Mon Sep 17 00:00:00 2001 From: David DiMaria Date: Wed, 28 Aug 2024 14:19:36 -0600 Subject: [PATCH 051/113] Run postinstall during the api build, better workaround for auth0/s3 env var deps --- quadratic-api/Dockerfile | 1 + quadratic-api/src/auth/auth.ts | 16 +------- quadratic-api/src/auth/auth0.ts | 24 ++++++++---- quadratic-api/src/storage/s3.ts | 56 ++++++++++++++++------------ quadratic-api/src/storage/storage.ts | 18 ++------- 5 files changed, 54 insertions(+), 61 deletions(-) diff --git a/quadratic-api/Dockerfile b/quadratic-api/Dockerfile index 7ac0726780..77a8ba4e16 100644 --- a/quadratic-api/Dockerfile +++ b/quadratic-api/Dockerfile @@ -10,5 +10,6 @@ FROM node:18-slim AS runtime WORKDIR /app COPY --from=builder /app . RUN apt-get update && apt install -y openssl +RUN npm run postinstall --workspace=quadratic-api RUN npm run build:prod --workspace=quadratic-api CMD ["npm", "start:prod"] diff --git a/quadratic-api/src/auth/auth.ts b/quadratic-api/src/auth/auth.ts index f00b88425b..74df06ee67 100644 --- a/quadratic-api/src/auth/auth.ts +++ b/quadratic-api/src/auth/auth.ts @@ -1,18 +1,6 @@ import { AUTH_TYPE } from '../env-vars'; - -let { getUsersFromAuth0, jwtConfigAuth0, lookupUsersFromAuth0ByEmail } = {} as any; -let { getUsersFromOry, getUsersFromOryByEmail, jwtConfigOry } = {} as any; - -switch (AUTH_TYPE) { - case 'auth0': - ({ getUsersFromAuth0, jwtConfigAuth0, lookupUsersFromAuth0ByEmail } = require('./auth0')); - break; - case 'ory': - ({ getUsersFromOry, getUsersFromOryByEmail, jwtConfigOry } = require('./ory')); - break; - default: - throw new Error(`Unsupported auth type in auth.ts: ${AUTH_TYPE}`); -} +import { getUsersFromAuth0, jwtConfigAuth0, lookupUsersFromAuth0ByEmail } from './auth0'; +import { getUsersFromOry, getUsersFromOryByEmail, jwtConfigOry } from './ory'; export type UsersRequest = { id: number; diff --git a/quadratic-api/src/auth/auth0.ts b/quadratic-api/src/auth/auth0.ts index f399c15674..8fbbe46378 100644 --- a/quadratic-api/src/auth/auth0.ts +++ b/quadratic-api/src/auth/auth0.ts @@ -21,12 +21,20 @@ import { ByEmailUser } from './auth'; // We need to use account linking to ensure only one account per user // https://auth0.com/docs/customize/extensions/account-link-extension -const auth0 = new ManagementClient({ - domain: AUTH0_DOMAIN, - clientId: AUTH0_CLIENT_ID, - clientSecret: AUTH0_CLIENT_SECRET, - scope: 'read:users', -}); +let auth0: ManagementClient | undefined; + +const getAuth0 = () => { + if (!auth0) { + auth0 = auth0 = new ManagementClient({ + domain: AUTH0_DOMAIN, + clientId: AUTH0_CLIENT_ID, + clientSecret: AUTH0_CLIENT_SECRET, + scope: 'read:users', + }); + } + + return auth0; +}; /** * Given a list of users from our system, we lookup their info in Auth0. @@ -55,7 +63,7 @@ export const getUsersFromAuth0 = async (users: { id: number; auth0Id: string }[] // Search for users on Auth0 const auth0Ids = users.map(({ auth0Id }) => auth0Id); - const auth0Users = await auth0.getUsers({ + const auth0Users = await getAuth0().getUsers({ q: `user_id:(${auth0Ids.join(' OR ')})`, }); @@ -103,7 +111,7 @@ export const getUsersFromAuth0 = async (users: { id: number; auth0Id: string }[] }; export const lookupUsersFromAuth0ByEmail = async (email: string): Promise => { - const auth0Users = await auth0.getUsersByEmail(email); + const auth0Users = await getAuth0().getUsersByEmail(email); return auth0Users; }; diff --git a/quadratic-api/src/storage/s3.ts b/quadratic-api/src/storage/s3.ts index fc5f2bf52e..4fe6db1b23 100644 --- a/quadratic-api/src/storage/s3.ts +++ b/quadratic-api/src/storage/s3.ts @@ -13,17 +13,24 @@ import { import { UploadFileResponse } from './storage'; const endpoint = AWS_S3_ENDPOINT; +let s3Client: S3Client; -// Initialize S3 client -export const s3Client = new S3Client({ - region: AWS_S3_REGION, - credentials: { - accessKeyId: AWS_S3_ACCESS_KEY_ID, - secretAccessKey: AWS_S3_SECRET_ACCESS_KEY, - }, - endpoint, - forcePathStyle: true, -}); +// Get S3 client slngleton +const getS3Client = () => { + if (!s3Client) { + s3Client = new S3Client({ + region: AWS_S3_REGION, + credentials: { + accessKeyId: AWS_S3_ACCESS_KEY_ID, + secretAccessKey: AWS_S3_SECRET_ACCESS_KEY, + }, + endpoint, + forcePathStyle: true, + }); + } + + return s3Client; +}; // Upload a string as a file to S3 export const uploadStringAsFileS3 = async (fileKey: string, contents: string): Promise => { @@ -34,7 +41,7 @@ export const uploadStringAsFileS3 = async (fileKey: string, contents: string): P // Optionally, you can add other configuration like ContentType // ContentType: 'text/plain' }); - const response = await s3Client.send(command); + const response = await getS3Client().send(command); // Check if the upload was successful if (response && response.$metadata.httpStatusCode === 200) { @@ -48,19 +55,20 @@ export const uploadStringAsFileS3 = async (fileKey: string, contents: string): P }; // Multer storage engine for S3 -export const multerS3Storage: multer.Multer = multer({ - storage: multerS3({ - s3: s3Client, - bucket: AWS_S3_BUCKET_NAME, - metadata: (req: Request, file: Express.Multer.File, cb: (error: Error | null, metadata: any) => void) => { - cb(null, { fieldName: file.fieldname }); - }, - key: (req: Request, file: Express.Multer.File, cb: (error: Error | null, key: string) => void) => { - const fileUuid = req.params.uuid; - cb(null, `${fileUuid}-${file.originalname}`); - }, - }) as StorageEngine, -}); +export const multerS3Storage = (): multer.Multer => + multer({ + storage: multerS3({ + s3: getS3Client(), + bucket: AWS_S3_BUCKET_NAME, + metadata: (req: Request, file: Express.Multer.File, cb: (error: Error | null, metadata: any) => void) => { + cb(null, { fieldName: file.fieldname }); + }, + key: (req: Request, file: Express.Multer.File, cb: (error: Error | null, key: string) => void) => { + const fileUuid = req.params.uuid; + cb(null, `${fileUuid}-${file.originalname}`); + }, + }) as StorageEngine, + }); // Get the presigned file URL from S3 export const generatePresignedUrl = async (key: string): Promise => { diff --git a/quadratic-api/src/storage/storage.ts b/quadratic-api/src/storage/storage.ts index 88444e98c7..0960194079 100644 --- a/quadratic-api/src/storage/storage.ts +++ b/quadratic-api/src/storage/storage.ts @@ -1,19 +1,7 @@ import multer from 'multer'; import { STORAGE_TYPE } from '../env-vars'; - -let { getPresignedStorageUrl, getStorageUrl, multerFileSystemStorage, upload } = {} as any; -let { generatePresignedUrl, multerS3Storage, uploadStringAsFileS3 } = {} as any; - -switch (STORAGE_TYPE) { - case 's3': - ({ generatePresignedUrl, multerS3Storage, uploadStringAsFileS3 } = require('./s3')); - break; - case 'file-system': - ({ getPresignedStorageUrl, getStorageUrl, multerFileSystemStorage, upload } = require('./fileSystem')); - break; - default: - throw new Error(`Unsupported storage type in storage.ts: ${STORAGE_TYPE}`); -} +import { getPresignedStorageUrl, getStorageUrl, multerFileSystemStorage, upload } from './fileSystem'; +import { generatePresignedUrl, multerS3Storage, uploadStringAsFileS3 } from './s3'; export type UploadFileResponse = { bucket: string; @@ -60,7 +48,7 @@ export const uploadFile = async (key: string, contents: string, jwt: string): Pr export const uploadMiddleware = (): multer.Multer => { switch (STORAGE_TYPE) { case 's3': - return multerS3Storage; + return multerS3Storage(); case 'file-system': return multerFileSystemStorage as unknown as multer.Multer; default: From 9a3b5375402831f2794bb937370bea7d701a9432 Mon Sep 17 00:00:00 2001 From: David DiMaria Date: Wed, 28 Aug 2024 14:37:36 -0600 Subject: [PATCH 052/113] Hard code LICENSE_API_URI --- quadratic-api/.env.docker | 1 - quadratic-api/.env.example | 1 - quadratic-api/.env.test | 1 - quadratic-api/src/env-vars.ts | 8 ++++---- self-hosting/README.md | 6 +++++- self-hosting/docker-compose.yml | 1 - 6 files changed, 9 insertions(+), 9 deletions(-) diff --git a/quadratic-api/.env.docker b/quadratic-api/.env.docker index 6f404d6180..471e456797 100644 --- a/quadratic-api/.env.docker +++ b/quadratic-api/.env.docker @@ -21,4 +21,3 @@ QUADRATIC_FILE_URI_PUBLIC=http://localhost:3002 # Admin LICENSE_KEY=LICENSE_KEY -LICENSE_API_URI=https://selfhost.quadratic-preview.com diff --git a/quadratic-api/.env.example b/quadratic-api/.env.example index 9cff5fe189..d0eb326f85 100644 --- a/quadratic-api/.env.example +++ b/quadratic-api/.env.example @@ -37,4 +37,3 @@ AWS_S3_BUCKET_NAME=quadratic-api-docker # Admin LICENSE_KEY=LICENSE_KEY -LICENSE_API_URI=https://selfhost.quadratic-preview.com diff --git a/quadratic-api/.env.test b/quadratic-api/.env.test index 426cfeefc7..9802401921 100644 --- a/quadratic-api/.env.test +++ b/quadratic-api/.env.test @@ -28,4 +28,3 @@ AWS_S3_ENDPOINT=http://0.0.0.0:4566 # Admin LICENSE_KEY=LICENSE_KEY -LICENSE_API_URI=https://selfhost.quadratic-preview.com diff --git a/quadratic-api/src/env-vars.ts b/quadratic-api/src/env-vars.ts index ecb2662239..6598e6e6db 100644 --- a/quadratic-api/src/env-vars.ts +++ b/quadratic-api/src/env-vars.ts @@ -31,10 +31,7 @@ export const ENCRYPTION_KEY = process.env.ENCRYPTION_KEY as string; export const STORAGE_TYPE = process.env.STORAGE_TYPE as string; export const AUTH_TYPE = process.env.AUTH_TYPE as string; export const LICENSE_KEY = process.env.LICENSE_KEY as string; -export const LICENSE_API_URI = process.env.LICENSE_API_URI as string; -['STRIPE_SECRET_KEY', 'ENCRYPTION_KEY', 'STORAGE_TYPE', 'AUTH_TYPE', 'LICENSE_KEY', 'LICENSE_API_URI'].forEach( - ensureEnvVarExists -); +['STRIPE_SECRET_KEY', 'ENCRYPTION_KEY', 'STORAGE_TYPE', 'AUTH_TYPE', 'LICENSE_KEY'].forEach(ensureEnvVarExists); // Required in prod, optional locally export const M2M_AUTH_TOKEN = process.env.M2M_AUTH_TOKEN; @@ -46,6 +43,9 @@ if (NODE_ENV === 'production') { ['M2M_AUTH_TOKEN', 'OPENAI_API_KEY', 'SLACK_FEEDBACK_URL'].forEach(ensureEnvVarExists); } +// Intentionally hard-coded to avoid this being environment-configurable +export const LICENSE_API_URI = 'https://selfhost.quadratic-preview.com/'; + ensureSampleTokenNotUsedInProduction(); function ensureEnvVarExists(key: string) { diff --git a/self-hosting/README.md b/self-hosting/README.md index 48502968ca..6ce73b9ca5 100644 --- a/self-hosting/README.md +++ b/self-hosting/README.md @@ -32,7 +32,7 @@ This will download the initialization script, which will prompt for a license ke Additionally, the docker compose network will start (see [Starting](#Starting)). Please allow several minutes for the docker images to downloaded. -Refer to the [Closing](#Closing) section. +Refer to the [Stopping](#Stopping) section. ## Starting @@ -41,3 +41,7 @@ Once the Quadratic is initialized, a single command is needed to start all of th ```shell ./start.sh ``` + +## Stopping + +To stop running docker images, simply press `ctrl + c` diff --git a/self-hosting/docker-compose.yml b/self-hosting/docker-compose.yml index b532b396df..a2957aacbb 100644 --- a/self-hosting/docker-compose.yml +++ b/self-hosting/docker-compose.yml @@ -108,7 +108,6 @@ services: # Admin LICENSE_KEY: "LICENSE_KEY" - LICENSE_API_URI: http://localhost:4000 restart: "always" ports: - "8000:8000" From 1688d1a399b88f89e330d2ed3e71885c5add09b7 Mon Sep 17 00:00:00 2001 From: David DiMaria Date: Wed, 28 Aug 2024 14:57:11 -0600 Subject: [PATCH 053/113] Fix axios headers --- quadratic-api/src/storage/fileSystem.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/quadratic-api/src/storage/fileSystem.ts b/quadratic-api/src/storage/fileSystem.ts index 9efd530159..d43f56eb4a 100644 --- a/quadratic-api/src/storage/fileSystem.ts +++ b/quadratic-api/src/storage/fileSystem.ts @@ -35,8 +35,7 @@ export const upload = async (key: string, contents: string | Uint8Array, jwt: st try { const response = await axios - .post(url, { - body: contents, + .post(url, contents, { headers: { 'Content-Type': 'text/plain', Authorization: `${jwt}`, From 2f4bb84afc0ccf853666c865b10343a477a79012 Mon Sep 17 00:00:00 2001 From: David DiMaria Date: Thu, 29 Aug 2024 07:37:57 -0600 Subject: [PATCH 054/113] Fix typo in init.sh --- self-hosting/README.md | 2 +- self-hosting/docker/postgres/scripts/init.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/self-hosting/README.md b/self-hosting/README.md index 6ce73b9ca5..5d25f93c7b 100644 --- a/self-hosting/README.md +++ b/self-hosting/README.md @@ -39,7 +39,7 @@ Refer to the [Stopping](#Stopping) section. Once the Quadratic is initialized, a single command is needed to start all of the images: ```shell -./start.sh +./quadratic/self-hosting/.start.sh ``` ## Stopping diff --git a/self-hosting/docker/postgres/scripts/init.sh b/self-hosting/docker/postgres/scripts/init.sh index 49102ba731..5e5b12df77 100755 --- a/self-hosting/docker/postgres/scripts/init.sh +++ b/self-hosting/docker/postgres/scripts/init.sh @@ -13,6 +13,6 @@ function create_user_and_database() { if [ -n "$ADDITIONAL_DATABASES" ]; then for i in ${ADDITIONAL_DATABASES//,/ } do - create_user_and_database $1 + create_user_and_database $i done fi From 81f4594c461c522bc263c3f1f3916eee260f3619 Mon Sep 17 00:00:00 2001 From: David DiMaria Date: Thu, 29 Aug 2024 09:37:41 -0600 Subject: [PATCH 055/113] Add more info to self-hosting README --- self-hosting/README.md | 11 +++++++++-- self-hosting/docker-compose.yml | 2 +- self-hosting/stop.sh | 7 +++++++ 3 files changed, 17 insertions(+), 3 deletions(-) create mode 100755 self-hosting/stop.sh diff --git a/self-hosting/README.md b/self-hosting/README.md index 5d25f93c7b..5dbccfd81c 100644 --- a/self-hosting/README.md +++ b/self-hosting/README.md @@ -17,6 +17,7 @@ Implement the entire Quadratic stack outside of Quadratic. The use cases we cur * MacOS or Linux (not tested on Windows) * License Key (available at https://selfhost.quadratic-preview.com) +* The following open ports: 80, 3000, 3001, 3002, 8000 ## Installation @@ -39,9 +40,15 @@ Refer to the [Stopping](#Stopping) section. Once the Quadratic is initialized, a single command is needed to start all of the images: ```shell -./quadratic/self-hosting/.start.sh +./quadratic/self-hosting/start.sh ``` ## Stopping -To stop running docker images, simply press `ctrl + c` +To stop running docker images, simply press `ctrl + c` if running in the foreground. + +If running in the background, run the `stop.sh` script: + +```shell +./quadratic/self-hosting/stop.sh +``` diff --git a/self-hosting/docker-compose.yml b/self-hosting/docker-compose.yml index a2957aacbb..f9c984e201 100644 --- a/self-hosting/docker-compose.yml +++ b/self-hosting/docker-compose.yml @@ -106,7 +106,7 @@ services: QUADRATIC_FILE_URI: http://host.docker.internal:3002 QUADRATIC_FILE_URI_PUBLIC: http://localhost:3002 - # Admin + # License UUID: leave "LICENSE_KEY" as this value is replaced during initialization LICENSE_KEY: "LICENSE_KEY" restart: "always" ports: diff --git a/self-hosting/stop.sh b/self-hosting/stop.sh new file mode 100755 index 0000000000..f47c32c3f6 --- /dev/null +++ b/self-hosting/stop.sh @@ -0,0 +1,7 @@ +#!/bin/sh + +stop() { + docker compose --profile "*" down +} + +stop \ No newline at end of file From c848618dc1c432efac8d35da87c05fbaea3c65e2 Mon Sep 17 00:00:00 2001 From: David DiMaria Date: Thu, 29 Aug 2024 12:54:37 -0600 Subject: [PATCH 056/113] Fix all broken things from the merge and beyond --- quadratic-api/src/routes/v0/examples.POST.ts | 7 +++++-- quadratic-api/src/routes/v0/files.$uuid.GET.ts | 2 -- quadratic-api/src/routes/v0/teams.$uuid.invites.POST.ts | 2 +- quadratic-client/src/routes/file.$uuid.tsx | 2 -- quadratic-client/src/shared/api/xhrFromApi.ts | 2 +- quadratic-client/src/shared/utils/auth0UserImageSrc.ts | 2 +- 6 files changed, 8 insertions(+), 9 deletions(-) diff --git a/quadratic-api/src/routes/v0/examples.POST.ts b/quadratic-api/src/routes/v0/examples.POST.ts index 0e77a136fa..dc40cdc36c 100644 --- a/quadratic-api/src/routes/v0/examples.POST.ts +++ b/quadratic-api/src/routes/v0/examples.POST.ts @@ -1,3 +1,4 @@ +import axios from 'axios'; import { Response } from 'express'; import { ApiSchemas, ApiTypes } from 'quadratic-shared/typesAndSchemas'; import z from 'zod'; @@ -48,10 +49,12 @@ async function handler(req: RequestWithUser, res: Response res.json())) as ApiTypes['/v0/files/:uuid.GET.response']; + } = (await axios.get(apiUrl).then((res) => res.data)) as ApiTypes['/v0/files/:uuid.GET.response']; // Fetch the contents of the file - const fileContents = await fetch(lastCheckpointDataUrl).then((res) => res.arrayBuffer()); + const fileContents = await axios + .get(lastCheckpointDataUrl, { responseType: 'arraybuffer' }) + .then((res) => res.data); const buffer = new Uint8Array(fileContents); // Create a private file for the user in the requested team diff --git a/quadratic-api/src/routes/v0/files.$uuid.GET.ts b/quadratic-api/src/routes/v0/files.$uuid.GET.ts index c208bd98f4..9a9c960a9e 100644 --- a/quadratic-api/src/routes/v0/files.$uuid.GET.ts +++ b/quadratic-api/src/routes/v0/files.$uuid.GET.ts @@ -68,8 +68,6 @@ async function handler( return res.status(500).json({ error: { message: 'Unable to retrieve license' } }); } - console.log('license', license); - const data = { file: { uuid, diff --git a/quadratic-api/src/routes/v0/teams.$uuid.invites.POST.ts b/quadratic-api/src/routes/v0/teams.$uuid.invites.POST.ts index dc7661dc8b..2e3866304f 100644 --- a/quadratic-api/src/routes/v0/teams.$uuid.invites.POST.ts +++ b/quadratic-api/src/routes/v0/teams.$uuid.invites.POST.ts @@ -151,7 +151,6 @@ async function handler(req: RequestWithUser, res: Response Date: Thu, 29 Aug 2024 14:10:01 -0600 Subject: [PATCH 057/113] Remove container_name from postgres and caddy --- self-hosting/docker-compose.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/self-hosting/docker-compose.yml b/self-hosting/docker-compose.yml index f9c984e201..debf44b22b 100644 --- a/self-hosting/docker-compose.yml +++ b/self-hosting/docker-compose.yml @@ -18,7 +18,6 @@ services: postgres: image: postgres:15 restart: always - container_name: postgres ports: - "5432:5432" environment: @@ -39,7 +38,6 @@ services: caddy: image: caddy:latest - container_name: caddy ports: - "80:80" - "443:443" From e20a821e40ec867b54fe61def4ca7aeebb7a8187 Mon Sep 17 00:00:00 2001 From: David DiMaria Date: Fri, 30 Aug 2024 12:06:23 -0600 Subject: [PATCH 058/113] Improve init experience for self-hosting --- self-hosting/init.sh | 37 ++++++++++++++++++++++++++++--------- 1 file changed, 28 insertions(+), 9 deletions(-) diff --git a/self-hosting/init.sh b/self-hosting/init.sh index 471bc7219b..752a6f9290 100755 --- a/self-hosting/init.sh +++ b/self-hosting/init.sh @@ -22,18 +22,37 @@ checkout() { cd quadratic git sparse-checkout set ${DIR}/ git checkout - cd $DIR } -LICENSE_KEY=$(get_license_key) +LICENSE_KEY="" -if [ "$LICENSE_KEY" = "$INVALID_LICENSE_KEY" ]; then - echo $INVALID_LICENSE_KEY -else - checkout +# check if LICENSE file exists +if ! [ -f "quadratic/LICENSE" ]; then + LICENSE_KEY=$(get_license_key) + + if [ "$LICENSE_KEY" = "$INVALID_LICENSE_KEY" ]; then + echo $INVALID_LICENSE_KEY + else + # retrieve the code from github + checkout + + # write license key to LICENSE file + touch LICENSE + echo $LICENSE_KEY > LICENSE + + cp -a self-hosting/. . + rm -rf self-hosting + rm ../init.sh + rm init.sh - sed -i '' "s/\"LICENSE_KEY\"/\"$LICENSE_KEY\"/g" "docker-compose.yml" - - sh start.sh + sed -i '' "s/\"LICENSE_KEY\"/\"$LICENSE_KEY\"/g" "docker-compose.yml" + + cd quadratic + fi +else + cd quadratic + LICENSE_KEY=$( Date: Fri, 30 Aug 2024 12:33:33 -0600 Subject: [PATCH 059/113] Update init.sh link --- self-hosting/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/self-hosting/README.md b/self-hosting/README.md index 5dbccfd81c..4f63295dc4 100644 --- a/self-hosting/README.md +++ b/self-hosting/README.md @@ -26,7 +26,7 @@ Implement the entire Quadratic stack outside of Quadratic. The use cases we cur Quadratic can be installed via a single command: ```shell -curl -sSf https://raw.githubusercontent.com/quadratichq/quadratic/27714636745e6c68e5d0412e2d0eafa16167aa30/self-hosting/init.sh -o init.sh && sh -i init.sh +curl -sSf https://raw.githubusercontent.com/quadratichq/quadratic/e20a821e40ec867b54fe61def4ca7aeebb7a8187/self-hosting/init.sh -o init.sh && sh -i init.sh ``` This will download the initialization script, which will prompt for a license key in order to register Quadratic. From a8575307c426df5fc0e3fe1170def01af0c59618 Mon Sep 17 00:00:00 2001 From: David DiMaria Date: Fri, 30 Aug 2024 13:37:34 -0600 Subject: [PATCH 060/113] Update init.sh link --- self-hosting/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/self-hosting/README.md b/self-hosting/README.md index 4f63295dc4..9184d67491 100644 --- a/self-hosting/README.md +++ b/self-hosting/README.md @@ -26,7 +26,7 @@ Implement the entire Quadratic stack outside of Quadratic. The use cases we cur Quadratic can be installed via a single command: ```shell -curl -sSf https://raw.githubusercontent.com/quadratichq/quadratic/e20a821e40ec867b54fe61def4ca7aeebb7a8187/self-hosting/init.sh -o init.sh && sh -i init.sh +curl -sSf https://raw.githubusercontent.com/quadratichq/quadratic/2fdb1ede0db8e7d4784792643f32bc773aa0f231/self-hosting/init.sh -o init.sh && sh -i init.sh ``` This will download the initialization script, which will prompt for a license key in order to register Quadratic. From e8d4d5dbc9c1185d4125e441657e972afcb55c29 Mon Sep 17 00:00:00 2001 From: David DiMaria Date: Fri, 30 Aug 2024 14:12:04 -0600 Subject: [PATCH 061/113] Migrate database on startup --- self-hosting/docker-compose.yml | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/self-hosting/docker-compose.yml b/self-hosting/docker-compose.yml index debf44b22b..c69a3a1937 100644 --- a/self-hosting/docker-compose.yml +++ b/self-hosting/docker-compose.yml @@ -305,18 +305,6 @@ services: networks: - host - admin: - image: stoplight/prism:latest - command: "mock -p 4000 -h 0.0.0.0 /tmp/openapi.yaml" - ports: - - 4000:4000 - volumes: - - ./docker/admin/config/openapi.yaml:/tmp/openapi.yaml:ro - profiles: - - admin - networks: - - host - volumes: docker: name: docker From 067b0838cd9cc432ef0fabd45623354ea0daf378 Mon Sep 17 00:00:00 2001 From: David DiMaria Date: Fri, 30 Aug 2024 14:14:02 -0600 Subject: [PATCH 062/113] Migrate database on startup v2 --- self-hosting/docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/self-hosting/docker-compose.yml b/self-hosting/docker-compose.yml index c69a3a1937..09cf263aa7 100644 --- a/self-hosting/docker-compose.yml +++ b/self-hosting/docker-compose.yml @@ -109,7 +109,7 @@ services: restart: "always" ports: - "8000:8000" - command: "npm run start:prod --workspace=quadratic-api" + command: bash -c "npx prisma migrate deploy --schema quadratic-api/prisma/schema.prisma && npm run start:prod --workspace=quadratic-api" depends_on: postgres: condition: service_healthy From 065f531da6d361bce1e1e9070debdd8e50a7473f Mon Sep 17 00:00:00 2001 From: David DiMaria Date: Fri, 30 Aug 2024 14:47:31 -0600 Subject: [PATCH 063/113] Redirect to login-result after kratos login --- self-hosting/docker/ory-auth/config/kratos.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/self-hosting/docker/ory-auth/config/kratos.yml b/self-hosting/docker/ory-auth/config/kratos.yml index 2be60c2bc5..16dac51f60 100644 --- a/self-hosting/docker/ory-auth/config/kratos.yml +++ b/self-hosting/docker/ory-auth/config/kratos.yml @@ -71,7 +71,7 @@ selfservice: ui_url: http://localhost:4455/verification use: code after: - default_browser_return_url: http://localhost + default_browser_return_url: http://localhost/login-result logout: after: From b31bec37b84ed4ea01afea53814ae34a1c190f4d Mon Sep 17 00:00:00 2001 From: David DiMaria Date: Fri, 30 Aug 2024 14:57:31 -0600 Subject: [PATCH 064/113] Redirect to login-result after kratos registration --- self-hosting/docker/ory-auth/config/kratos.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/self-hosting/docker/ory-auth/config/kratos.yml b/self-hosting/docker/ory-auth/config/kratos.yml index 16dac51f60..897077450c 100644 --- a/self-hosting/docker/ory-auth/config/kratos.yml +++ b/self-hosting/docker/ory-auth/config/kratos.yml @@ -89,6 +89,7 @@ selfservice: hooks: - hook: session - hook: show_verification_ui + default_browser_return_url: http://localhost/login-result session: whoami: From 6c4c62b58ae55251cb16d95b0076af47ed209cc8 Mon Sep 17 00:00:00 2001 From: David DiMaria Date: Tue, 3 Sep 2024 12:22:13 -0600 Subject: [PATCH 065/113] Use bash over sh in the init.sh script --- self-hosting/init.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/self-hosting/init.sh b/self-hosting/init.sh index 752a6f9290..d28937366c 100755 --- a/self-hosting/init.sh +++ b/self-hosting/init.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash REPO="https://github.com/quadratichq/quadratic.git" BRANCH="self-hosting-setup" From 86e1cf6b1a62e96c1bfab55177cf246927753871 Mon Sep 17 00:00:00 2001 From: David DiMaria Date: Tue, 3 Sep 2024 12:44:00 -0600 Subject: [PATCH 066/113] Update new init.sh link --- self-hosting/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/self-hosting/README.md b/self-hosting/README.md index 9184d67491..9a825cdd2b 100644 --- a/self-hosting/README.md +++ b/self-hosting/README.md @@ -26,7 +26,7 @@ Implement the entire Quadratic stack outside of Quadratic. The use cases we cur Quadratic can be installed via a single command: ```shell -curl -sSf https://raw.githubusercontent.com/quadratichq/quadratic/2fdb1ede0db8e7d4784792643f32bc773aa0f231/self-hosting/init.sh -o init.sh && sh -i init.sh +curl -sSf https://raw.githubusercontent.com/quadratichq/quadratic/6c4c62b58ae55251cb16d95b0076af47ed209cc8/self-hosting/init.sh -o init.sh && sh -i init.sh ``` This will download the initialization script, which will prompt for a license key in order to register Quadratic. From c3e246148aaf8fcb5dd155162fb3dce546044349 Mon Sep 17 00:00:00 2001 From: David DiMaria Date: Tue, 3 Sep 2024 12:56:16 -0600 Subject: [PATCH 067/113] Add Installing on Ubuntu instructions --- self-hosting/README.md | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/self-hosting/README.md b/self-hosting/README.md index 9a825cdd2b..3c12f82ba2 100644 --- a/self-hosting/README.md +++ b/self-hosting/README.md @@ -26,7 +26,7 @@ Implement the entire Quadratic stack outside of Quadratic. The use cases we cur Quadratic can be installed via a single command: ```shell -curl -sSf https://raw.githubusercontent.com/quadratichq/quadratic/6c4c62b58ae55251cb16d95b0076af47ed209cc8/self-hosting/init.sh -o init.sh && sh -i init.sh +curl -sSf https://raw.githubusercontent.com/quadratichq/quadratic/6c4c62b58ae55251cb16d95b0076af47ed209cc8/self-hosting/init.sh -o init.sh && bash -i init.sh ``` This will download the initialization script, which will prompt for a license key in order to register Quadratic. @@ -52,3 +52,21 @@ If running in the background, run the `stop.sh` script: ```shell ./quadratic/self-hosting/stop.sh ``` + +## Installing on Ubuntu + +```shell +sudo apt-get update +sudo apt-get install apt-transport-https ca-certificates curl software-properties-common +curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg +echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null +sudo apt-get update +sudo apt-get -y install docker-ce docker-ce-cli containerd.io +sudo docker --version +sudo curl -L "https://github.com/docker/compose/releases/download/v2.21.0/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose +sudo chmod +x /usr/local/bin/docker-compose +sudo chown $USER /var/run/docker.sock +sudo systemctl enable docker +sudo systemctl start docker +curl -sSf https://raw.githubusercontent.com/quadratichq/quadratic/6c4c62b58ae55251cb16d95b0076af47ed209cc8/self-hosting/init.sh -o init.sh && bash -i init.sh +``` From 5ca28b9b7a7520330ecff4bbe94ffe079cc92e44 Mon Sep 17 00:00:00 2001 From: David DiMaria Date: Tue, 3 Sep 2024 13:05:03 -0600 Subject: [PATCH 068/113] Prefer 127.0.0.1 over host.docker.internal --- self-hosting/docker-compose.yml | 32 +++++++++---------- .../docker/ory-auth/config/kratos.yml | 4 +-- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/self-hosting/docker-compose.yml b/self-hosting/docker-compose.yml index 09cf263aa7..7e6f17a168 100644 --- a/self-hosting/docker-compose.yml +++ b/self-hosting/docker-compose.yml @@ -67,7 +67,7 @@ services: sh -c "/client/scripts/replace_env_vars.sh && nginx -g \"daemon off;\"" healthcheck: - test: ["CMD-SHELL", "curl -f http://host.docker.internal:3000/ || exit 1"] + test: ["CMD-SHELL", "curl -f http://127.0.0.1:3000/ || exit 1"] interval: 10s timeout: 5s volumes: @@ -84,7 +84,7 @@ services: image: public.ecr.aws/l3i4i9z2/quadratic-api-staging:latest environment: CORS: "*" - DATABASE_URL: "postgresql://postgres:postgres@host.docker.internal:5432/postgres" + DATABASE_URL: "postgresql://postgres:postgres@127.0.0.1:5432/postgres" ENVIRONMENT: docker STRIPE_SECRET_KEY: STRIPE_SECRET_KEY STRIPE_WEBHOOK_SECRET: STRIPE_WEBHOOK_SECRET @@ -96,12 +96,12 @@ services: # Auth AUTH_TYPE: ory - ORY_JWKS_URI: "http://host.docker.internal:3000/.well-known/jwks.json" - ORY_ADMIN_HOST: http://host.docker.internal:4434 + ORY_JWKS_URI: "http://127.0.0.1:3000/.well-known/jwks.json" + ORY_ADMIN_HOST: http://127.0.0.1:4434 # Storage STORAGE_TYPE: file-system - QUADRATIC_FILE_URI: http://host.docker.internal:3002 + QUADRATIC_FILE_URI: http://127.0.0.1:3002 QUADRATIC_FILE_URI_PUBLIC: http://localhost:3002 # License UUID: leave "LICENSE_KEY" as this value is replaced during initialization @@ -127,16 +127,16 @@ services: PORT: 3001 HEARTBEAT_CHECK_S: 3 HEARTBEAT_TIMEOUT_S: 600 - QUADRATIC_API_URI: http://host.docker.internal:8000 + QUADRATIC_API_URI: http://127.0.0.1:8000 M2M_AUTH_TOKEN: M2M_AUTH_TOKEN ENVIRONMENT: docker - PUBSUB_HOST: host.docker.internal + PUBSUB_HOST: 127.0.0.1 PUBSUB_PORT: 6379 PUBSUB_PASSWORD: "" PUBSUB_ACTIVE_CHANNELS: active_channels - AUTH0_JWKS_URI: http://host.docker.internal:3000/.well-known/jwks.json + AUTH0_JWKS_URI: http://127.0.0.1:3000/.well-known/jwks.json AUTHENTICATE_JWT: true restart: "always" ports: @@ -168,11 +168,11 @@ services: TRUNCATE_TRANSACTION_AGE_DAYS: 5 # ENVIRONMENT: docker - AUTH0_JWKS_URI: http://host.docker.internal:3000/.well-known/jwks.json - QUADRATIC_API_URI: http://host.docker.internal:8000 + AUTH0_JWKS_URI: http://127.0.0.1:3000/.well-known/jwks.json + QUADRATIC_API_URI: http://127.0.0.1:8000 M2M_AUTH_TOKEN: M2M_AUTH_TOKEN - PUBSUB_HOST: host.docker.internal + PUBSUB_HOST: 127.0.0.1 PUBSUB_PORT: 6379 PUBSUB_PASSWORD: "" PUBSUB_ACTIVE_CHANNELS: active_channels @@ -219,8 +219,8 @@ services: PORT: 3003 ENVIRONMENT: docker - AUTH0_JWKS_URI: http://host.docker.internal:3000/.well-known/jwks.json - QUADRATIC_API_URI: http://host.docker.internal:8000 + AUTH0_JWKS_URI: http://127.0.0.1:3000/.well-known/jwks.json + QUADRATIC_API_URI: http://127.0.0.1:8000 M2M_AUTH_TOKEN: M2M_AUTH_TOKEN MAX_RESPONSE_BYTES: 15728640 # 15MB STATIC_IPS: 0.0.0.0,127.0.0.1 @@ -250,7 +250,7 @@ services: volumes: - ./docker/ory-auth/config:/etc/config/kratos environment: - DSN: postgresql://postgres:postgres@host.docker.internal:5432/kratos?sslmode=disable + DSN: postgresql://postgres:postgres@127.0.0.1:5432/kratos?sslmode=disable LOG_LEVEL: trace restart: unless-stopped depends_on: @@ -267,7 +267,7 @@ services: volumes: - ./docker/ory-auth/config:/etc/config/kratos environment: - DSN: postgresql://postgres:postgres@host.docker.internal:5432/kratos?sslmode=disable + DSN: postgresql://postgres:postgres@127.0.0.1:5432/kratos?sslmode=disable restart: on-failure depends_on: - postgres @@ -282,7 +282,7 @@ services: - "4455:4455" environment: PORT: 4455 - KRATOS_PUBLIC_URL: http://host.docker.internal:4433/ + KRATOS_PUBLIC_URL: http://127.0.0.1:4433/ KRATOS_BROWSER_URL: http://localhost:4433/ COOKIE_SECRET: changeme CSRF_COOKIE_NAME: __HOST-localhost-x-csrf-token diff --git a/self-hosting/docker/ory-auth/config/kratos.yml b/self-hosting/docker/ory-auth/config/kratos.yml index 897077450c..c87ca7b23a 100644 --- a/self-hosting/docker/ory-auth/config/kratos.yml +++ b/self-hosting/docker/ory-auth/config/kratos.yml @@ -96,7 +96,7 @@ session: tokenizer: templates: jwt_template: - jwks_url: http://host.docker.internal:3000/.well-known/jwks.json + jwks_url: http://127.0.0.1:3000/.well-known/jwks.json # claims_mapper_url: base64://... # A JsonNet template for modifying the claims ttl: 24h # 24 hours (defaults to 10 minutes) cookies: @@ -132,7 +132,7 @@ identity: courier: smtp: - connection_uri: smtps://test:test@host.docker.internal:1025/?skip_ssl_verify=true + connection_uri: smtps://test:test@127.0.0.1:1025/?skip_ssl_verify=true feature_flags: use_continue_with_transitions: true \ No newline at end of file From c6ebfd2a37614cd21ba55fecac6ecf00d921609d Mon Sep 17 00:00:00 2001 From: David DiMaria Date: Tue, 3 Sep 2024 13:07:48 -0600 Subject: [PATCH 069/113] Prefer 0.0.0.0 over 127.0.0.1 for self hosting --- self-hosting/docker-compose.yml | 32 +++++++++---------- .../docker/ory-auth/config/kratos.yml | 4 +-- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/self-hosting/docker-compose.yml b/self-hosting/docker-compose.yml index 7e6f17a168..a2e188c2d0 100644 --- a/self-hosting/docker-compose.yml +++ b/self-hosting/docker-compose.yml @@ -67,7 +67,7 @@ services: sh -c "/client/scripts/replace_env_vars.sh && nginx -g \"daemon off;\"" healthcheck: - test: ["CMD-SHELL", "curl -f http://127.0.0.1:3000/ || exit 1"] + test: ["CMD-SHELL", "curl -f http://0.0.0.0:3000/ || exit 1"] interval: 10s timeout: 5s volumes: @@ -84,7 +84,7 @@ services: image: public.ecr.aws/l3i4i9z2/quadratic-api-staging:latest environment: CORS: "*" - DATABASE_URL: "postgresql://postgres:postgres@127.0.0.1:5432/postgres" + DATABASE_URL: "postgresql://postgres:postgres@0.0.0.0:5432/postgres" ENVIRONMENT: docker STRIPE_SECRET_KEY: STRIPE_SECRET_KEY STRIPE_WEBHOOK_SECRET: STRIPE_WEBHOOK_SECRET @@ -96,12 +96,12 @@ services: # Auth AUTH_TYPE: ory - ORY_JWKS_URI: "http://127.0.0.1:3000/.well-known/jwks.json" - ORY_ADMIN_HOST: http://127.0.0.1:4434 + ORY_JWKS_URI: "http://0.0.0.0:3000/.well-known/jwks.json" + ORY_ADMIN_HOST: http://0.0.0.0:4434 # Storage STORAGE_TYPE: file-system - QUADRATIC_FILE_URI: http://127.0.0.1:3002 + QUADRATIC_FILE_URI: http://0.0.0.0:3002 QUADRATIC_FILE_URI_PUBLIC: http://localhost:3002 # License UUID: leave "LICENSE_KEY" as this value is replaced during initialization @@ -127,16 +127,16 @@ services: PORT: 3001 HEARTBEAT_CHECK_S: 3 HEARTBEAT_TIMEOUT_S: 600 - QUADRATIC_API_URI: http://127.0.0.1:8000 + QUADRATIC_API_URI: http://0.0.0.0:8000 M2M_AUTH_TOKEN: M2M_AUTH_TOKEN ENVIRONMENT: docker - PUBSUB_HOST: 127.0.0.1 + PUBSUB_HOST: 0.0.0.0 PUBSUB_PORT: 6379 PUBSUB_PASSWORD: "" PUBSUB_ACTIVE_CHANNELS: active_channels - AUTH0_JWKS_URI: http://127.0.0.1:3000/.well-known/jwks.json + AUTH0_JWKS_URI: http://0.0.0.0:3000/.well-known/jwks.json AUTHENTICATE_JWT: true restart: "always" ports: @@ -168,11 +168,11 @@ services: TRUNCATE_TRANSACTION_AGE_DAYS: 5 # ENVIRONMENT: docker - AUTH0_JWKS_URI: http://127.0.0.1:3000/.well-known/jwks.json - QUADRATIC_API_URI: http://127.0.0.1:8000 + AUTH0_JWKS_URI: http://0.0.0.0:3000/.well-known/jwks.json + QUADRATIC_API_URI: http://0.0.0.0:8000 M2M_AUTH_TOKEN: M2M_AUTH_TOKEN - PUBSUB_HOST: 127.0.0.1 + PUBSUB_HOST: 0.0.0.0 PUBSUB_PORT: 6379 PUBSUB_PASSWORD: "" PUBSUB_ACTIVE_CHANNELS: active_channels @@ -219,8 +219,8 @@ services: PORT: 3003 ENVIRONMENT: docker - AUTH0_JWKS_URI: http://127.0.0.1:3000/.well-known/jwks.json - QUADRATIC_API_URI: http://127.0.0.1:8000 + AUTH0_JWKS_URI: http://0.0.0.0:3000/.well-known/jwks.json + QUADRATIC_API_URI: http://0.0.0.0:8000 M2M_AUTH_TOKEN: M2M_AUTH_TOKEN MAX_RESPONSE_BYTES: 15728640 # 15MB STATIC_IPS: 0.0.0.0,127.0.0.1 @@ -250,7 +250,7 @@ services: volumes: - ./docker/ory-auth/config:/etc/config/kratos environment: - DSN: postgresql://postgres:postgres@127.0.0.1:5432/kratos?sslmode=disable + DSN: postgresql://postgres:postgres@0.0.0.0:5432/kratos?sslmode=disable LOG_LEVEL: trace restart: unless-stopped depends_on: @@ -267,7 +267,7 @@ services: volumes: - ./docker/ory-auth/config:/etc/config/kratos environment: - DSN: postgresql://postgres:postgres@127.0.0.1:5432/kratos?sslmode=disable + DSN: postgresql://postgres:postgres@0.0.0.0:5432/kratos?sslmode=disable restart: on-failure depends_on: - postgres @@ -282,7 +282,7 @@ services: - "4455:4455" environment: PORT: 4455 - KRATOS_PUBLIC_URL: http://127.0.0.1:4433/ + KRATOS_PUBLIC_URL: http://0.0.0.0:4433/ KRATOS_BROWSER_URL: http://localhost:4433/ COOKIE_SECRET: changeme CSRF_COOKIE_NAME: __HOST-localhost-x-csrf-token diff --git a/self-hosting/docker/ory-auth/config/kratos.yml b/self-hosting/docker/ory-auth/config/kratos.yml index c87ca7b23a..f7b2f0de62 100644 --- a/self-hosting/docker/ory-auth/config/kratos.yml +++ b/self-hosting/docker/ory-auth/config/kratos.yml @@ -96,7 +96,7 @@ session: tokenizer: templates: jwt_template: - jwks_url: http://127.0.0.1:3000/.well-known/jwks.json + jwks_url: http://0.0.0.0:3000/.well-known/jwks.json # claims_mapper_url: base64://... # A JsonNet template for modifying the claims ttl: 24h # 24 hours (defaults to 10 minutes) cookies: @@ -132,7 +132,7 @@ identity: courier: smtp: - connection_uri: smtps://test:test@127.0.0.1:1025/?skip_ssl_verify=true + connection_uri: smtps://test:test@0.0.0.0:1025/?skip_ssl_verify=true feature_flags: use_continue_with_transitions: true \ No newline at end of file From e425d5aa4f29d70fd9f242c71cdc37515f65fa2a Mon Sep 17 00:00:00 2001 From: David DiMaria Date: Tue, 3 Sep 2024 13:13:48 -0600 Subject: [PATCH 070/113] Add extra_hosts in docker compose for ubuntu --- self-hosting/docker-compose.yml | 52 +++++++++++++------ .../docker/ory-auth/config/kratos.yml | 4 +- 2 files changed, 38 insertions(+), 18 deletions(-) diff --git a/self-hosting/docker-compose.yml b/self-hosting/docker-compose.yml index a2e188c2d0..be5308ec4e 100644 --- a/self-hosting/docker-compose.yml +++ b/self-hosting/docker-compose.yml @@ -50,6 +50,8 @@ services: - frontend networks: - host + extra_hosts: + - "host.docker.internal:host-gateway" quadratic-client: image: public.ecr.aws/l3i4i9z2/quadratic-client-staging:latest @@ -67,7 +69,7 @@ services: sh -c "/client/scripts/replace_env_vars.sh && nginx -g \"daemon off;\"" healthcheck: - test: ["CMD-SHELL", "curl -f http://0.0.0.0:3000/ || exit 1"] + test: ["CMD-SHELL", "curl -f http://host.docker.internal:3000/ || exit 1"] interval: 10s timeout: 5s volumes: @@ -79,12 +81,14 @@ services: - frontend networks: - host + extra_hosts: + - "host.docker.internal:host-gateway" quadratic-api: image: public.ecr.aws/l3i4i9z2/quadratic-api-staging:latest environment: CORS: "*" - DATABASE_URL: "postgresql://postgres:postgres@0.0.0.0:5432/postgres" + DATABASE_URL: "postgresql://postgres:postgres@host.docker.internal:5432/postgres" ENVIRONMENT: docker STRIPE_SECRET_KEY: STRIPE_SECRET_KEY STRIPE_WEBHOOK_SECRET: STRIPE_WEBHOOK_SECRET @@ -96,12 +100,12 @@ services: # Auth AUTH_TYPE: ory - ORY_JWKS_URI: "http://0.0.0.0:3000/.well-known/jwks.json" - ORY_ADMIN_HOST: http://0.0.0.0:4434 + ORY_JWKS_URI: "http://host.docker.internal:3000/.well-known/jwks.json" + ORY_ADMIN_HOST: http://host.docker.internal:4434 # Storage STORAGE_TYPE: file-system - QUADRATIC_FILE_URI: http://0.0.0.0:3002 + QUADRATIC_FILE_URI: http://host.docker.internal:3002 QUADRATIC_FILE_URI_PUBLIC: http://localhost:3002 # License UUID: leave "LICENSE_KEY" as this value is replaced during initialization @@ -118,6 +122,8 @@ services: - frontend networks: - host + extra_hosts: + - "host.docker.internal:host-gateway" quadratic-multiplayer: image: public.ecr.aws/l3i4i9z2/quadratic-multiplayer-staging:latest @@ -127,16 +133,16 @@ services: PORT: 3001 HEARTBEAT_CHECK_S: 3 HEARTBEAT_TIMEOUT_S: 600 - QUADRATIC_API_URI: http://0.0.0.0:8000 + QUADRATIC_API_URI: http://host.docker.internal:8000 M2M_AUTH_TOKEN: M2M_AUTH_TOKEN ENVIRONMENT: docker - PUBSUB_HOST: 0.0.0.0 + PUBSUB_HOST: host.docker.internal PUBSUB_PORT: 6379 PUBSUB_PASSWORD: "" PUBSUB_ACTIVE_CHANNELS: active_channels - AUTH0_JWKS_URI: http://0.0.0.0:3000/.well-known/jwks.json + AUTH0_JWKS_URI: http://host.docker.internal:3000/.well-known/jwks.json AUTHENTICATE_JWT: true restart: "always" ports: @@ -155,6 +161,8 @@ services: - multiplayer networks: - host + extra_hosts: + - "host.docker.internal:host-gateway" quadratic-files: image: public.ecr.aws/l3i4i9z2/quadratic-files-staging:latest @@ -168,11 +176,11 @@ services: TRUNCATE_TRANSACTION_AGE_DAYS: 5 # ENVIRONMENT: docker - AUTH0_JWKS_URI: http://0.0.0.0:3000/.well-known/jwks.json - QUADRATIC_API_URI: http://0.0.0.0:8000 + AUTH0_JWKS_URI: http://host.docker.internal:3000/.well-known/jwks.json + QUADRATIC_API_URI: http://host.docker.internal:8000 M2M_AUTH_TOKEN: M2M_AUTH_TOKEN - PUBSUB_HOST: 0.0.0.0 + PUBSUB_HOST: host.docker.internal PUBSUB_PORT: 6379 PUBSUB_PASSWORD: "" PUBSUB_ACTIVE_CHANNELS: active_channels @@ -210,6 +218,8 @@ services: - files networks: - host + extra_hosts: + - "host.docker.internal:host-gateway" quadratic-connection: image: public.ecr.aws/l3i4i9z2/quadratic-connection-staging:latest @@ -219,8 +229,8 @@ services: PORT: 3003 ENVIRONMENT: docker - AUTH0_JWKS_URI: http://0.0.0.0:3000/.well-known/jwks.json - QUADRATIC_API_URI: http://0.0.0.0:8000 + AUTH0_JWKS_URI: http://host.docker.internal:3000/.well-known/jwks.json + QUADRATIC_API_URI: http://host.docker.internal:8000 M2M_AUTH_TOKEN: M2M_AUTH_TOKEN MAX_RESPONSE_BYTES: 15728640 # 15MB STATIC_IPS: 0.0.0.0,127.0.0.1 @@ -238,6 +248,8 @@ services: - connection networks: - host + extra_hosts: + - "host.docker.internal:host-gateway" # Auth Providers @@ -250,7 +262,7 @@ services: volumes: - ./docker/ory-auth/config:/etc/config/kratos environment: - DSN: postgresql://postgres:postgres@0.0.0.0:5432/kratos?sslmode=disable + DSN: postgresql://postgres:postgres@host.docker.internal:5432/kratos?sslmode=disable LOG_LEVEL: trace restart: unless-stopped depends_on: @@ -260,6 +272,8 @@ services: - ory networks: - host + extra_hosts: + - "host.docker.internal:host-gateway" ory-auth-migrate: image: oryd/kratos:v1.2.0 @@ -267,7 +281,7 @@ services: volumes: - ./docker/ory-auth/config:/etc/config/kratos environment: - DSN: postgresql://postgres:postgres@0.0.0.0:5432/kratos?sslmode=disable + DSN: postgresql://postgres:postgres@host.docker.internal:5432/kratos?sslmode=disable restart: on-failure depends_on: - postgres @@ -275,6 +289,8 @@ services: - ory networks: - host + extra_hosts: + - "host.docker.internal:host-gateway" ory-auth-node: image: oryd/kratos-selfservice-ui-node:v1.2.0 @@ -282,7 +298,7 @@ services: - "4455:4455" environment: PORT: 4455 - KRATOS_PUBLIC_URL: http://0.0.0.0:4433/ + KRATOS_PUBLIC_URL: http://host.docker.internal:4433/ KRATOS_BROWSER_URL: http://localhost:4433/ COOKIE_SECRET: changeme CSRF_COOKIE_NAME: __HOST-localhost-x-csrf-token @@ -292,6 +308,8 @@ services: - ory networks: - host + extra_hosts: + - "host.docker.internal:host-gateway" ory-auth-mail: image: oryd/mailslurper:latest-smtps @@ -304,6 +322,8 @@ services: - ory networks: - host + extra_hosts: + - "host.docker.internal:host-gateway" volumes: docker: diff --git a/self-hosting/docker/ory-auth/config/kratos.yml b/self-hosting/docker/ory-auth/config/kratos.yml index f7b2f0de62..897077450c 100644 --- a/self-hosting/docker/ory-auth/config/kratos.yml +++ b/self-hosting/docker/ory-auth/config/kratos.yml @@ -96,7 +96,7 @@ session: tokenizer: templates: jwt_template: - jwks_url: http://0.0.0.0:3000/.well-known/jwks.json + jwks_url: http://host.docker.internal:3000/.well-known/jwks.json # claims_mapper_url: base64://... # A JsonNet template for modifying the claims ttl: 24h # 24 hours (defaults to 10 minutes) cookies: @@ -132,7 +132,7 @@ identity: courier: smtp: - connection_uri: smtps://test:test@0.0.0.0:1025/?skip_ssl_verify=true + connection_uri: smtps://test:test@host.docker.internal:1025/?skip_ssl_verify=true feature_flags: use_continue_with_transitions: true \ No newline at end of file From 50b8ed5b8aa9a73481a78aaab3a17522cf1f4caa Mon Sep 17 00:00:00 2001 From: David DiMaria Date: Tue, 3 Sep 2024 14:16:03 -0600 Subject: [PATCH 071/113] Remove trailing slash from LICENSE_API_URI --- quadratic-api/src/env-vars.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/quadratic-api/src/env-vars.ts b/quadratic-api/src/env-vars.ts index 6598e6e6db..3754b7ddbf 100644 --- a/quadratic-api/src/env-vars.ts +++ b/quadratic-api/src/env-vars.ts @@ -44,7 +44,7 @@ if (NODE_ENV === 'production') { } // Intentionally hard-coded to avoid this being environment-configurable -export const LICENSE_API_URI = 'https://selfhost.quadratic-preview.com/'; +export const LICENSE_API_URI = 'https://selfhost.quadratic-preview.com'; ensureSampleTokenNotUsedInProduction(); From c4b8f44d61b590d924a341f26489fea46bf76a9a Mon Sep 17 00:00:00 2001 From: David DiMaria Date: Tue, 3 Sep 2024 17:32:54 -0600 Subject: [PATCH 072/113] File perms logging --- quadratic-rust-shared/src/quadratic_api.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/quadratic-rust-shared/src/quadratic_api.rs b/quadratic-rust-shared/src/quadratic_api.rs index ace2f83869..cb498d62a5 100644 --- a/quadratic-rust-shared/src/quadratic_api.rs +++ b/quadratic-rust-shared/src/quadratic_api.rs @@ -69,7 +69,12 @@ pub async fn get_file_perms( ) -> Result<(Vec, u64)> { let url = format!("{base_url}/v0/files/{file_id}"); let client = get_client(&url, &jwt); + tracing::info!("Requesting file perms from quadratic API: {url} {jwt}"); let response = client.send().await?; + tracing::info!( + "Response from file perms from quadratic API:", + response.to_string() + ); handle_response(&response)?; From fbf4b284df5c7cd8a590ff06c1d43b88c02808bb Mon Sep 17 00:00:00 2001 From: David DiMaria Date: Tue, 3 Sep 2024 17:40:28 -0600 Subject: [PATCH 073/113] File perms logging v2 --- quadratic-rust-shared/src/quadratic_api.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/quadratic-rust-shared/src/quadratic_api.rs b/quadratic-rust-shared/src/quadratic_api.rs index cb498d62a5..9df04434e4 100644 --- a/quadratic-rust-shared/src/quadratic_api.rs +++ b/quadratic-rust-shared/src/quadratic_api.rs @@ -72,8 +72,8 @@ pub async fn get_file_perms( tracing::info!("Requesting file perms from quadratic API: {url} {jwt}"); let response = client.send().await?; tracing::info!( - "Response from file perms from quadratic API:", - response.to_string() + "Response from file perms from quadratic API: {:?}", + response ); handle_response(&response)?; From de599cf9e6fca75bdbed2304b9cc60714a4dc4ce Mon Sep 17 00:00:00 2001 From: David DiMaria Date: Tue, 3 Sep 2024 18:31:23 -0600 Subject: [PATCH 074/113] Improve the parseDomain() function --- quadratic-client/src/auth/auth.ts | 23 ++++++++++++++-------- quadratic-rust-shared/src/quadratic_api.rs | 5 ----- 2 files changed, 15 insertions(+), 13 deletions(-) diff --git a/quadratic-client/src/auth/auth.ts b/quadratic-client/src/auth/auth.ts index 9422429e07..fab4351bc3 100644 --- a/quadratic-client/src/auth/auth.ts +++ b/quadratic-client/src/auth/auth.ts @@ -89,15 +89,22 @@ export function waitForAuthClientToRedirect() { * Utility function parse the domain from a url */ export function parseDomain(url: string): string { - // check for classic URLs - let matches = url.match(/([^.]*\.[^.]{2,3}(?:\.[^.]{2,3})?$)/); - - if (matches) { - return '.' + matches[0]; - } else { - // check for IP addresses or localhost (ignore ports) or just return the url - return url.match(/(?:(?!:).)*/)![0] ?? url; + // remove the port if present + const [hostname] = url.split(':'); + + // check if the hostname is an IP address + const isIpAddress = /^[\d.]+$/.test(hostname); + + if (isIpAddress) return hostname; + + const parts = hostname.split('.'); + + // remove subdomain + if (parts.length > 2) { + return parts.slice(-2).join('.'); } + + return hostname; } /** diff --git a/quadratic-rust-shared/src/quadratic_api.rs b/quadratic-rust-shared/src/quadratic_api.rs index 9df04434e4..ace2f83869 100644 --- a/quadratic-rust-shared/src/quadratic_api.rs +++ b/quadratic-rust-shared/src/quadratic_api.rs @@ -69,12 +69,7 @@ pub async fn get_file_perms( ) -> Result<(Vec, u64)> { let url = format!("{base_url}/v0/files/{file_id}"); let client = get_client(&url, &jwt); - tracing::info!("Requesting file perms from quadratic API: {url} {jwt}"); let response = client.send().await?; - tracing::info!( - "Response from file perms from quadratic API: {:?}", - response - ); handle_response(&response)?; From 3cc92988bfb24f57644d2647bcdb5cacc62ef551 Mon Sep 17 00:00:00 2001 From: David DiMaria Date: Tue, 3 Sep 2024 18:36:34 -0600 Subject: [PATCH 075/113] Add CORS headers to nginx --- self-hosting/docker/client/config/default.conf | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/self-hosting/docker/client/config/default.conf b/self-hosting/docker/client/config/default.conf index 1d92083399..cdb53aa4ac 100644 --- a/self-hosting/docker/client/config/default.conf +++ b/self-hosting/docker/client/config/default.conf @@ -14,4 +14,8 @@ server { access_log off; add_header Cache-Control "public, no-transform"; } + + # add CORS headers + add_header Cross-Origin-Opener-Policy "same-origin"; + add_header Cross-Origin-Embedder-Policy "require-corp"; } \ No newline at end of file From c0212b479e91ed19b863119f9303315461294a24 Mon Sep 17 00:00:00 2001 From: David DiMaria Date: Tue, 3 Sep 2024 18:59:24 -0600 Subject: [PATCH 076/113] Add tests for parseDomain --- quadratic-client/src/auth/auth.test.ts | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 quadratic-client/src/auth/auth.test.ts diff --git a/quadratic-client/src/auth/auth.test.ts b/quadratic-client/src/auth/auth.test.ts new file mode 100644 index 0000000000..8785104667 --- /dev/null +++ b/quadratic-client/src/auth/auth.test.ts @@ -0,0 +1,15 @@ +import { describe, expect, it } from 'vitest'; +import { parseDomain } from './auth'; + +describe('auth', () => { + it('parses domains', () => { + const localhostWithPort = parseDomain('localhost:3000'); + expect(localhostWithPort).toBe('localhost'); + + const hasSubdomain = parseDomain('app.quadratichq.com'); + expect(hasSubdomain).toBe('quadratichq.com'); + + const ipAddress = parseDomain('35.161.33.29'); + expect(ipAddress).toBe('35.161.33.29'); + }); +}); From 2cddc1461d138cb187f9b11c3b36c82c3ff18c0a Mon Sep 17 00:00:00 2001 From: David DiMaria Date: Wed, 4 Sep 2024 08:43:43 -0600 Subject: [PATCH 077/113] Hard-code kratos login and registration URLs --- docker/ory-auth/config/kratos.yml | 7 ++++--- quadratic-client/src/auth/ory.ts | 4 ++-- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/docker/ory-auth/config/kratos.yml b/docker/ory-auth/config/kratos.yml index 347b900d1b..37847c04f5 100644 --- a/docker/ory-auth/config/kratos.yml +++ b/docker/ory-auth/config/kratos.yml @@ -62,12 +62,13 @@ selfservice: recovery: enabled: true ui_url: http://localhost:4455/recovery - use: code + use: link verification: - enabled: true + # we disable verification for self-hosting + enabled: false ui_url: http://localhost:4455/verification - use: code + use: link after: default_browser_return_url: http://localhost:3000 diff --git a/quadratic-client/src/auth/ory.ts b/quadratic-client/src/auth/ory.ts index aa9c53cf1a..70a028662c 100644 --- a/quadratic-client/src/auth/ory.ts +++ b/quadratic-client/src/auth/ory.ts @@ -79,8 +79,8 @@ export const oryClient: OryAuthClient = { * If `isSignupFlow` is true, the user will be redirected to the registration flow. */ async login(redirectTo: string, isSignupFlow: boolean = false) { - const sdkUrl = isSignupFlow ? await sdk.createBrowserRegistrationFlow() : await sdk.createBrowserLoginFlow(); - const url = new URL(sdkUrl.data.ui.action); + const urlSegment = isSignupFlow ? 'registration' : 'login'; + const url = new URL(`${ORY_HOST}/self-service/${urlSegment}/browser`); url.searchParams.set('return_to', redirectTo); // redirect to the login/signup flow From e93082d1162050d56f52a2891a5b74ab4331777e Mon Sep 17 00:00:00 2001 From: David DiMaria Date: Wed, 4 Sep 2024 08:44:18 -0600 Subject: [PATCH 078/113] Update to mac friendly sed --- self-hosting/init.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/self-hosting/init.sh b/self-hosting/init.sh index d28937366c..abe2ffab5c 100755 --- a/self-hosting/init.sh +++ b/self-hosting/init.sh @@ -46,7 +46,7 @@ if ! [ -f "quadratic/LICENSE" ]; then rm ../init.sh rm init.sh - sed -i '' "s/\"LICENSE_KEY\"/\"$LICENSE_KEY\"/g" "docker-compose.yml" + sed -i.bak "s/\"LICENSE_KEY\"/\"$LICENSE_KEY\"/g" "docker-compose.yml" cd quadratic fi From 2b9f2eaecac7eb7e990c3de5f364fa20fe907a3a Mon Sep 17 00:00:00 2001 From: David DiMaria Date: Wed, 4 Sep 2024 08:56:57 -0600 Subject: [PATCH 079/113] Prompt user for host name during initialization --- self-hosting/docker-compose.yml | 14 ++++---- .../docker/ory-auth/config/kratos.yml | 34 +++++++++---------- self-hosting/init.sh | 22 +++++++++++- 3 files changed, 45 insertions(+), 25 deletions(-) diff --git a/self-hosting/docker-compose.yml b/self-hosting/docker-compose.yml index be5308ec4e..e2e7cdaec3 100644 --- a/self-hosting/docker-compose.yml +++ b/self-hosting/docker-compose.yml @@ -58,11 +58,11 @@ services: restart: "always" environment: VITE_DEBUG: 1 - VITE_QUADRATIC_API_URL: http://localhost:8000 + VITE_QUADRATIC_API_URL: http://#HOST#:8000 VITE_QUADRATIC_MULTIPLAYER_URL: ws://localhost:3001/ws - VITE_QUADRATIC_CONNECTION_URL: http://localhost:3003 + VITE_QUADRATIC_CONNECTION_URL: http://#HOST#:3003 VITE_AUTH_TYPE: ory - VITE_ORY_HOST: http://localhost:4433 + VITE_ORY_HOST: http://#HOST#:4433 ports: - "3000:80" command: > @@ -106,10 +106,10 @@ services: # Storage STORAGE_TYPE: file-system QUADRATIC_FILE_URI: http://host.docker.internal:3002 - QUADRATIC_FILE_URI_PUBLIC: http://localhost:3002 + QUADRATIC_FILE_URI_PUBLIC: http://#HOST#:3002 - # License UUID: leave "LICENSE_KEY" as this value is replaced during initialization - LICENSE_KEY: "LICENSE_KEY" + # License UUID: leave "#LICENSE_KEY#" as this value is replaced during initialization + LICENSE_KEY: "#LICENSE_KEY#" restart: "always" ports: - "8000:8000" @@ -299,7 +299,7 @@ services: environment: PORT: 4455 KRATOS_PUBLIC_URL: http://host.docker.internal:4433/ - KRATOS_BROWSER_URL: http://localhost:4433/ + KRATOS_BROWSER_URL: http://#HOST#:4433/ COOKIE_SECRET: changeme CSRF_COOKIE_NAME: __HOST-localhost-x-csrf-token CSRF_COOKIE_SECRET: changeme diff --git a/self-hosting/docker/ory-auth/config/kratos.yml b/self-hosting/docker/ory-auth/config/kratos.yml index 897077450c..1599bb3f31 100644 --- a/self-hosting/docker/ory-auth/config/kratos.yml +++ b/self-hosting/docker/ory-auth/config/kratos.yml @@ -5,12 +5,12 @@ dsn: memory serve: public: - base_url: http://localhost:4433/ + base_url: http://#host#:4433/ cors: enabled: true allowed_origins: - - http://localhost - - http://localhost:3000 + - http://#host# + - http://#host#:3000 allowed_methods: - POST - GET @@ -29,12 +29,12 @@ serve: base_url: http://kratos:4434/ selfservice: - default_browser_return_url: http://localhost + default_browser_return_url: http://#host# allowed_return_urls: - - http://localhost - - http://localhost:4455 - - http://localhost:3000 - - http://localhost:19006/Callback + - http://#host# + - http://#host#:4455 + - http://#host#:3000 + - http://#host#:19006/Callback - exp://localhost:8081/--/Callback methods: @@ -53,43 +53,43 @@ selfservice: flows: error: - ui_url: http://localhost:4455/error + ui_url: http://#host#:4455/error settings: - ui_url: http://localhost:4455/settings + ui_url: http://#host#:4455/settings privileged_session_max_age: 15m required_aal: highest_available recovery: enabled: true - ui_url: http://localhost:4455/recovery + ui_url: http://#host#:4455/recovery use: code verification: # we disable verification for self-hosting enabled: false - ui_url: http://localhost:4455/verification + ui_url: http://#host#:4455/verification use: code after: - default_browser_return_url: http://localhost/login-result + default_browser_return_url: http://#host#/login-result logout: after: - default_browser_return_url: http://localhost:4455/login + default_browser_return_url: http://#host#:4455/login login: - ui_url: http://localhost:4455/login + ui_url: http://#host#:4455/login lifespan: 10m registration: lifespan: 10m - ui_url: http://localhost:4455/registration + ui_url: http://#host#:4455/registration after: password: hooks: - hook: session - hook: show_verification_ui - default_browser_return_url: http://localhost/login-result + default_browser_return_url: http://#host#/login-result session: whoami: diff --git a/self-hosting/init.sh b/self-hosting/init.sh index abe2ffab5c..b85f8c02ef 100755 --- a/self-hosting/init.sh +++ b/self-hosting/init.sh @@ -17,6 +17,13 @@ get_license_key() { fi } +get_host() { + read -p "What public host name or public IP address are you using for this setup (e.g. localhost, app.quadratic.com, or other): " user_input + + # TODO: validate host + echo $user_input +} + checkout() { git clone -b $BRANCH --filter=blob:none --no-checkout --depth 1 --sparse $REPO cd quadratic @@ -26,6 +33,7 @@ checkout() { } LICENSE_KEY="" +HOST="" # check if LICENSE file exists if ! [ -f "quadratic/LICENSE" ]; then @@ -34,25 +42,37 @@ if ! [ -f "quadratic/LICENSE" ]; then if [ "$LICENSE_KEY" = "$INVALID_LICENSE_KEY" ]; then echo $INVALID_LICENSE_KEY else + + if ! [ -f "quadratic/HOST" ]; then + HOST=$(get_host) + fi + # retrieve the code from github checkout # write license key to LICENSE file touch LICENSE echo $LICENSE_KEY > LICENSE + + # write host to HOST file + touch HOST + echo $HOST > HOST cp -a self-hosting/. . rm -rf self-hosting rm ../init.sh rm init.sh - sed -i.bak "s/\"LICENSE_KEY\"/\"$LICENSE_KEY\"/g" "docker-compose.yml" + sed -i.bak "s/#LICENSE_KEY#/$LICENSE_KEY/g" "docker-compose.yml" + sed -i.bak "s/#HOST#/$HOST/g" "docker-compose.yml" + sed -i.bak "s/#HOST#/$HOST/g" "docker/ory-auth/config/kratos.yml" cd quadratic fi else cd quadratic LICENSE_KEY=$( Date: Wed, 4 Sep 2024 09:14:20 -0600 Subject: [PATCH 080/113] Remove .bak files --- self-hosting/README.md | 4 +-- self-hosting/docker-compose.yml | 4 +-- .../docker/ory-auth/config/kratos.yml | 36 +++++++++---------- self-hosting/init.sh | 4 +++ 4 files changed, 26 insertions(+), 22 deletions(-) diff --git a/self-hosting/README.md b/self-hosting/README.md index 3c12f82ba2..972f5a06d5 100644 --- a/self-hosting/README.md +++ b/self-hosting/README.md @@ -26,7 +26,7 @@ Implement the entire Quadratic stack outside of Quadratic. The use cases we cur Quadratic can be installed via a single command: ```shell -curl -sSf https://raw.githubusercontent.com/quadratichq/quadratic/6c4c62b58ae55251cb16d95b0076af47ed209cc8/self-hosting/init.sh -o init.sh && bash -i init.sh +curl -sSf https://raw.githubusercontent.com/quadratichq/quadratic/2b9f2eaecac7eb7e990c3de5f364fa20fe907a3a/self-hosting/init.sh -o init.sh && bash -i init.sh ``` This will download the initialization script, which will prompt for a license key in order to register Quadratic. @@ -68,5 +68,5 @@ sudo chmod +x /usr/local/bin/docker-compose sudo chown $USER /var/run/docker.sock sudo systemctl enable docker sudo systemctl start docker -curl -sSf https://raw.githubusercontent.com/quadratichq/quadratic/6c4c62b58ae55251cb16d95b0076af47ed209cc8/self-hosting/init.sh -o init.sh && bash -i init.sh +curl -sSf https://raw.githubusercontent.com/quadratichq/quadratic/2b9f2eaecac7eb7e990c3de5f364fa20fe907a3a/self-hosting/init.sh -o init.sh && bash -i init.sh ``` diff --git a/self-hosting/docker-compose.yml b/self-hosting/docker-compose.yml index e2e7cdaec3..3b74619cdc 100644 --- a/self-hosting/docker-compose.yml +++ b/self-hosting/docker-compose.yml @@ -59,7 +59,7 @@ services: environment: VITE_DEBUG: 1 VITE_QUADRATIC_API_URL: http://#HOST#:8000 - VITE_QUADRATIC_MULTIPLAYER_URL: ws://localhost:3001/ws + VITE_QUADRATIC_MULTIPLAYER_URL: ws://#HOST#:3001/ws VITE_QUADRATIC_CONNECTION_URL: http://#HOST#:3003 VITE_AUTH_TYPE: ory VITE_ORY_HOST: http://#HOST#:4433 @@ -301,7 +301,7 @@ services: KRATOS_PUBLIC_URL: http://host.docker.internal:4433/ KRATOS_BROWSER_URL: http://#HOST#:4433/ COOKIE_SECRET: changeme - CSRF_COOKIE_NAME: __HOST-localhost-x-csrf-token + CSRF_COOKIE_NAME: __HOST-#HOST#-x-csrf-token CSRF_COOKIE_SECRET: changeme restart: on-failure profiles: diff --git a/self-hosting/docker/ory-auth/config/kratos.yml b/self-hosting/docker/ory-auth/config/kratos.yml index 1599bb3f31..45e3588c4b 100644 --- a/self-hosting/docker/ory-auth/config/kratos.yml +++ b/self-hosting/docker/ory-auth/config/kratos.yml @@ -5,12 +5,12 @@ dsn: memory serve: public: - base_url: http://#host#:4433/ + base_url: http://#HOST#:4433/ cors: enabled: true allowed_origins: - - http://#host# - - http://#host#:3000 + - http://#HOST# + - http://#HOST#:3000 allowed_methods: - POST - GET @@ -29,12 +29,12 @@ serve: base_url: http://kratos:4434/ selfservice: - default_browser_return_url: http://#host# + default_browser_return_url: http://#HOST# allowed_return_urls: - - http://#host# - - http://#host#:4455 - - http://#host#:3000 - - http://#host#:19006/Callback + - http://#HOST# + - http://#HOST#:4455 + - http://#HOST#:3000 + - http://#HOST#:19006/Callback - exp://localhost:8081/--/Callback methods: @@ -53,43 +53,43 @@ selfservice: flows: error: - ui_url: http://#host#:4455/error + ui_url: http://#HOST#:4455/error settings: - ui_url: http://#host#:4455/settings + ui_url: http://#HOST#:4455/settings privileged_session_max_age: 15m required_aal: highest_available recovery: enabled: true - ui_url: http://#host#:4455/recovery + ui_url: http://#HOST#:4455/recovery use: code verification: # we disable verification for self-hosting enabled: false - ui_url: http://#host#:4455/verification + ui_url: http://#HOST#:4455/verification use: code after: - default_browser_return_url: http://#host#/login-result + default_browser_return_url: http://#HOST#/login-result logout: after: - default_browser_return_url: http://#host#:4455/login + default_browser_return_url: http://#HOST#:4455/login login: - ui_url: http://#host#:4455/login + ui_url: http://#HOST#:4455/login lifespan: 10m registration: lifespan: 10m - ui_url: http://#host#:4455/registration + ui_url: http://#HOST#:4455/registration after: password: hooks: - hook: session - hook: show_verification_ui - default_browser_return_url: http://#host#/login-result + default_browser_return_url: http://#HOST#/login-result session: whoami: @@ -100,7 +100,7 @@ session: # claims_mapper_url: base64://... # A JsonNet template for modifying the claims ttl: 24h # 24 hours (defaults to 10 minutes) cookies: - domain: localhost + domain: "#HOST#" path: / same_site: Lax diff --git a/self-hosting/init.sh b/self-hosting/init.sh index b85f8c02ef..eae47752db 100755 --- a/self-hosting/init.sh +++ b/self-hosting/init.sh @@ -63,10 +63,14 @@ if ! [ -f "quadratic/LICENSE" ]; then rm ../init.sh rm init.sh + # adding .bak for compatibility with both GNU (Linux) and BSD (MacOS) sed sed -i.bak "s/#LICENSE_KEY#/$LICENSE_KEY/g" "docker-compose.yml" sed -i.bak "s/#HOST#/$HOST/g" "docker-compose.yml" sed -i.bak "s/#HOST#/$HOST/g" "docker/ory-auth/config/kratos.yml" + rm docker-compose.yml.bak + rm docker/ory-auth/config/kratos.yml.bak + cd quadratic fi else From e208253f45f626d93dc31c7cf80820f43216eae2 Mon Sep 17 00:00:00 2001 From: David DiMaria Date: Wed, 4 Sep 2024 09:16:18 -0600 Subject: [PATCH 081/113] Update init.sh link in README --- self-hosting/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/self-hosting/README.md b/self-hosting/README.md index 972f5a06d5..fe2bb13d60 100644 --- a/self-hosting/README.md +++ b/self-hosting/README.md @@ -26,7 +26,7 @@ Implement the entire Quadratic stack outside of Quadratic. The use cases we cur Quadratic can be installed via a single command: ```shell -curl -sSf https://raw.githubusercontent.com/quadratichq/quadratic/2b9f2eaecac7eb7e990c3de5f364fa20fe907a3a/self-hosting/init.sh -o init.sh && bash -i init.sh +curl -sSf https://raw.githubusercontent.com/quadratichq/quadratic/7a8b99cc0154baada8a8bd535584e2e3c369e71a/self-hosting/init.sh -o init.sh && bash -i init.sh ``` This will download the initialization script, which will prompt for a license key in order to register Quadratic. @@ -68,5 +68,5 @@ sudo chmod +x /usr/local/bin/docker-compose sudo chown $USER /var/run/docker.sock sudo systemctl enable docker sudo systemctl start docker -curl -sSf https://raw.githubusercontent.com/quadratichq/quadratic/2b9f2eaecac7eb7e990c3de5f364fa20fe907a3a/self-hosting/init.sh -o init.sh && bash -i init.sh +curl -sSf https://raw.githubusercontent.com/quadratichq/quadratic/7a8b99cc0154baada8a8bd535584e2e3c369e71a/self-hosting/init.sh -o init.sh && bash -i init.sh ``` From bf35b7a28bf6e2c7e04ce6ecbcd968e45d1af66e Mon Sep 17 00:00:00 2001 From: David DiMaria Date: Wed, 4 Sep 2024 09:38:42 -0600 Subject: [PATCH 082/113] Update instructions to include Creating an EC2 Instance --- self-hosting/README.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/self-hosting/README.md b/self-hosting/README.md index fe2bb13d60..28109dbe40 100644 --- a/self-hosting/README.md +++ b/self-hosting/README.md @@ -53,6 +53,15 @@ If running in the background, run the `stop.sh` script: ./quadratic/self-hosting/stop.sh ``` +## Creating an EC2 Instance + +* Click on Launch and Instance from the main EC2 screen +* Select the Ubuntu option +* The minium size should be a t2.xlarge +* Either create a new security group with `Allow HTTPS traffic from the internet` or `Allow HTTP traffic from the internet` (not using certs) OR select an existing security group with this setting enabled +* Configure storage to 30GiB +* Click on the "Launch Instance" button + ## Installing on Ubuntu ```shell From 3951a55aec74ee661a26ef4d47d84ffaaf5cf523 Mon Sep 17 00:00:00 2001 From: David DiMaria Date: Wed, 4 Sep 2024 11:52:06 -0600 Subject: [PATCH 083/113] Ensure login-redirect after ory registration --- docker/ory-auth/config/kratos.yml | 2 ++ quadratic-api/src/middleware/user.ts | 1 + self-hosting/docker/ory-auth/config/kratos.yml | 1 + 3 files changed, 4 insertions(+) diff --git a/docker/ory-auth/config/kratos.yml b/docker/ory-auth/config/kratos.yml index 37847c04f5..c815b5713f 100644 --- a/docker/ory-auth/config/kratos.yml +++ b/docker/ory-auth/config/kratos.yml @@ -84,7 +84,9 @@ selfservice: lifespan: 10m ui_url: http://localhost:4455/registration after: + default_browser_return_url: http://localhost:3000/login-result password: + default_browser_return_url: http://localhost:3000/login-result hooks: - hook: session - hook: show_verification_ui diff --git a/quadratic-api/src/middleware/user.ts b/quadratic-api/src/middleware/user.ts index 46c2207804..e2fe6f2fb9 100644 --- a/quadratic-api/src/middleware/user.ts +++ b/quadratic-api/src/middleware/user.ts @@ -59,6 +59,7 @@ const getOrCreateUser = async (auth0Id: string) => { auth0Id, }, }); + if (user) { return user; } diff --git a/self-hosting/docker/ory-auth/config/kratos.yml b/self-hosting/docker/ory-auth/config/kratos.yml index 45e3588c4b..1d68d27e1c 100644 --- a/self-hosting/docker/ory-auth/config/kratos.yml +++ b/self-hosting/docker/ory-auth/config/kratos.yml @@ -86,6 +86,7 @@ selfservice: ui_url: http://#HOST#:4455/registration after: password: + default_browser_return_url: http://#HOST#/login-result hooks: - hook: session - hook: show_verification_ui From 21057a2d3b9e7e43e08ca239f3a6c14da2b9f8f9 Mon Sep 17 00:00:00 2001 From: David DiMaria Date: Wed, 4 Sep 2024 16:44:44 -0600 Subject: [PATCH 084/113] Allow command line arguments to be used in init.sh --- self-hosting/init.sh | 83 +++++++++++++++++++++++++------------------- 1 file changed, 48 insertions(+), 35 deletions(-) diff --git a/self-hosting/init.sh b/self-hosting/init.sh index eae47752db..703d7c8776 100755 --- a/self-hosting/init.sh +++ b/self-hosting/init.sh @@ -1,5 +1,22 @@ #!/bin/bash +# Self-Hosting Initialization +# +# Usage: +# +# ./init.sh 83f0ebdf-eafb-4c8d-bd7b-04ea07d61b7f localhost +# +# +# Flow: +# +# First, check to see if there is a VERSION file, if so, use that version. +# If not, then check for the first command line argument, if so, use that version. +# Else, prompt the user. +# +# First, check to see if there is a HOST file, if so, use that host. +# If not, then check for the first command line argument, if so, use that host. +# Else, prompt the user. + REPO="https://github.com/quadratichq/quadratic.git" BRANCH="self-hosting-setup" DIR="self-hosting" @@ -35,48 +52,44 @@ checkout() { LICENSE_KEY="" HOST="" -# check if LICENSE file exists -if ! [ -f "quadratic/LICENSE" ]; then +if [ -f "quadratic/LICENSE" ]; then + LICENSE_KEY=$( LICENSE - - # write host to HOST file - touch HOST - echo $HOST > HOST +# write license key to LICENSE file +touch LICENSE +echo $LICENSE_KEY > LICENSE - cp -a self-hosting/. . - rm -rf self-hosting - rm ../init.sh - rm init.sh +# write host to HOST file +touch HOST +echo $HOST > HOST - # adding .bak for compatibility with both GNU (Linux) and BSD (MacOS) sed - sed -i.bak "s/#LICENSE_KEY#/$LICENSE_KEY/g" "docker-compose.yml" - sed -i.bak "s/#HOST#/$HOST/g" "docker-compose.yml" - sed -i.bak "s/#HOST#/$HOST/g" "docker/ory-auth/config/kratos.yml" +cp -a self-hosting/. . +rm -rf self-hosting +rm ../init.sh +rm init.sh - rm docker-compose.yml.bak - rm docker/ory-auth/config/kratos.yml.bak +# adding .bak for compatibility with both GNU (Linux) and BSD (MacOS) sed +sed -i.bak "s/#LICENSE_KEY#/$LICENSE_KEY/g" "docker-compose.yml" +sed -i.bak "s/#HOST#/$HOST/g" "docker-compose.yml" +sed -i.bak "s/#HOST#/$HOST/g" "docker/ory-auth/config/kratos.yml" - cd quadratic - fi -else - cd quadratic - LICENSE_KEY=$( Date: Wed, 4 Sep 2024 16:47:56 -0600 Subject: [PATCH 085/113] Update readme with new init link and ports to open --- self-hosting/README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/self-hosting/README.md b/self-hosting/README.md index 28109dbe40..02add03677 100644 --- a/self-hosting/README.md +++ b/self-hosting/README.md @@ -26,7 +26,7 @@ Implement the entire Quadratic stack outside of Quadratic. The use cases we cur Quadratic can be installed via a single command: ```shell -curl -sSf https://raw.githubusercontent.com/quadratichq/quadratic/7a8b99cc0154baada8a8bd535584e2e3c369e71a/self-hosting/init.sh -o init.sh && bash -i init.sh +curl -sSf https://raw.githubusercontent.com/quadratichq/quadratic/21057a2d3b9e7e43e08ca239f3a6c14da2b9f8f9/self-hosting/init.sh -o init.sh && bash -i init.sh ``` This will download the initialization script, which will prompt for a license key in order to register Quadratic. @@ -59,6 +59,7 @@ If running in the background, run the `stop.sh` script: * Select the Ubuntu option * The minium size should be a t2.xlarge * Either create a new security group with `Allow HTTPS traffic from the internet` or `Allow HTTP traffic from the internet` (not using certs) OR select an existing security group with this setting enabled + * Open ports 80, 443, 3001, 3002, 3003, 4433, 4455, and 8000 for TCP traffic with 0.0.0.0/0 source * Configure storage to 30GiB * Click on the "Launch Instance" button @@ -77,5 +78,5 @@ sudo chmod +x /usr/local/bin/docker-compose sudo chown $USER /var/run/docker.sock sudo systemctl enable docker sudo systemctl start docker -curl -sSf https://raw.githubusercontent.com/quadratichq/quadratic/7a8b99cc0154baada8a8bd535584e2e3c369e71a/self-hosting/init.sh -o init.sh && bash -i init.sh +curl -sSf https://raw.githubusercontent.com/quadratichq/quadratic/21057a2d3b9e7e43e08ca239f3a6c14da2b9f8f9/self-hosting/init.sh -o init.sh && bash -i init.sh ``` From d90f78c0249059c31da4f7a221177dd564698dca Mon Sep 17 00:00:00 2001 From: David DiMaria Date: Wed, 4 Sep 2024 16:49:18 -0600 Subject: [PATCH 086/113] Update readme requirement to include all ports --- self-hosting/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/self-hosting/README.md b/self-hosting/README.md index 02add03677..02e1e51e79 100644 --- a/self-hosting/README.md +++ b/self-hosting/README.md @@ -17,7 +17,7 @@ Implement the entire Quadratic stack outside of Quadratic. The use cases we cur * MacOS or Linux (not tested on Windows) * License Key (available at https://selfhost.quadratic-preview.com) -* The following open ports: 80, 3000, 3001, 3002, 8000 +* The following open ports: 80, 443, 3001, 3002, 3003, 4433, 4455, and 8000 ## Installation From b818eccb16374dc7222eb91b1889ee35956ccafc Mon Sep 17 00:00:00 2001 From: David DiMaria Date: Fri, 13 Sep 2024 14:01:05 -0600 Subject: [PATCH 087/113] Remove self-hosting directory (in favor of new self-hosting repo --- self-hosting/.gitignore | 6 - self-hosting/README.md | 82 ----- self-hosting/docker-compose.yml | 333 ------------------ self-hosting/docker/admin/config/openapi.yaml | 48 --- self-hosting/docker/caddy/config/Caddyfile | 3 - .../docker/client/config/default.conf | 21 -- .../docker/client/scripts/replace_env_vars.sh | 34 -- .../ory-auth/config/identity.schema.json | 47 --- .../docker/ory-auth/config/kratos.yml | 139 -------- self-hosting/docker/postgres/.gitignore | 0 self-hosting/docker/postgres/scripts/init.sh | 18 - self-hosting/docker/redis/.gitignore | 0 self-hosting/init.sh | 95 ----- self-hosting/start.sh | 9 - self-hosting/stop.sh | 7 - 15 files changed, 842 deletions(-) delete mode 100644 self-hosting/.gitignore delete mode 100644 self-hosting/README.md delete mode 100644 self-hosting/docker-compose.yml delete mode 100644 self-hosting/docker/admin/config/openapi.yaml delete mode 100644 self-hosting/docker/caddy/config/Caddyfile delete mode 100644 self-hosting/docker/client/config/default.conf delete mode 100755 self-hosting/docker/client/scripts/replace_env_vars.sh delete mode 100644 self-hosting/docker/ory-auth/config/identity.schema.json delete mode 100644 self-hosting/docker/ory-auth/config/kratos.yml delete mode 100644 self-hosting/docker/postgres/.gitignore delete mode 100755 self-hosting/docker/postgres/scripts/init.sh delete mode 100644 self-hosting/docker/redis/.gitignore delete mode 100755 self-hosting/init.sh delete mode 100755 self-hosting/start.sh delete mode 100755 self-hosting/stop.sh diff --git a/self-hosting/.gitignore b/self-hosting/.gitignore deleted file mode 100644 index 74d7674042..0000000000 --- a/self-hosting/.gitignore +++ /dev/null @@ -1,6 +0,0 @@ -docker/caddy/certs -docker/file-storage -docker/localstack/data -docker/mysql/data -docker/postgres/data -docker/redis/data \ No newline at end of file diff --git a/self-hosting/README.md b/self-hosting/README.md deleted file mode 100644 index 02e1e51e79..0000000000 --- a/self-hosting/README.md +++ /dev/null @@ -1,82 +0,0 @@ -# Quadratic Self-Hosting - -Implement the entire Quadratic stack outside of Quadratic. The use cases we currently support: - -- [x] Localhost -- [x] EC2 (using your own load balancer) -- [ ] EC2 (using [Caddy's load balancer](https://caddyserver.com/docs/caddyfile/directives/reverse_proxy)) -- [ ] Multiple Docker instance setup (for any cloud provider) -- [ ] Kubernetes - -## Dependencies - -* [Git](https://github.com/git-guides/install-git) -* [Docker](https://docs.docker.com/engine/install/) - -## Requirements - -* MacOS or Linux (not tested on Windows) -* License Key (available at https://selfhost.quadratic-preview.com) -* The following open ports: 80, 443, 3001, 3002, 3003, 4433, 4455, and 8000 - -## Installation - -> **NOTE:** _Before installing, please create a license and copy the key at https://selfhost.quadratic-preview.com._ - -Quadratic can be installed via a single command: - -```shell -curl -sSf https://raw.githubusercontent.com/quadratichq/quadratic/21057a2d3b9e7e43e08ca239f3a6c14da2b9f8f9/self-hosting/init.sh -o init.sh && bash -i init.sh -``` - -This will download the initialization script, which will prompt for a license key in order to register Quadratic. - -Additionally, the docker compose network will start (see [Starting](#Starting)). Please allow several minutes for the docker images to downloaded. - -Refer to the [Stopping](#Stopping) section. - -## Starting - -Once the Quadratic is initialized, a single command is needed to start all of the images: - -```shell -./quadratic/self-hosting/start.sh -``` - -## Stopping - -To stop running docker images, simply press `ctrl + c` if running in the foreground. - -If running in the background, run the `stop.sh` script: - -```shell -./quadratic/self-hosting/stop.sh -``` - -## Creating an EC2 Instance - -* Click on Launch and Instance from the main EC2 screen -* Select the Ubuntu option -* The minium size should be a t2.xlarge -* Either create a new security group with `Allow HTTPS traffic from the internet` or `Allow HTTP traffic from the internet` (not using certs) OR select an existing security group with this setting enabled - * Open ports 80, 443, 3001, 3002, 3003, 4433, 4455, and 8000 for TCP traffic with 0.0.0.0/0 source -* Configure storage to 30GiB -* Click on the "Launch Instance" button - -## Installing on Ubuntu - -```shell -sudo apt-get update -sudo apt-get install apt-transport-https ca-certificates curl software-properties-common -curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg -echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null -sudo apt-get update -sudo apt-get -y install docker-ce docker-ce-cli containerd.io -sudo docker --version -sudo curl -L "https://github.com/docker/compose/releases/download/v2.21.0/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose -sudo chmod +x /usr/local/bin/docker-compose -sudo chown $USER /var/run/docker.sock -sudo systemctl enable docker -sudo systemctl start docker -curl -sSf https://raw.githubusercontent.com/quadratichq/quadratic/21057a2d3b9e7e43e08ca239f3a6c14da2b9f8f9/self-hosting/init.sh -o init.sh && bash -i init.sh -``` diff --git a/self-hosting/docker-compose.yml b/self-hosting/docker-compose.yml deleted file mode 100644 index 3b74619cdc..0000000000 --- a/self-hosting/docker-compose.yml +++ /dev/null @@ -1,333 +0,0 @@ -version: "3.8" - -services: - redis: - image: redis/redis-stack:latest - restart: always - ports: - - "6379:6379" - - "8001:8001" - healthcheck: - test: ["CMD", "redis-cli", "ping"] - interval: "5s" - volumes: - - ./docker/redis/data:/data - profiles: - - base - - postgres: - image: postgres:15 - restart: always - ports: - - "5432:5432" - environment: - POSTGRES_USER: postgres - PGUSER: postgres - POSTGRES_PASSWORD: postgres - ADDITIONAL_DATABASES: kratos - healthcheck: - test: ["CMD-SHELL", "pg_isready -U postgres -d postgres"] - interval: 10s - timeout: 5s - retries: 5 - volumes: - - ./docker/postgres/data:/var/lib/postgresql/data - - ./docker/postgres/scripts:/docker-entrypoint-initdb.d - profiles: - - base - - caddy: - image: caddy:latest - ports: - - "80:80" - - "443:443" - volumes: - - ./docker/caddy/config/Caddyfile:/etc/caddy/Caddyfile - - ./docker/caddy/certs:/data/caddy/pki/authorities/local - # - ./docker/caddy/quadratic-client/html:/srv - profiles: - - caddy - - frontend - networks: - - host - extra_hosts: - - "host.docker.internal:host-gateway" - - quadratic-client: - image: public.ecr.aws/l3i4i9z2/quadratic-client-staging:latest - restart: "always" - environment: - VITE_DEBUG: 1 - VITE_QUADRATIC_API_URL: http://#HOST#:8000 - VITE_QUADRATIC_MULTIPLAYER_URL: ws://#HOST#:3001/ws - VITE_QUADRATIC_CONNECTION_URL: http://#HOST#:3003 - VITE_AUTH_TYPE: ory - VITE_ORY_HOST: http://#HOST#:4433 - ports: - - "3000:80" - command: > - sh -c "/client/scripts/replace_env_vars.sh && - nginx -g \"daemon off;\"" - healthcheck: - test: ["CMD-SHELL", "curl -f http://host.docker.internal:3000/ || exit 1"] - interval: 10s - timeout: 5s - volumes: - - ./docker/client:/client - - ./docker/client/config/default.conf:/etc/nginx/conf.d/default.conf - # - ./docker/client/build:/usr/share/nginx/html - profiles: - - client - - frontend - networks: - - host - extra_hosts: - - "host.docker.internal:host-gateway" - - quadratic-api: - image: public.ecr.aws/l3i4i9z2/quadratic-api-staging:latest - environment: - CORS: "*" - DATABASE_URL: "postgresql://postgres:postgres@host.docker.internal:5432/postgres" - ENVIRONMENT: docker - STRIPE_SECRET_KEY: STRIPE_SECRET_KEY - STRIPE_WEBHOOK_SECRET: STRIPE_WEBHOOK_SECRET - OPENAI_API_KEY: - M2M_AUTH_TOKEN: M2M_AUTH_TOKEN - - # Hex string to be used as the key for enctyption, use npm run key:generate - ENCRYPTION_KEY: eb4758047f74bdb2603cce75c4370327ca2c3662c4786867659126da8e64dfcc - - # Auth - AUTH_TYPE: ory - ORY_JWKS_URI: "http://host.docker.internal:3000/.well-known/jwks.json" - ORY_ADMIN_HOST: http://host.docker.internal:4434 - - # Storage - STORAGE_TYPE: file-system - QUADRATIC_FILE_URI: http://host.docker.internal:3002 - QUADRATIC_FILE_URI_PUBLIC: http://#HOST#:3002 - - # License UUID: leave "#LICENSE_KEY#" as this value is replaced during initialization - LICENSE_KEY: "#LICENSE_KEY#" - restart: "always" - ports: - - "8000:8000" - command: bash -c "npx prisma migrate deploy --schema quadratic-api/prisma/schema.prisma && npm run start:prod --workspace=quadratic-api" - depends_on: - postgres: - condition: service_healthy - profiles: - - api - - frontend - networks: - - host - extra_hosts: - - "host.docker.internal:host-gateway" - - quadratic-multiplayer: - image: public.ecr.aws/l3i4i9z2/quadratic-multiplayer-staging:latest - environment: - RUST_LOG: info - HOST: 0.0.0.0 - PORT: 3001 - HEARTBEAT_CHECK_S: 3 - HEARTBEAT_TIMEOUT_S: 600 - QUADRATIC_API_URI: http://host.docker.internal:8000 - M2M_AUTH_TOKEN: M2M_AUTH_TOKEN - ENVIRONMENT: docker - - PUBSUB_HOST: host.docker.internal - PUBSUB_PORT: 6379 - PUBSUB_PASSWORD: "" - PUBSUB_ACTIVE_CHANNELS: active_channels - - AUTH0_JWKS_URI: http://host.docker.internal:3000/.well-known/jwks.json - AUTHENTICATE_JWT: true - restart: "always" - ports: - - "3001:3001" - depends_on: - postgres: - condition: service_healthy - redis: - condition: service_healthy - quadratic-api: - condition: service_started - quadratic-client: - condition: service_healthy - profiles: - - backend - - multiplayer - networks: - - host - extra_hosts: - - "host.docker.internal:host-gateway" - - quadratic-files: - image: public.ecr.aws/l3i4i9z2/quadratic-files-staging:latest - environment: - RUST_LOG: info - HOST: 0.0.0.0 - PORT: 3002 - FILE_CHECK_S: 5 - FILES_PER_CHECK: 1000 - TRUNCATE_FILE_CHECK_S: 60 - TRUNCATE_TRANSACTION_AGE_DAYS: 5 # - ENVIRONMENT: docker - - AUTH0_JWKS_URI: http://host.docker.internal:3000/.well-known/jwks.json - QUADRATIC_API_URI: http://host.docker.internal:8000 - M2M_AUTH_TOKEN: M2M_AUTH_TOKEN - - PUBSUB_HOST: host.docker.internal - PUBSUB_PORT: 6379 - PUBSUB_PASSWORD: "" - PUBSUB_ACTIVE_CHANNELS: active_channels - PUBSUB_PROCESSED_TRANSACTIONS_CHANNEL: processed_transactions - - # Storage - STORAGE_TYPE: file-system # s3 or file-system - - # Storage: s3 - AWS_S3_REGION: - AWS_S3_BUCKET_NAME: quadratic-api-docker - AWS_S3_ACCESS_KEY_ID: - AWS_S3_SECRET_ACCESS_KEY: - - # Storage: file-system - STORAGE_DIR: /file-storage - STORAGE_ENCRYPTION_KEYS: eb4758047f74bdb2603cce75c4370327ca2c3662c4786867659126da8e64dfcc - - restart: "always" - ports: - - "3002:3002" - depends_on: - postgres: - condition: service_healthy - redis: - condition: service_healthy - quadratic-api: - condition: service_started - quadratic-client: - condition: service_healthy - volumes: - - ./docker/file-storage:/file-storage - profiles: - - backend - - files - networks: - - host - extra_hosts: - - "host.docker.internal:host-gateway" - - quadratic-connection: - image: public.ecr.aws/l3i4i9z2/quadratic-connection-staging:latest - environment: - RUST_LOG: info - HOST: 0.0.0.0 - PORT: 3003 - ENVIRONMENT: docker - - AUTH0_JWKS_URI: http://host.docker.internal:3000/.well-known/jwks.json - QUADRATIC_API_URI: http://host.docker.internal:8000 - M2M_AUTH_TOKEN: M2M_AUTH_TOKEN - MAX_RESPONSE_BYTES: 15728640 # 15MB - STATIC_IPS: 0.0.0.0,127.0.0.1 - - restart: "always" - ports: - - "3003:3003" - depends_on: - caddy: - condition: service_started - quadratic-client: - condition: service_healthy - profiles: - - backend - - connection - networks: - - host - extra_hosts: - - "host.docker.internal:host-gateway" - - # Auth Providers - - ory-auth: - image: oryd/kratos:v1.2.0 - ports: - - "4433:4433" # public - - "4434:4434" # admin - command: serve -c /etc/config/kratos/kratos.yml --dev --watch-courier - volumes: - - ./docker/ory-auth/config:/etc/config/kratos - environment: - DSN: postgresql://postgres:postgres@host.docker.internal:5432/kratos?sslmode=disable - LOG_LEVEL: trace - restart: unless-stopped - depends_on: - - postgres - - ory-auth-migrate - profiles: - - ory - networks: - - host - extra_hosts: - - "host.docker.internal:host-gateway" - - ory-auth-migrate: - image: oryd/kratos:v1.2.0 - command: migrate -c /etc/config/kratos/kratos.yml sql -e --yes - volumes: - - ./docker/ory-auth/config:/etc/config/kratos - environment: - DSN: postgresql://postgres:postgres@host.docker.internal:5432/kratos?sslmode=disable - restart: on-failure - depends_on: - - postgres - profiles: - - ory - networks: - - host - extra_hosts: - - "host.docker.internal:host-gateway" - - ory-auth-node: - image: oryd/kratos-selfservice-ui-node:v1.2.0 - ports: - - "4455:4455" - environment: - PORT: 4455 - KRATOS_PUBLIC_URL: http://host.docker.internal:4433/ - KRATOS_BROWSER_URL: http://#HOST#:4433/ - COOKIE_SECRET: changeme - CSRF_COOKIE_NAME: __HOST-#HOST#-x-csrf-token - CSRF_COOKIE_SECRET: changeme - restart: on-failure - profiles: - - ory - networks: - - host - extra_hosts: - - "host.docker.internal:host-gateway" - - ory-auth-mail: - image: oryd/mailslurper:latest-smtps - ports: - - "1025:1025" - - "4436:4436" - - "4437:4437" - - "8080:8080" - profiles: - - ory - networks: - - host - extra_hosts: - - "host.docker.internal:host-gateway" - -volumes: - docker: - name: docker - -networks: - host: diff --git a/self-hosting/docker/admin/config/openapi.yaml b/self-hosting/docker/admin/config/openapi.yaml deleted file mode 100644 index ff6fd4fe92..0000000000 --- a/self-hosting/docker/admin/config/openapi.yaml +++ /dev/null @@ -1,48 +0,0 @@ -openapi: 3.0.3 -info: - title: My API - version: '1.0' - x-logo: - url: '' -paths: - /license/{licenseKey}: - post: - tags: [] - operationId: license - parameters: - - name: licenseKey - in: path - required: true - deprecated: false - example: sdaf - schema: - type: string - x-last-modified: 1724376657454 - responses: - '200': - content: - application/json: - schema: - type: object - properties: - limits: - type: object - properties: - seats: - type: number - example: 5 - x-last-modified: 1724376729775 - description: '' - headers: {} - links: {} - x-last-modified: 1724376538356 -components: - securitySchemes: {} - schemas: {} - headers: {} - responses: {} - parameters: {} -tags: [] -servers: - - url: https://api.example.io -security: [] diff --git a/self-hosting/docker/caddy/config/Caddyfile b/self-hosting/docker/caddy/config/Caddyfile deleted file mode 100644 index 979148ec01..0000000000 --- a/self-hosting/docker/caddy/config/Caddyfile +++ /dev/null @@ -1,3 +0,0 @@ -:80 { - reverse_proxy http://host.docker.internal:3000 -} diff --git a/self-hosting/docker/client/config/default.conf b/self-hosting/docker/client/config/default.conf deleted file mode 100644 index cdb53aa4ac..0000000000 --- a/self-hosting/docker/client/config/default.conf +++ /dev/null @@ -1,21 +0,0 @@ -server { - listen 80; - server_name localhost; - - root /usr/share/nginx/html; - index index.html; - - location / { - try_files $uri $uri/ /index.html =404; - } - - location ~* \.(?:css|js|json|gif|png|jpg|jpeg|svg|ico)$ { - expires 1y; - access_log off; - add_header Cache-Control "public, no-transform"; - } - - # add CORS headers - add_header Cross-Origin-Opener-Policy "same-origin"; - add_header Cross-Origin-Embedder-Policy "require-corp"; -} \ No newline at end of file diff --git a/self-hosting/docker/client/scripts/replace_env_vars.sh b/self-hosting/docker/client/scripts/replace_env_vars.sh deleted file mode 100755 index 60160a8ccf..0000000000 --- a/self-hosting/docker/client/scripts/replace_env_vars.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/bin/sh - -escape_for_sed() { - input="$1" - printf '%s\n' "$input" | sed -e 's/[\/&]/\\&/g' -} - -replace_env_vars() { - vite_vars="" - - for env_var in $(env); do - case "$env_var" in - VITE_*) - vite_vars="$vite_vars $env_var" - ;; - esac - done - - find "/usr/share/nginx/html/assets" -type f -name "*.js" | xargs grep -l "VITE_" | while read file; do - - for env_var in $vite_vars; do - var="$(echo "$env_var" | cut -d'=' -f1)" - val="$(echo "$env_var" | cut -d'=' -f2-)" - appended_var="${var}_VAL" - escaped_val=$(escape_for_sed "$val") - - # echo "Replacing $appended_var with $escaped_val in $file" - sed -i "s/${appended_var}/${escaped_val}/g" "$file" - done - done -} - -echo "Replacing .env values in $ENV_PATH" -replace_env_vars diff --git a/self-hosting/docker/ory-auth/config/identity.schema.json b/self-hosting/docker/ory-auth/config/identity.schema.json deleted file mode 100644 index a953fc68ec..0000000000 --- a/self-hosting/docker/ory-auth/config/identity.schema.json +++ /dev/null @@ -1,47 +0,0 @@ -{ - "$id": "https://schemas.ory.sh/presets/kratos/quickstart/email-password/identity.schema.json", - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Person", - "type": "object", - "properties": { - "traits": { - "type": "object", - "properties": { - "email": { - "type": "string", - "format": "email", - "title": "E-Mail", - "minLength": 3, - "ory.sh/kratos": { - "credentials": { - "password": { - "identifier": true - } - }, - "verification": { - "via": "email" - }, - "recovery": { - "via": "email" - } - } - }, - "name": { - "type": "object", - "properties": { - "first": { - "title": "First Name", - "type": "string" - }, - "last": { - "title": "Last Name", - "type": "string" - } - } - } - }, - "required": ["email"], - "additionalProperties": false - } - } -} diff --git a/self-hosting/docker/ory-auth/config/kratos.yml b/self-hosting/docker/ory-auth/config/kratos.yml deleted file mode 100644 index 1d68d27e1c..0000000000 --- a/self-hosting/docker/ory-auth/config/kratos.yml +++ /dev/null @@ -1,139 +0,0 @@ -# https://raw.githubusercontent.com/ory/kratos/v1.2.0/.schemastore/config.schema.json -version: v1.2.0 - -dsn: memory - -serve: - public: - base_url: http://#HOST#:4433/ - cors: - enabled: true - allowed_origins: - - http://#HOST# - - http://#HOST#:3000 - allowed_methods: - - POST - - GET - - PUT - - PATCH - - DELETE - allowed_headers: - - Authorization - - Access-Control-Allow-Origin - - Cookie - - Content-Type - exposed_headers: - - Content-Type - - Set-Cookie - admin: - base_url: http://kratos:4434/ - -selfservice: - default_browser_return_url: http://#HOST# - allowed_return_urls: - - http://#HOST# - - http://#HOST#:4455 - - http://#HOST#:3000 - - http://#HOST#:19006/Callback - - exp://localhost:8081/--/Callback - - methods: - password: - enabled: true - totp: - config: - issuer: Kratos - enabled: true - lookup_secret: - enabled: true - link: - enabled: true - code: - enabled: true - - flows: - error: - ui_url: http://#HOST#:4455/error - - settings: - ui_url: http://#HOST#:4455/settings - privileged_session_max_age: 15m - required_aal: highest_available - - recovery: - enabled: true - ui_url: http://#HOST#:4455/recovery - use: code - - verification: - # we disable verification for self-hosting - enabled: false - ui_url: http://#HOST#:4455/verification - use: code - after: - default_browser_return_url: http://#HOST#/login-result - - logout: - after: - default_browser_return_url: http://#HOST#:4455/login - - login: - ui_url: http://#HOST#:4455/login - lifespan: 10m - - registration: - lifespan: 10m - ui_url: http://#HOST#:4455/registration - after: - password: - default_browser_return_url: http://#HOST#/login-result - hooks: - - hook: session - - hook: show_verification_ui - default_browser_return_url: http://#HOST#/login-result - -session: - whoami: - tokenizer: - templates: - jwt_template: - jwks_url: http://host.docker.internal:3000/.well-known/jwks.json - # claims_mapper_url: base64://... # A JsonNet template for modifying the claims - ttl: 24h # 24 hours (defaults to 10 minutes) -cookies: - domain: "#HOST#" - path: / - same_site: Lax - -log: - level: warning - format: json - redaction_text: "" - leak_sensitive_values: false - -secrets: - cookie: - - PLEASE-CHANGE-ME-I-AM-VERY-INSECURE - cipher: - - 32-LONG-SECRET-NOT-SECURE-AT-ALL - -ciphers: - algorithm: xchacha20-poly1305 - -hashers: - algorithm: bcrypt - bcrypt: - cost: 8 - -identity: - default_schema_id: default - schemas: - - id: default - url: file:///etc/config/kratos/identity.schema.json - -courier: - smtp: - connection_uri: smtps://test:test@host.docker.internal:1025/?skip_ssl_verify=true - -feature_flags: - use_continue_with_transitions: true \ No newline at end of file diff --git a/self-hosting/docker/postgres/.gitignore b/self-hosting/docker/postgres/.gitignore deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/self-hosting/docker/postgres/scripts/init.sh b/self-hosting/docker/postgres/scripts/init.sh deleted file mode 100755 index 5e5b12df77..0000000000 --- a/self-hosting/docker/postgres/scripts/init.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash - -set -e -set -u - -function create_user_and_database() { - local database=$1 - echo "Creating database '$database' with user '$POSTGRES_USER'" - psql -c "CREATE DATABASE $database;" || { echo "Failed to create database '$database'"; exit 1; } - echo "Database '$database' created" -} - -if [ -n "$ADDITIONAL_DATABASES" ]; then - for i in ${ADDITIONAL_DATABASES//,/ } - do - create_user_and_database $i - done -fi diff --git a/self-hosting/docker/redis/.gitignore b/self-hosting/docker/redis/.gitignore deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/self-hosting/init.sh b/self-hosting/init.sh deleted file mode 100755 index 703d7c8776..0000000000 --- a/self-hosting/init.sh +++ /dev/null @@ -1,95 +0,0 @@ -#!/bin/bash - -# Self-Hosting Initialization -# -# Usage: -# -# ./init.sh 83f0ebdf-eafb-4c8d-bd7b-04ea07d61b7f localhost -# -# -# Flow: -# -# First, check to see if there is a VERSION file, if so, use that version. -# If not, then check for the first command line argument, if so, use that version. -# Else, prompt the user. -# -# First, check to see if there is a HOST file, if so, use that host. -# If not, then check for the first command line argument, if so, use that host. -# Else, prompt the user. - -REPO="https://github.com/quadratichq/quadratic.git" -BRANCH="self-hosting-setup" -DIR="self-hosting" -SELF_HOSTING_URI="https://selfhost.quadratic-preview.com" -INVALID_LICENSE_KEY="Invalid license key." - -get_license_key() { - read -p "Enter your license key (Get one for free instantly at $SELF_HOSTING_URI): " user_input - - if [[ $user_input =~ ^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$ ]]; then - echo $user_input - else - echo $INVALID_LICENSE_KEY - return 1 - fi -} - -get_host() { - read -p "What public host name or public IP address are you using for this setup (e.g. localhost, app.quadratic.com, or other): " user_input - - # TODO: validate host - echo $user_input -} - -checkout() { - git clone -b $BRANCH --filter=blob:none --no-checkout --depth 1 --sparse $REPO - cd quadratic - git sparse-checkout set ${DIR}/ - git checkout - -} - -LICENSE_KEY="" -HOST="" - -if [ -f "quadratic/LICENSE" ]; then - LICENSE_KEY=$( LICENSE - -# write host to HOST file -touch HOST -echo $HOST > HOST - -cp -a self-hosting/. . -rm -rf self-hosting -rm ../init.sh -rm init.sh - -# adding .bak for compatibility with both GNU (Linux) and BSD (MacOS) sed -sed -i.bak "s/#LICENSE_KEY#/$LICENSE_KEY/g" "docker-compose.yml" -sed -i.bak "s/#HOST#/$HOST/g" "docker-compose.yml" -sed -i.bak "s/#HOST#/$HOST/g" "docker/ory-auth/config/kratos.yml" - -rm docker-compose.yml.bak -rm docker/ory-auth/config/kratos.yml.bak - -sh start.sh diff --git a/self-hosting/start.sh b/self-hosting/start.sh deleted file mode 100755 index a11d58e1ca..0000000000 --- a/self-hosting/start.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/sh - -start() { - docker compose --profile "*" down - yes | docker compose rm quadratic-client - docker compose --profile "*" up -} - -start \ No newline at end of file diff --git a/self-hosting/stop.sh b/self-hosting/stop.sh deleted file mode 100755 index f47c32c3f6..0000000000 --- a/self-hosting/stop.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/sh - -stop() { - docker compose --profile "*" down -} - -stop \ No newline at end of file From 9b59b3f7927d80687c23e9733709ae5a8b7b8b88 Mon Sep 17 00:00:00 2001 From: David DiMaria Date: Fri, 13 Sep 2024 14:17:20 -0600 Subject: [PATCH 088/113] Update URLs to self-hosting portal production domain --- quadratic-api/src/env-vars.ts | 3 ++- quadratic-api/src/licenseClient.ts | 4 ++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/quadratic-api/src/env-vars.ts b/quadratic-api/src/env-vars.ts index a7c9b09812..fc5c6224d4 100644 --- a/quadratic-api/src/env-vars.ts +++ b/quadratic-api/src/env-vars.ts @@ -45,7 +45,8 @@ if (NODE_ENV === 'production') { } // Intentionally hard-coded to avoid this being environment-configurable -export const LICENSE_API_URI = 'https://selfhost.quadratic-preview.com'; +// NOTE: Modifying this license check is violating the Quadratic Terms and Conditions and is stealing software, and we will come after you. +export const LICENSE_API_URI = 'https://selfhost.quadratichq.com'; ensureSampleTokenNotUsedInProduction(); diff --git a/quadratic-api/src/licenseClient.ts b/quadratic-api/src/licenseClient.ts index a1ca469bed..3d05922009 100644 --- a/quadratic-api/src/licenseClient.ts +++ b/quadratic-api/src/licenseClient.ts @@ -1,3 +1,7 @@ +//! License Client +//! +//! Modifying this license check is violating the Quadratic Terms and Conditions and is stealing software, and we will come after you. + import axios from 'axios'; import { LicenseSchema } from 'quadratic-shared/typesAndSchemas'; import z from 'zod'; From 8c9c9de81a3ea44e56a1b20eb5816393e8339c11 Mon Sep 17 00:00:00 2001 From: David DiMaria Date: Fri, 13 Sep 2024 14:33:24 -0600 Subject: [PATCH 089/113] Fix clippy lints --- quadratic-api/src/licenseClient.ts | 2 -- quadratic-multiplayer/src/message/handle.rs | 2 +- quadratic-rust-shared/src/pubsub/redis.rs | 2 +- quadratic-rust-shared/src/pubsub/redis_streams.rs | 6 +++--- quadratic-rust-shared/src/sql/mod.rs | 4 ++-- quadratic-rust-shared/src/sql/mssql_connection.rs | 2 +- 6 files changed, 8 insertions(+), 10 deletions(-) diff --git a/quadratic-api/src/licenseClient.ts b/quadratic-api/src/licenseClient.ts index 3d05922009..515e2e51d1 100644 --- a/quadratic-api/src/licenseClient.ts +++ b/quadratic-api/src/licenseClient.ts @@ -9,8 +9,6 @@ import dbClient from './dbClient'; import { LICENSE_API_URI, LICENSE_KEY } from './env-vars'; type LicenseResponse = z.infer; -// const StatusEnum = LicenseSchema.shape.status; -// type StatusType = z.infer; let cachedResult: LicenseResponse | null = null; let lastCheckedTime: number | null = null; diff --git a/quadratic-multiplayer/src/message/handle.rs b/quadratic-multiplayer/src/message/handle.rs index 9f35a7d8a7..4dfae63479 100644 --- a/quadratic-multiplayer/src/message/handle.rs +++ b/quadratic-multiplayer/src/message/handle.rs @@ -202,7 +202,7 @@ pub(crate) async fn handle_message( let response = MessageResponse::Transaction { id, file_id, - operations: operations, + operations, sequence_num, }; diff --git a/quadratic-rust-shared/src/pubsub/redis.rs b/quadratic-rust-shared/src/pubsub/redis.rs index 5fb2e5811f..2d2edbf5e3 100644 --- a/quadratic-rust-shared/src/pubsub/redis.rs +++ b/quadratic-rust-shared/src/pubsub/redis.rs @@ -97,7 +97,7 @@ impl super::PubSub for RedisConnection { value: &[u8], _active_channel: Option<&str>, ) -> Result<()> { - self.multiplex.publish(channel, value).await?; + () = self.multiplex.publish(channel, value).await?; Ok(()) } diff --git a/quadratic-rust-shared/src/pubsub/redis_streams.rs b/quadratic-rust-shared/src/pubsub/redis_streams.rs index 4b0502df2c..9b7b8de089 100644 --- a/quadratic-rust-shared/src/pubsub/redis_streams.rs +++ b/quadratic-rust-shared/src/pubsub/redis_streams.rs @@ -153,14 +153,14 @@ impl super::PubSub for RedisConnection { /// Insert or update a key within an active channel async fn upsert_active_channel(&mut self, set_key: &str, channel: &str) -> Result<()> { let score = Utc::now().timestamp_millis(); - self.multiplex.zadd(set_key, channel, score).await?; + () = self.multiplex.zadd(set_key, channel, score).await?; Ok(()) } /// Remove an a key within an active channel async fn remove_active_channel(&mut self, set_key: &str, channel: &str) -> Result<()> { - self.multiplex.zrem(set_key, channel).await?; + () = self.multiplex.zrem(set_key, channel).await?; Ok(()) } @@ -195,7 +195,7 @@ impl super::PubSub for RedisConnection { active_channel: Option<&str>, ) -> Result<()> { // add the message to the stream - self.multiplex.xadd(channel, key, &[(key, value)]).await?; + () = self.multiplex.xadd(channel, key, &[(key, value)]).await?; // add the channel to the active channels set if let Some(active_channel) = active_channel { diff --git a/quadratic-rust-shared/src/sql/mod.rs b/quadratic-rust-shared/src/sql/mod.rs index 58fc845430..c6c2501090 100644 --- a/quadratic-rust-shared/src/sql/mod.rs +++ b/quadratic-rust-shared/src/sql/mod.rs @@ -77,7 +77,7 @@ pub trait Connection { for row in &data { for (col_index, col) in Self::row_columns(row).enumerate() { - let value = Self::to_arrow(row, &col, col_index); + let value = Self::to_arrow(row, col, col_index); transposed[col_index].push(value); } } @@ -93,7 +93,7 @@ pub trait Connection { .enumerate() .map(|(index, col)| { Field::new( - Self::column_name(&col).to_string(), + Self::column_name(col).to_string(), cols[index].data_type().to_owned(), true, ) diff --git a/quadratic-rust-shared/src/sql/mssql_connection.rs b/quadratic-rust-shared/src/sql/mssql_connection.rs index d928e23940..8999927357 100644 --- a/quadratic-rust-shared/src/sql/mssql_connection.rs +++ b/quadratic-rust-shared/src/sql/mssql_connection.rs @@ -297,7 +297,7 @@ where .unwrap_or(ArrowType::Void) } -fn convert_mssql_type_owned<'a, T, F>( +fn convert_mssql_type_owned( column_data: ColumnData<'static>, map_to_arrow_type: F, ) -> ArrowType From 414a74e3c0d633acd7ca23819b724a25699cd1bc Mon Sep 17 00:00:00 2001 From: David DiMaria Date: Fri, 13 Sep 2024 14:57:59 -0600 Subject: [PATCH 090/113] Fix some api test failures --- quadratic-api/.env.docker | 10 ++-------- quadratic-api/jest.setup.js | 2 +- 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/quadratic-api/.env.docker b/quadratic-api/.env.docker index 471e456797..eea0b280b9 100644 --- a/quadratic-api/.env.docker +++ b/quadratic-api/.env.docker @@ -10,14 +10,8 @@ M2M_AUTH_TOKEN=M2M_AUTH_TOKEN ENCRYPTION_KEY=eb4758047f74bdb2603cce75c4370327ca2c3662c4786867659126da8e64dfcc # Auth -AUTH_TYPE=ory -ORY_JWKS_URI="http://host.docker.internal:3000/.well-known/jwks.json" -ORY_ADMIN_HOST=http://host.docker.internal:4434 - -# Storage -STORAGE_TYPE=file-system -QUADRATIC_FILE_URI=http://host.docker.internal:3002 -QUADRATIC_FILE_URI_PUBLIC=http://localhost:3002 +AUTH_TYPE=auth0 +AWS_S3_ENDPOINT=http://localstack:4566 # Admin LICENSE_KEY=LICENSE_KEY diff --git a/quadratic-api/jest.setup.js b/quadratic-api/jest.setup.js index 46579e6f08..ba8f6593f4 100644 --- a/quadratic-api/jest.setup.js +++ b/quadratic-api/jest.setup.js @@ -26,7 +26,7 @@ jest.mock('./src/storage/storage', () => { uploadFile: jest.fn().mockImplementation(async () => { return { bucket: 'test-bucket', key: 'test-key' }; }), - uploadMiddleware: () => multerS3Storage, + uploadMiddleware: multerS3Storage, }; }); From ad9bcda5ce2a951d94fbf6b3e026ab10ff6b3479 Mon Sep 17 00:00:00 2001 From: David DiMaria Date: Fri, 13 Sep 2024 15:09:08 -0600 Subject: [PATCH 091/113] Mock licenseClient --- quadratic-api/jest.setup.js | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/quadratic-api/jest.setup.js b/quadratic-api/jest.setup.js index ba8f6593f4..fac9d779ee 100644 --- a/quadratic-api/jest.setup.js +++ b/quadratic-api/jest.setup.js @@ -18,6 +18,19 @@ jest.mock('./src/middleware/validateAccessToken', () => { }; }); +jest.mock('./src/licenseClient', () => { + return { + post: async () => { + return { + limits: { + seats: 10, + }, + status: 'active', + }; + }, + }; +}); + jest.mock('./src/storage/storage', () => { return { s3Client: {}, From 5e822a24b04747fd2f2231451bed476370094d04 Mon Sep 17 00:00:00 2001 From: David DiMaria Date: Fri, 13 Sep 2024 15:21:18 -0600 Subject: [PATCH 092/113] Fix quadratic-client lints --- quadratic-api/.env.docker | 3 +++ quadratic-client/src/app/ui/hooks/useAI.tsx | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/quadratic-api/.env.docker b/quadratic-api/.env.docker index eea0b280b9..f4f8144b51 100644 --- a/quadratic-api/.env.docker +++ b/quadratic-api/.env.docker @@ -13,5 +13,8 @@ ENCRYPTION_KEY=eb4758047f74bdb2603cce75c4370327ca2c3662c4786867659126da8e64dfcc AUTH_TYPE=auth0 AWS_S3_ENDPOINT=http://localstack:4566 +# Storage +STORAGE_TYPE=s3 + # Admin LICENSE_KEY=LICENSE_KEY diff --git a/quadratic-client/src/app/ui/hooks/useAI.tsx b/quadratic-client/src/app/ui/hooks/useAI.tsx index a2a0ae926b..9dd7d0df00 100644 --- a/quadratic-client/src/app/ui/hooks/useAI.tsx +++ b/quadratic-client/src/app/ui/hooks/useAI.tsx @@ -1,4 +1,4 @@ -import { authClient } from '@/auth'; +import { authClient } from '@/auth/auth'; import { AI } from '@/shared/constants/routes'; import { AIMessage, From 22df91530f2f66edaaeb4800ebb1731ba5ae66a9 Mon Sep 17 00:00:00 2001 From: David DiMaria Date: Fri, 13 Sep 2024 17:30:21 -0600 Subject: [PATCH 093/113] Fix quadratic-api tests --- quadratic-api/.env.test | 2 ++ quadratic-api/jest.setup.js | 18 +++++++++++------- quadratic-api/src/licenseClient.ts | 1 - quadratic-api/src/routes/v0/files.$uuid.GET.ts | 9 ++++----- quadratic-shared/package.json | 2 +- 5 files changed, 18 insertions(+), 14 deletions(-) diff --git a/quadratic-api/.env.test b/quadratic-api/.env.test index 9802401921..27d0e2cfca 100644 --- a/quadratic-api/.env.test +++ b/quadratic-api/.env.test @@ -1,3 +1,5 @@ +# DEBUG=express:* + DATABASE_URL='postgresql://prisma:prisma@localhost:5433/quadratic-api’' M2M_AUTH_TOKEN=M2M_AUTH_TOKEN STRIPE_SECRET_KEY=STRIPE_SECRET_KEY diff --git a/quadratic-api/jest.setup.js b/quadratic-api/jest.setup.js index fac9d779ee..ce11ad241b 100644 --- a/quadratic-api/jest.setup.js +++ b/quadratic-api/jest.setup.js @@ -18,15 +18,19 @@ jest.mock('./src/middleware/validateAccessToken', () => { }; }); +const licenseClientResponse = { + limits: { + seats: 10, + }, + status: 'active', +}; + jest.mock('./src/licenseClient', () => { return { - post: async () => { - return { - limits: { - seats: 10, - }, - status: 'active', - }; + licenseClient: { + post: async () => licenseClientResponse, + checkFromServer: async () => licenseClientResponse, + check: async () => licenseClientResponse, }, }; }); diff --git a/quadratic-api/src/licenseClient.ts b/quadratic-api/src/licenseClient.ts index 515e2e51d1..460885212a 100644 --- a/quadratic-api/src/licenseClient.ts +++ b/quadratic-api/src/licenseClient.ts @@ -42,7 +42,6 @@ export const licenseClient = { // Use cached result if within the cache duration return cachedResult; } - // Otherwise, perform the check const result = await licenseClient.checkFromServer(); diff --git a/quadratic-api/src/routes/v0/files.$uuid.GET.ts b/quadratic-api/src/routes/v0/files.$uuid.GET.ts index 9a9c960a9e..fc2e5cf781 100644 --- a/quadratic-api/src/routes/v0/files.$uuid.GET.ts +++ b/quadratic-api/src/routes/v0/files.$uuid.GET.ts @@ -9,7 +9,6 @@ import { validateOptionalAccessToken } from '../../middleware/validateOptionalAc import { validateRequestSchema } from '../../middleware/validateRequestSchema'; import { getFileUrl } from '../../storage/storage'; import { RequestWithOptionalUser } from '../../types/Request'; -import { ResponseError } from '../../types/Response'; export default [ validateRequestSchema( @@ -24,10 +23,7 @@ export default [ handler, ]; -async function handler( - req: RequestWithOptionalUser, - res: Response -) { +async function handler(req: RequestWithOptionalUser, res: Response) { const userId = req.user?.id; const { file: { id, thumbnail, uuid, name, createdDate, updatedDate, publicLinkAccess, ownerUserId, ownerTeam }, @@ -45,6 +41,7 @@ async function handler( sequenceNumber: 'desc', }, }); + if (!checkpoint) { return res.status(500).json({ error: { message: 'No Checkpoints exist for this file' } }); } @@ -62,7 +59,9 @@ async function handler( fileTeamPrivacy = 'PUBLIC_TO_TEAM'; } + console.log('a'); const license = await licenseClient.check(); + console.log('b'); if (license === null) { return res.status(500).json({ error: { message: 'Unable to retrieve license' } }); diff --git a/quadratic-shared/package.json b/quadratic-shared/package.json index 0febb95417..e1d67b45fa 100644 --- a/quadratic-shared/package.json +++ b/quadratic-shared/package.json @@ -4,7 +4,7 @@ "description": "Code shared between quadratic-api and quadratic-client (each app has to compile this code itself)", "main": "index.js", "scripts": { - "compile": "tsc ./*.ts", + "compile": "tsc --project ./tsconfig.json", "test": "echo \"Error: no test specified\" && exit 1" }, "author": "", From fa43dac1b9b2044db8a495823fa60d1778b6abc4 Mon Sep 17 00:00:00 2001 From: David DiMaria Date: Mon, 16 Sep 2024 12:57:47 -0600 Subject: [PATCH 094/113] Create .env.test for quadratic-client to fix unit tests --- quadratic-client/.env.test | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 quadratic-client/.env.test diff --git a/quadratic-client/.env.test b/quadratic-client/.env.test new file mode 100644 index 0000000000..079422f782 --- /dev/null +++ b/quadratic-client/.env.test @@ -0,0 +1,8 @@ +VITE_DEBUG=1 // use =1 to enable debug flags +VITE_QUADRATIC_API_URL=http://localhost:8000 +VITE_QUADRATIC_MULTIPLAYER_URL=ws://localhost:3001/ws +VITE_QUADRATIC_CONNECTION_URL=http://localhost:3003 + +# Auth +VITE_AUTH_TYPE=ory # auth0 or ory +VITE_ORY_HOST=http://localhost:4433 From e3f8b8c973f65a1cec69d4978390086176f20c5c Mon Sep 17 00:00:00 2001 From: David DiMaria Date: Mon, 16 Sep 2024 13:13:16 -0600 Subject: [PATCH 095/113] Prefer npm run compile --workspace=quadratic-shared in the client Dockerfile --- client.Dockerfile | 2 +- quadratic-client/Dockerfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/client.Dockerfile b/client.Dockerfile index dc4369d120..ea1366e10e 100644 --- a/client.Dockerfile +++ b/client.Dockerfile @@ -57,7 +57,7 @@ RUN echo 'Building quadratic-rust-client...' && npm run build --workspace=quadra # Build the quadratic-shared WORKDIR /app -RUN echo 'Building quadratic-shared...' && npx tsc ./quadratic-shared/*.ts +RUN echo 'Building quadratic-shared...' && npm run compile --workspace=quadratic-shared # Build the front-end WORKDIR /app diff --git a/quadratic-client/Dockerfile b/quadratic-client/Dockerfile index 1633f99787..d58567287b 100644 --- a/quadratic-client/Dockerfile +++ b/quadratic-client/Dockerfile @@ -57,7 +57,7 @@ RUN echo 'Building quadratic-rust-client...' && npm run build --workspace=quadra # Build the quadratic-shared WORKDIR /app -RUN echo 'Building quadratic-shared...' && npx tsc ./quadratic-shared/*.ts +RUN echo 'Building quadratic-shared...' && npm run compile --workspace=quadratic-shared # Build the front-end WORKDIR /app From 8151a40119c87fe3b645619373e7b49e2627b6d5 Mon Sep 17 00:00:00 2001 From: David DiMaria Date: Wed, 23 Oct 2024 12:04:19 -0600 Subject: [PATCH 096/113] Add in bypass for quadratic license key --- quadratic-api/src/licenseClient.ts | 14 ++++++++++++++ quadratic-api/src/server.ts | 3 ++- quadratic-api/src/utils/crypto.test.ts | 9 ++++++++- quadratic-api/src/utils/crypto.ts | 7 +++++++ 4 files changed, 31 insertions(+), 2 deletions(-) diff --git a/quadratic-api/src/licenseClient.ts b/quadratic-api/src/licenseClient.ts index 460885212a..d65e1c7117 100644 --- a/quadratic-api/src/licenseClient.ts +++ b/quadratic-api/src/licenseClient.ts @@ -7,6 +7,7 @@ import { LicenseSchema } from 'quadratic-shared/typesAndSchemas'; import z from 'zod'; import dbClient from './dbClient'; import { LICENSE_API_URI, LICENSE_KEY } from './env-vars'; +import { encryptFromEnv, hash } from './utils/crypto'; type LicenseResponse = z.infer; @@ -31,10 +32,23 @@ export const licenseClient = { } }, checkFromServer: async (): Promise => { + // NOTE: Modifying this license check is violating the Quadratic Terms and Conditions and is stealing software, and we will come after you. + if (hash(LICENSE_KEY) === '2ef876ddfe6cc783b83ac63cbef0ae84e6807c69fa72066801f130706e2a935a') { + return licenseClient.adminLicenseResponse(); + } + const userCount = await dbClient.user.count(); return licenseClient.post(userCount); }, + adminLicenseResponse: async (): Promise => { + return { + limits: { + seats: 100000000000, + }, + status: 'active', + }; + }, check: async (): Promise => { const currentTime = Date.now(); diff --git a/quadratic-api/src/server.ts b/quadratic-api/src/server.ts index 4d28f3b718..2b34f90a59 100644 --- a/quadratic-api/src/server.ts +++ b/quadratic-api/src/server.ts @@ -1,5 +1,6 @@ import { app } from './app'; -import { PORT } from './env-vars'; +import { ENCRYPTION_KEY, LICENSE_KEY, PORT } from './env-vars'; +import { encryptFromEnv, hash } from './utils/crypto'; // Start the server app.listen(PORT, () => { diff --git a/quadratic-api/src/utils/crypto.test.ts b/quadratic-api/src/utils/crypto.test.ts index 43e70b79d6..a228b42fdf 100644 --- a/quadratic-api/src/utils/crypto.test.ts +++ b/quadratic-api/src/utils/crypto.test.ts @@ -1,5 +1,5 @@ import crypto from 'crypto'; -import { decrypt, encrypt } from './crypto'; +import { decrypt, encrypt, hash } from './crypto'; // Convert a hex string to a buffer. // @@ -13,6 +13,13 @@ describe('Encryption and Decryption', () => { const key = hexStringToBuffer(keyBytes.toString('hex')); const text = 'Hello, world!'; + it('should hash a value', () => { + const hashed = hash(text); + const expected = '315f5bdb76d078c43b8ac0064e4a0164612b1fce77c869345bfc94c75894edd3'; + + expect(hashed).toEqual(expected); + }); + it('should convert a hex to a buffer', () => { const hex = keyBytes.toString('hex'); const buffer = hexStringToBuffer(hex); diff --git a/quadratic-api/src/utils/crypto.ts b/quadratic-api/src/utils/crypto.ts index 4f7e5a0360..98a290c859 100644 --- a/quadratic-api/src/utils/crypto.ts +++ b/quadratic-api/src/utils/crypto.ts @@ -7,6 +7,13 @@ const algorithm = 'aes-256-cbc'; // Get the encryption key from the env and convert it to a buffer. const encryption_key = Buffer.from(ENCRYPTION_KEY, 'hex'); +export const hash = (text: string): string => { + const hash = crypto.createHash('sha256'); + hash.update(text); + + return hash.digest('hex'); +}; + // Encrypts the given text using the given key. // Store the IV with the encrypted text (prepended). export const encrypt = (key: Buffer, text: string): string => { From e8cdd2936c1d9631f79731db240bbc5adaa85a63 Mon Sep 17 00:00:00 2001 From: David Kircos Date: Wed, 23 Oct 2024 14:00:05 -0600 Subject: [PATCH 097/113] productionize image creation --- .github/workflows/production-publish-images.yml | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/.github/workflows/production-publish-images.yml b/.github/workflows/production-publish-images.yml index 25a1d567ef..4ef590b9e2 100644 --- a/.github/workflows/production-publish-images.yml +++ b/.github/workflows/production-publish-images.yml @@ -3,7 +3,8 @@ name: Build and Publish Images to ECR on: push: branches: - - self-hosting-setup + - self-hosting-setup #remove + - main concurrency: group: production-publish-images @@ -33,7 +34,7 @@ jobs: - name: Define repository name id: repo-name run: | - echo "REPO_NAME=quadratic-${{ matrix.service }}-staging" >> $GITHUB_OUTPUT + echo "REPO_NAME=quadratic-${{ matrix.service }}" >> $GITHUB_OUTPUT - name: Create Public ECR Repository if not exists id: create-ecr @@ -45,10 +46,14 @@ jobs: ECR_URL=$(echo $REPO_INFO | jq -r '.repositories[0].repositoryUri') echo "ECR_URL=$ECR_URL" >> $GITHUB_OUTPUT + - name: Read VERSION file + id: version + run: echo "VERSION=$(cat VERSION)" >> $GITHUB_OUTPUT + - name: Build, Tag, and Push Image to Amazon ECR Public env: ECR_URL: ${{ steps.create-ecr.outputs.ECR_URL }} - IMAGE_TAG: 0.1.0 + IMAGE_TAG: ${{ steps.version.outputs.VERSION }} run: | docker build -t $ECR_URL:$IMAGE_TAG -t $ECR_URL:latest -f quadratic-${{ matrix.service }}/Dockerfile . docker push $ECR_URL:$IMAGE_TAG From 9e6fc6bf6724bb8b6d3b87e89f36be525abe7fee Mon Sep 17 00:00:00 2001 From: David DiMaria Date: Wed, 23 Oct 2024 14:47:21 -0600 Subject: [PATCH 098/113] Address PR feedback --- quadratic-api/src/routes/v0/files.$uuid.GET.ts | 13 ++++++++----- quadratic-api/src/routes/v0/teams.$uuid.GET.ts | 3 ++- quadratic-client/package.json | 1 - .../app/web-workers/pythonLanguageServer/client.ts | 2 +- 4 files changed, 11 insertions(+), 8 deletions(-) diff --git a/quadratic-api/src/routes/v0/files.$uuid.GET.ts b/quadratic-api/src/routes/v0/files.$uuid.GET.ts index fc2e5cf781..b83847f0bc 100644 --- a/quadratic-api/src/routes/v0/files.$uuid.GET.ts +++ b/quadratic-api/src/routes/v0/files.$uuid.GET.ts @@ -9,6 +9,8 @@ import { validateOptionalAccessToken } from '../../middleware/validateOptionalAc import { validateRequestSchema } from '../../middleware/validateRequestSchema'; import { getFileUrl } from '../../storage/storage'; import { RequestWithOptionalUser } from '../../types/Request'; +import { ResponseError } from '@sendgrid/mail'; +import { ApiError } from '../../utils/ApiError'; export default [ validateRequestSchema( @@ -23,7 +25,10 @@ export default [ handler, ]; -async function handler(req: RequestWithOptionalUser, res: Response) { +async function handler( + req: RequestWithOptionalUser, + res: Response +) { const userId = req.user?.id; const { file: { id, thumbnail, uuid, name, createdDate, updatedDate, publicLinkAccess, ownerUserId, ownerTeam }, @@ -43,7 +48,7 @@ async function handler(req: RequestWithOptionalUser, res: Response) { }); if (!checkpoint) { - return res.status(500).json({ error: { message: 'No Checkpoints exist for this file' } }); + throw new ApiError(500, 'No Checkpoints exist for this file'); } const lastCheckpointDataUrl = await getFileUrl(checkpoint.s3Key); @@ -59,12 +64,10 @@ async function handler(req: RequestWithOptionalUser, res: Response) { fileTeamPrivacy = 'PUBLIC_TO_TEAM'; } - console.log('a'); const license = await licenseClient.check(); - console.log('b'); if (license === null) { - return res.status(500).json({ error: { message: 'Unable to retrieve license' } }); + throw new ApiError(500, 'Unable to retrieve license'); } const data = { diff --git a/quadratic-api/src/routes/v0/teams.$uuid.GET.ts b/quadratic-api/src/routes/v0/teams.$uuid.GET.ts index 6fc904ad36..d4f73c6fe8 100644 --- a/quadratic-api/src/routes/v0/teams.$uuid.GET.ts +++ b/quadratic-api/src/routes/v0/teams.$uuid.GET.ts @@ -12,6 +12,7 @@ import { getPresignedFileUrl } from '../../storage/storage'; import { RequestWithUser } from '../../types/Request'; import { ResponseError } from '../../types/Response'; import { getFilePermissions } from '../../utils/permissions'; +import { ApiError } from '../../utils/ApiError'; export default [validateAccessToken, userMiddleware, handler]; @@ -111,7 +112,7 @@ async function handler(req: Request, res: Response Date: Wed, 23 Oct 2024 18:56:09 -0600 Subject: [PATCH 099/113] Refresh user count when added or removed from a team --- quadratic-api/src/internal/addUserToTeam.ts | 4 ++++ quadratic-api/src/internal/removeUserFromTeam.ts | 4 ++++ quadratic-api/src/licenseClient.ts | 9 +++++++-- quadratic-api/src/routes/v0/files.$uuid.GET.ts | 2 +- quadratic-api/src/routes/v0/teams.$uuid.GET.ts | 2 +- 5 files changed, 17 insertions(+), 4 deletions(-) diff --git a/quadratic-api/src/internal/addUserToTeam.ts b/quadratic-api/src/internal/addUserToTeam.ts index 2c8fadf2cd..0bce08cf28 100644 --- a/quadratic-api/src/internal/addUserToTeam.ts +++ b/quadratic-api/src/internal/addUserToTeam.ts @@ -1,5 +1,6 @@ import { TeamRole } from '@prisma/client'; import dbClient from '../dbClient'; +import { licenseClient } from '../licenseClient'; export const addUserToTeam = async (args: { userId: number; teamId: number; role: TeamRole }) => { const { userId, teamId, role } = args; @@ -13,6 +14,9 @@ export const addUserToTeam = async (args: { userId: number; teamId: number; role }, }); + // update user count in the license server + await licenseClient.check(true); + // Update the seat quantity on the team's stripe subscription // await updateSeatQuantity(teamId); diff --git a/quadratic-api/src/internal/removeUserFromTeam.ts b/quadratic-api/src/internal/removeUserFromTeam.ts index 1e5f2ac94d..63fb5e14cf 100644 --- a/quadratic-api/src/internal/removeUserFromTeam.ts +++ b/quadratic-api/src/internal/removeUserFromTeam.ts @@ -1,4 +1,5 @@ import dbClient from '../dbClient'; +import { licenseClient } from '../licenseClient'; export const removeUserFromTeam = async (userId: number, teamId: number) => { await dbClient.$transaction(async (prisma) => { @@ -23,6 +24,9 @@ export const removeUserFromTeam = async (userId: number, teamId: number) => { }); }); + // update user count in the license server + await licenseClient.check(true); + // Update the seat quantity on the team's stripe subscription // await updateSeatQuantity(teamId); }; diff --git a/quadratic-api/src/licenseClient.ts b/quadratic-api/src/licenseClient.ts index d65e1c7117..2044663733 100644 --- a/quadratic-api/src/licenseClient.ts +++ b/quadratic-api/src/licenseClient.ts @@ -49,10 +49,15 @@ export const licenseClient = { status: 'active', }; }, - check: async (): Promise => { + /** + * + * @param force boolean to force a license check (ignoring the cache) + * @returns + */ + check: async (force: boolean): Promise => { const currentTime = Date.now(); - if (cachedResult && lastCheckedTime && currentTime - lastCheckedTime < cacheDuration) { + if (!force && cachedResult && lastCheckedTime && currentTime - lastCheckedTime < cacheDuration) { // Use cached result if within the cache duration return cachedResult; } diff --git a/quadratic-api/src/routes/v0/files.$uuid.GET.ts b/quadratic-api/src/routes/v0/files.$uuid.GET.ts index b83847f0bc..41062392a0 100644 --- a/quadratic-api/src/routes/v0/files.$uuid.GET.ts +++ b/quadratic-api/src/routes/v0/files.$uuid.GET.ts @@ -64,7 +64,7 @@ async function handler( fileTeamPrivacy = 'PUBLIC_TO_TEAM'; } - const license = await licenseClient.check(); + const license = await licenseClient.check(false); if (license === null) { throw new ApiError(500, 'Unable to retrieve license'); diff --git a/quadratic-api/src/routes/v0/teams.$uuid.GET.ts b/quadratic-api/src/routes/v0/teams.$uuid.GET.ts index d4f73c6fe8..4c6a07ff06 100644 --- a/quadratic-api/src/routes/v0/teams.$uuid.GET.ts +++ b/quadratic-api/src/routes/v0/teams.$uuid.GET.ts @@ -109,7 +109,7 @@ async function handler(req: Request, res: Response Date: Thu, 24 Oct 2024 08:27:34 -0600 Subject: [PATCH 100/113] Fix lint issue related to different event emitter library --- .../src/app/web-workers/pythonLanguageServer/client.ts | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/quadratic-client/src/app/web-workers/pythonLanguageServer/client.ts b/quadratic-client/src/app/web-workers/pythonLanguageServer/client.ts index eb6ab4773e..190afc1852 100644 --- a/quadratic-client/src/app/web-workers/pythonLanguageServer/client.ts +++ b/quadratic-client/src/app/web-workers/pythonLanguageServer/client.ts @@ -50,9 +50,12 @@ export class LanguageServerClient extends EventEmitter { super(); } - on(event: 'diagnostics', listener: (params: PublishDiagnosticsParams) => void): this { - super.on(event, listener); - return this; + on(event: T, listener: (...args: any[]) => void): this { + return super.on(event, listener); + } + + onDiagnostics(listener: (params: PublishDiagnosticsParams) => void): this { + return super.on('diagnostics', listener); } currentDiagnostics(uri: string): Diagnostic[] { From 398ee36cebaae3e4e369207b592ca488386c663c Mon Sep 17 00:00:00 2001 From: David Kircos Date: Thu, 24 Oct 2024 12:49:08 -0600 Subject: [PATCH 101/113] productionize --- .github/workflows/production-publish-images.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/production-publish-images.yml b/.github/workflows/production-publish-images.yml index 4ef590b9e2..5e4315b3e9 100644 --- a/.github/workflows/production-publish-images.yml +++ b/.github/workflows/production-publish-images.yml @@ -21,8 +21,8 @@ jobs: - name: Configure AWS Credentials uses: aws-actions/configure-aws-credentials@v4 with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID_DEVELOPMENT }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY_DEVELOPMENT }} + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} aws-region: us-east-1 - name: Login to Amazon ECR Public From 764324bed68f1751ca524d8b207d2dd2c85d8ce8 Mon Sep 17 00:00:00 2001 From: "David D." Date: Wed, 30 Oct 2024 08:53:13 -0600 Subject: [PATCH 102/113] Update _dashboard.tsx, remove useTheme --- quadratic-client/src/routes/_dashboard.tsx | 1 - 1 file changed, 1 deletion(-) diff --git a/quadratic-client/src/routes/_dashboard.tsx b/quadratic-client/src/routes/_dashboard.tsx index 2edfc0680d..4a27657e2d 100644 --- a/quadratic-client/src/routes/_dashboard.tsx +++ b/quadratic-client/src/routes/_dashboard.tsx @@ -8,7 +8,6 @@ import { NewFileDialog } from '@/dashboard/components/NewFileDialog'; import { apiClient } from '@/shared/api/apiClient'; import { MenuIcon } from '@/shared/components/Icons'; import { ROUTES, ROUTE_LOADER_IDS, SEARCH_PARAMS } from '@/shared/constants/routes'; -import { useTheme } from '@/shared/hooks/useTheme'; import { CONTACT_URL } from '@/shared/constants/urls'; import { Button } from '@/shared/shadcn/ui/button'; import { Sheet, SheetContent, SheetTrigger } from '@/shared/shadcn/ui/sheet'; From 6fdcf3e51bbdc7a0ef1a2ae999d78dcbdf7c046e Mon Sep 17 00:00:00 2001 From: David DiMaria Date: Wed, 30 Oct 2024 13:39:04 -0600 Subject: [PATCH 103/113] Fix import issue --- quadratic-client/src/routes/_dashboard.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/quadratic-client/src/routes/_dashboard.tsx b/quadratic-client/src/routes/_dashboard.tsx index 4a27657e2d..e139089340 100644 --- a/quadratic-client/src/routes/_dashboard.tsx +++ b/quadratic-client/src/routes/_dashboard.tsx @@ -8,7 +8,7 @@ import { NewFileDialog } from '@/dashboard/components/NewFileDialog'; import { apiClient } from '@/shared/api/apiClient'; import { MenuIcon } from '@/shared/components/Icons'; import { ROUTES, ROUTE_LOADER_IDS, SEARCH_PARAMS } from '@/shared/constants/routes'; -import { CONTACT_URL } from '@/shared/constants/urls'; +import { CONTACT_URL, SCHEDULE_MEETING } from '@/shared/constants/urls'; import { Button } from '@/shared/shadcn/ui/button'; import { Sheet, SheetContent, SheetTrigger } from '@/shared/shadcn/ui/sheet'; import { TooltipProvider } from '@/shared/shadcn/ui/tooltip'; From 38ba7aff068150d1be875d6d0c4b786566f0c8e2 Mon Sep 17 00:00:00 2001 From: David DiMaria Date: Wed, 30 Oct 2024 14:46:02 -0600 Subject: [PATCH 104/113] Try export-table option when building rust-client --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 962f4d4195..f456475e5a 100644 --- a/package.json +++ b/package.json @@ -33,7 +33,7 @@ "watch:wasm:perf:javascript": "cd quadratic-core && cargo watch -s 'wasm-pack build --target web --out-dir ../quadratic-client/src/app/quadratic-core --weak-refs'", "build:wasm:perf:javascript": "cd quadratic-core && wasm-pack build --target web --out-dir ../quadratic-client/src/app/quadratic-core --weak-refs", "watch:rust-client": "cd quadratic-rust-client && cargo watch -s 'wasm-pack build --dev --target web --out-dir ../quadratic-client/src/app/quadratic-rust-client --weak-refs'", - "build:rust-client": "cd quadratic-rust-client && wasm-pack build --target web --out-dir ../quadratic-client/src/app/quadratic-rust-client --weak-refs", + "build:rust-client": "cd quadratic-rust-client && wasm-pack build --target web --out-dir ../quadratic-client/src/app/quadratic-rust-client --weak-refs -- --export-table", "watch:python": "cd quadratic-kernels/python-wasm && npm run dev", "build:python": "./quadratic-kernels/python-wasm/package.sh", "build:file:blank": "cd quadratic-core && cargo run --bin generate_blank_current_file", From b4357235071df51d8685c24774e3fb55e0cf1918 Mon Sep 17 00:00:00 2001 From: David DiMaria Date: Wed, 30 Oct 2024 16:16:39 -0600 Subject: [PATCH 105/113] Remove wasm target --- quadratic-client/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/quadratic-client/Dockerfile b/quadratic-client/Dockerfile index d58567287b..802106032f 100644 --- a/quadratic-client/Dockerfile +++ b/quadratic-client/Dockerfile @@ -9,7 +9,7 @@ ENV PATH="/root/.cargo/bin:${PATH}" RUN echo 'Installing wasm-pack...' && curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh # Install wasm32-unknown-unknown target -RUN rustup target add wasm32-unknown-unknown +# RUN rustup target add wasm32-unknown-unknown # Install python, binaryen & clean up RUN apt-get update && apt-get install -y python-is-python3 python3-pip binaryen && apt-get clean && rm -rf /var/lib/apt/lists/* From 19ac8423d1101cd5b79e8caa9c967030c33b8777 Mon Sep 17 00:00:00 2001 From: David DiMaria Date: Thu, 31 Oct 2024 14:14:28 -0600 Subject: [PATCH 106/113] Remove export-tables option --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index f456475e5a..962f4d4195 100644 --- a/package.json +++ b/package.json @@ -33,7 +33,7 @@ "watch:wasm:perf:javascript": "cd quadratic-core && cargo watch -s 'wasm-pack build --target web --out-dir ../quadratic-client/src/app/quadratic-core --weak-refs'", "build:wasm:perf:javascript": "cd quadratic-core && wasm-pack build --target web --out-dir ../quadratic-client/src/app/quadratic-core --weak-refs", "watch:rust-client": "cd quadratic-rust-client && cargo watch -s 'wasm-pack build --dev --target web --out-dir ../quadratic-client/src/app/quadratic-rust-client --weak-refs'", - "build:rust-client": "cd quadratic-rust-client && wasm-pack build --target web --out-dir ../quadratic-client/src/app/quadratic-rust-client --weak-refs -- --export-table", + "build:rust-client": "cd quadratic-rust-client && wasm-pack build --target web --out-dir ../quadratic-client/src/app/quadratic-rust-client --weak-refs", "watch:python": "cd quadratic-kernels/python-wasm && npm run dev", "build:python": "./quadratic-kernels/python-wasm/package.sh", "build:file:blank": "cd quadratic-core && cargo run --bin generate_blank_current_file", From e65c3e663f4d23c6d5edd220d81db306a97cbcb8 Mon Sep 17 00:00:00 2001 From: David DiMaria Date: Thu, 31 Oct 2024 15:26:22 -0600 Subject: [PATCH 107/113] Fix kratos creation bug --- docker/postgres/scripts/init.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/postgres/scripts/init.sh b/docker/postgres/scripts/init.sh index 49102ba731..5e5b12df77 100755 --- a/docker/postgres/scripts/init.sh +++ b/docker/postgres/scripts/init.sh @@ -13,6 +13,6 @@ function create_user_and_database() { if [ -n "$ADDITIONAL_DATABASES" ]; then for i in ${ADDITIONAL_DATABASES//,/ } do - create_user_and_database $1 + create_user_and_database $i done fi From f43d0515ed79bdb00c00cfe6fa15a3c80bf917c6 Mon Sep 17 00:00:00 2001 From: David Kircos Date: Mon, 4 Nov 2024 11:46:07 -0700 Subject: [PATCH 108/113] echo versions --- quadratic-client/Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/quadratic-client/Dockerfile b/quadratic-client/Dockerfile index 802106032f..dc18458c03 100644 --- a/quadratic-client/Dockerfile +++ b/quadratic-client/Dockerfile @@ -7,6 +7,8 @@ ENV PATH="/root/.cargo/bin:${PATH}" # Install wasm-pack RUN echo 'Installing wasm-pack...' && curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh +RUN echo 'wasm-pack version:' && wasm-pack --version +RUN echo 'wasm-opt version:' && wasm-opt --version # Install wasm32-unknown-unknown target # RUN rustup target add wasm32-unknown-unknown From 249e0c92aa35589d78c284f7524def927ec85c42 Mon Sep 17 00:00:00 2001 From: David Kircos Date: Mon, 4 Nov 2024 11:56:37 -0700 Subject: [PATCH 109/113] try --- quadratic-client/Dockerfile | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/quadratic-client/Dockerfile b/quadratic-client/Dockerfile index dc18458c03..e814a28455 100644 --- a/quadratic-client/Dockerfile +++ b/quadratic-client/Dockerfile @@ -8,13 +8,12 @@ ENV PATH="/root/.cargo/bin:${PATH}" # Install wasm-pack RUN echo 'Installing wasm-pack...' && curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh RUN echo 'wasm-pack version:' && wasm-pack --version -RUN echo 'wasm-opt version:' && wasm-opt --version # Install wasm32-unknown-unknown target # RUN rustup target add wasm32-unknown-unknown -# Install python, binaryen & clean up -RUN apt-get update && apt-get install -y python-is-python3 python3-pip binaryen && apt-get clean && rm -rf /var/lib/apt/lists/* +# Install python & clean up +RUN apt-get update && apt-get install -y python-is-python3 python3-pip && apt-get clean && rm -rf /var/lib/apt/lists/* # Install npm dependencies WORKDIR /app From bea8d4f332cbedf992cee91ecc747a2d1c20defa Mon Sep 17 00:00:00 2001 From: David DiMaria Date: Mon, 4 Nov 2024 15:53:02 -0700 Subject: [PATCH 110/113] Respect the S3 storage type on the client --- package-lock.json | 1 - quadratic-api/src/licenseClient.ts | 4 ++-- quadratic-client/.env.docker | 3 ++- quadratic-client/.env.example | 3 ++- quadratic-client/.env.test | 3 ++- .../web-workers/quadraticCore/worker/core.ts | 22 ++++++++++++------- .../quadraticCore/worker/coreClient.ts | 5 ++++- 7 files changed, 26 insertions(+), 15 deletions(-) diff --git a/package-lock.json b/package-lock.json index a26e400108..c991d65442 100644 --- a/package-lock.json +++ b/package-lock.json @@ -28243,7 +28243,6 @@ "color": "^4.2.3", "esbuild-wasm": "^0.20.2", "eventemitter3": "^5.0.1", - "events": "^3.3.0", "fontfaceobserver": "^2.3.0", "fuzzysort": "^2.0.4", "localforage": "^1.10.0", diff --git a/quadratic-api/src/licenseClient.ts b/quadratic-api/src/licenseClient.ts index 2044663733..245dad7294 100644 --- a/quadratic-api/src/licenseClient.ts +++ b/quadratic-api/src/licenseClient.ts @@ -7,7 +7,7 @@ import { LicenseSchema } from 'quadratic-shared/typesAndSchemas'; import z from 'zod'; import dbClient from './dbClient'; import { LICENSE_API_URI, LICENSE_KEY } from './env-vars'; -import { encryptFromEnv, hash } from './utils/crypto'; +import { hash } from './utils/crypto'; type LicenseResponse = z.infer; @@ -25,7 +25,7 @@ export const licenseClient = { return LicenseSchema.parse(response.data) as LicenseResponse; } catch (err) { if (err instanceof Error) { - console.error('Failed to get the license info from the license service', err.message); + console.error('Failed to get the license info from the license service:', err.message); } return null; diff --git a/quadratic-client/.env.docker b/quadratic-client/.env.docker index 079422f782..ee0c206b50 100644 --- a/quadratic-client/.env.docker +++ b/quadratic-client/.env.docker @@ -4,5 +4,6 @@ VITE_QUADRATIC_MULTIPLAYER_URL=ws://localhost:3001/ws VITE_QUADRATIC_CONNECTION_URL=http://localhost:3003 # Auth -VITE_AUTH_TYPE=ory # auth0 or ory +VITE_AUTH_TYPE=ory +VITE_STORAGE_TYPE=file-system VITE_ORY_HOST=http://localhost:4433 diff --git a/quadratic-client/.env.example b/quadratic-client/.env.example index c433de836a..59862ad5c7 100644 --- a/quadratic-client/.env.example +++ b/quadratic-client/.env.example @@ -8,7 +8,8 @@ VITE_QUADRATIC_MULTIPLAYER_URL=ws://localhost:3001/ws VITE_QUADRATIC_CONNECTION_URL=http://localhost:3003 # Auth -VITE_AUTH_TYPE=auth0 # auth0 or ory +VITE_AUTH_TYPE=auth0 +VITE_STORAGE_TYPE=s3 VITE_AUTH0_ISSUER=https://quadratic-community.us.auth0.com/ VITE_AUTH0_DOMAIN=quadratic-community.us.auth0.com VITE_AUTH0_CLIENT_ID=DCPCvqyU5Q0bJD8Q3QmJEoV48x1zLH7W diff --git a/quadratic-client/.env.test b/quadratic-client/.env.test index 079422f782..2d1cdff22f 100644 --- a/quadratic-client/.env.test +++ b/quadratic-client/.env.test @@ -4,5 +4,6 @@ VITE_QUADRATIC_MULTIPLAYER_URL=ws://localhost:3001/ws VITE_QUADRATIC_CONNECTION_URL=http://localhost:3003 # Auth -VITE_AUTH_TYPE=ory # auth0 or ory +VITE_AUTH_TYPE=auth0 +VITE_STORAGE_TYPE=s3 VITE_ORY_HOST=http://localhost:4433 diff --git a/quadratic-client/src/app/web-workers/quadraticCore/worker/core.ts b/quadratic-client/src/app/web-workers/quadraticCore/worker/core.ts index bc76171925..5b3862b5b1 100644 --- a/quadratic-client/src/app/web-workers/quadraticCore/worker/core.ts +++ b/quadratic-client/src/app/web-workers/quadraticCore/worker/core.ts @@ -59,13 +59,15 @@ class Core { private clientQueue: Function[] = []; private renderQueue: Function[] = []; - private async loadGridFile(file: string): Promise { + private async loadGridFile(file: string, addToken: boolean): Promise { const jwt = await coreClient.getJwt(); - const res = await fetch(file, { - headers: { - Authorization: `Bearer ${jwt}`, - }, - }); + let requestInit = {}; + + if (addToken) { + requestInit = { headers: { Authorization: `Bearer ${jwt}` } }; + } + + const res = await fetch(file, requestInit); return new Uint8Array(await res.arrayBuffer()); } @@ -88,9 +90,13 @@ class Core { }; // Creates a Grid from a file. Initializes bother coreClient and coreRender w/metadata. - async loadFile(message: ClientCoreLoad, renderPort: MessagePort): Promise<{ version: string } | { error: string }> { + async loadFile( + message: ClientCoreLoad, + renderPort: MessagePort, + addToken: boolean + ): Promise<{ version: string } | { error: string }> { coreRender.init(renderPort); - const results = await Promise.all([this.loadGridFile(message.url), initCore()]); + const results = await Promise.all([this.loadGridFile(message.url, addToken), initCore()]); try { this.gridController = GridController.newFromFile(results[0], message.sequenceNumber, true); } catch (e) { diff --git a/quadratic-client/src/app/web-workers/quadraticCore/worker/coreClient.ts b/quadratic-client/src/app/web-workers/quadraticCore/worker/coreClient.ts index d2b368d029..83a29860fe 100644 --- a/quadratic-client/src/app/web-workers/quadraticCore/worker/coreClient.ts +++ b/quadratic-client/src/app/web-workers/quadraticCore/worker/coreClient.ts @@ -137,10 +137,13 @@ class CoreClient { switch (e.data.type) { case 'clientCoreLoad': await offline.init(e.data.fileId); + + const addToken = this.env.STORAGE_TYPE === 'file-system'; + this.send({ type: 'coreClientLoad', id: e.data.id, - ...(await core.loadFile(e.data, e.ports[0])), + ...(await core.loadFile(e.data, e.ports[0], addToken)), }); return; From 3fa9673aeb141121a86ec0c2eac536c5f46bf155 Mon Sep 17 00:00:00 2001 From: David DiMaria Date: Mon, 4 Nov 2024 16:03:47 -0700 Subject: [PATCH 111/113] Hide examples where VITE_STORAGE_TYPE === 'file-system' --- .../dashboard/components/DashboardSidebar.tsx | 4 ++- .../dashboard/components/NewFileDialog.tsx | 30 +++++++++++-------- .../dashboard/components/OnboardingBanner.tsx | 24 ++++++++------- 3 files changed, 34 insertions(+), 24 deletions(-) diff --git a/quadratic-client/src/dashboard/components/DashboardSidebar.tsx b/quadratic-client/src/dashboard/components/DashboardSidebar.tsx index 8ce46f8c35..b41141a99b 100644 --- a/quadratic-client/src/dashboard/components/DashboardSidebar.tsx +++ b/quadratic-client/src/dashboard/components/DashboardSidebar.tsx @@ -29,6 +29,8 @@ import { ReactNode, useState } from 'react'; import { NavLink, useLocation, useNavigation, useSearchParams, useSubmit } from 'react-router-dom'; import { useSetRecoilState } from 'recoil'; +const SHOW_EXAMPLES = import.meta.env.VITE_STORAGE_TYPE !== 'file-system'; + /** * Dashboard Navbar */ @@ -112,7 +114,7 @@ export function DashboardSidebar({ isLoading }: { isLoading: boolean }) { Resources
- {canEditTeam && ( + {canEditTeam && SHOW_EXAMPLES && ( Examples diff --git a/quadratic-client/src/dashboard/components/NewFileDialog.tsx b/quadratic-client/src/dashboard/components/NewFileDialog.tsx index ae94a521d3..d561cc6034 100644 --- a/quadratic-client/src/dashboard/components/NewFileDialog.tsx +++ b/quadratic-client/src/dashboard/components/NewFileDialog.tsx @@ -17,6 +17,8 @@ import { useCallback, useMemo, useState } from 'react'; import { Link, useLocation, useNavigation } from 'react-router-dom'; import { useSetRecoilState } from 'recoil'; +const SHOW_EXAMPLES = import.meta.env.VITE_STORAGE_TYPE !== 'file-system'; + type Props = { connections: ConnectionList; onClose: () => void; @@ -141,19 +143,21 @@ export function NewFileDialog({ connections, teamUuid, onClose, isPrivate: initi Fetch data from an API -
  • - - - - - Learn from an example file - -
    • + {SHOW_EXAMPLES && ( +
  • + + + + + Learn from an example file + +
    • + )}
  • diff --git a/quadratic-client/src/dashboard/components/OnboardingBanner.tsx b/quadratic-client/src/dashboard/components/OnboardingBanner.tsx index 834535e1f2..9708000de3 100644 --- a/quadratic-client/src/dashboard/components/OnboardingBanner.tsx +++ b/quadratic-client/src/dashboard/components/OnboardingBanner.tsx @@ -36,6 +36,8 @@ import { FormEvent, useEffect, useRef, useState } from 'react'; import { Link, useSubmit } from 'react-router-dom'; import { z } from 'zod'; +const SHOW_EXAMPLES = import.meta.env.VITE_STORAGE_TYPE !== 'file-system'; + export function OnboardingBanner() { const { activeTeam: { @@ -89,16 +91,18 @@ export function OnboardingBanner() { Create blank file - + {SHOW_EXAMPLES && ( + + )}

    Or bring your own data:

    From 9062aef9ea46b10334b40538c592da3fc8909295 Mon Sep 17 00:00:00 2001 From: David DiMaria Date: Mon, 4 Nov 2024 16:30:52 -0700 Subject: [PATCH 112/113] Only show revoked message when the license is revoked, not exceeded --- quadratic-api/src/licenseClient.ts | 2 +- quadratic-api/src/routes/v0/files.$uuid.GET.ts | 2 +- .../src/app/web-workers/quadraticCore/worker/coreClient.ts | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/quadratic-api/src/licenseClient.ts b/quadratic-api/src/licenseClient.ts index 245dad7294..5a82ee3bf0 100644 --- a/quadratic-api/src/licenseClient.ts +++ b/quadratic-api/src/licenseClient.ts @@ -65,7 +65,7 @@ export const licenseClient = { const result = await licenseClient.checkFromServer(); // don't cache errors or non-active licenses - if (!result || result.status !== 'active') { + if (!result || result.status === 'revoked') { return null; } diff --git a/quadratic-api/src/routes/v0/files.$uuid.GET.ts b/quadratic-api/src/routes/v0/files.$uuid.GET.ts index 41062392a0..87c5a9a6eb 100644 --- a/quadratic-api/src/routes/v0/files.$uuid.GET.ts +++ b/quadratic-api/src/routes/v0/files.$uuid.GET.ts @@ -1,3 +1,4 @@ +import { ResponseError } from '@sendgrid/mail'; import { Response } from 'express'; import { ApiTypes } from 'quadratic-shared/typesAndSchemas'; import z from 'zod'; @@ -9,7 +10,6 @@ import { validateOptionalAccessToken } from '../../middleware/validateOptionalAc import { validateRequestSchema } from '../../middleware/validateRequestSchema'; import { getFileUrl } from '../../storage/storage'; import { RequestWithOptionalUser } from '../../types/Request'; -import { ResponseError } from '@sendgrid/mail'; import { ApiError } from '../../utils/ApiError'; export default [ diff --git a/quadratic-client/src/app/web-workers/quadraticCore/worker/coreClient.ts b/quadratic-client/src/app/web-workers/quadraticCore/worker/coreClient.ts index 83a29860fe..1d1fed38aa 100644 --- a/quadratic-client/src/app/web-workers/quadraticCore/worker/coreClient.ts +++ b/quadratic-client/src/app/web-workers/quadraticCore/worker/coreClient.ts @@ -138,7 +138,7 @@ class CoreClient { case 'clientCoreLoad': await offline.init(e.data.fileId); - const addToken = this.env.STORAGE_TYPE === 'file-system'; + const addToken = this.env.VITE_STORAGE_TYPE === 'file-system'; this.send({ type: 'coreClientLoad', From 7653c0781c3658602a6da67259f5d3dd061d06f3 Mon Sep 17 00:00:00 2001 From: David DiMaria Date: Tue, 5 Nov 2024 12:40:35 -0700 Subject: [PATCH 113/113] Move JWT initialization --- .../src/app/web-workers/quadraticCore/worker/core.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/quadratic-client/src/app/web-workers/quadraticCore/worker/core.ts b/quadratic-client/src/app/web-workers/quadraticCore/worker/core.ts index 5b3862b5b1..29b791e136 100644 --- a/quadratic-client/src/app/web-workers/quadraticCore/worker/core.ts +++ b/quadratic-client/src/app/web-workers/quadraticCore/worker/core.ts @@ -60,10 +60,10 @@ class Core { private renderQueue: Function[] = []; private async loadGridFile(file: string, addToken: boolean): Promise { - const jwt = await coreClient.getJwt(); let requestInit = {}; if (addToken) { + const jwt = await coreClient.getJwt(); requestInit = { headers: { Authorization: `Bearer ${jwt}` } }; }