From 445bb5dae46e5a058e40da4950a69ae6a18304af Mon Sep 17 00:00:00 2001 From: Natoandro Date: Sat, 11 Jan 2025 12:21:10 +0300 Subject: [PATCH] perf: use predefined function for context check policies (#959) Improve performance by running the functions in the main thread when it can be done securely: - Use predefined function for context check policies - Use context check policy for `admin_only` #### Migration notes --- - [ ] The change comes with new or modified tests - [ ] Hard-to-understand functions have explanatory comments - [ ] End-user documentation is updated to reflect the change ## Summary by CodeRabbit ## Release Notes - **New Features** - Enhanced predefined function handling with optional parameters. - Introduced a more flexible context checking mechanism. - Simplified policy definition for admin access. - **Improvements** - Updated runtime function registration process. - Improved type safety for predefined function validation. - Streamlined error handling for function materialization. - **Changes** - Removed deprecated error handling functions. - Modified internal representations of predefined functions. - Updated function signatures for predefined Deno functions. --- src/common/src/typegraph/runtimes/deno.rs | 42 ++++++++++++ src/typegate/src/runtimes/deno/deno.ts | 45 ++++++++++--- src/typegate/src/typegraphs/typegate.json | 14 ++-- src/typegate/src/typegraphs/typegate.py | 9 +-- src/typegraph/core/src/conversion/runtimes.rs | 5 +- src/typegraph/core/src/errors.rs | 4 -- src/typegraph/core/src/global_store.rs | 32 ++++----- src/typegraph/core/src/lib.rs | 66 ++++++++----------- src/typegraph/core/src/runtimes/deno.rs | 4 +- src/typegraph/core/src/runtimes/mod.rs | 2 +- src/typegraph/core/src/validation/errors.rs | 4 -- .../core/src/validation/materializers.rs | 37 +++++++---- src/typegraph/core/wit/typegraph.wit | 1 + src/typegraph/deno/src/runtimes/deno.ts | 4 +- .../python/typegraph/runtimes/deno.py | 2 +- 15 files changed, 164 insertions(+), 107 deletions(-) diff --git a/src/common/src/typegraph/runtimes/deno.rs b/src/common/src/typegraph/runtimes/deno.rs index c370c06d7f..073c7ee045 100644 --- a/src/common/src/typegraph/runtimes/deno.rs +++ b/src/common/src/typegraph/runtimes/deno.rs @@ -1,6 +1,7 @@ // Copyright Metatype OÜ, licensed under the Mozilla Public License Version 2.0. // SPDX-License-Identifier: MPL-2.0 +use anyhow::{anyhow, Context, Result}; use indexmap::IndexMap; use serde::{Deserialize, Serialize}; use serde_json::Value; @@ -11,6 +12,47 @@ pub struct FunctionMatData { pub script: String, } +#[derive(Serialize, Deserialize, Clone, Debug, Hash, PartialEq, Eq)] +#[serde(tag = "type", content = "value")] +#[serde(rename_all = "snake_case")] +pub enum ContextCheckX { + NotNull, + Value(String), + Pattern(String), +} + +#[derive(PartialEq, Eq, Hash, Serialize, Deserialize, Clone, Debug)] +#[serde(rename_all = "snake_case")] +#[serde(tag = "name", content = "param")] +pub enum PredefinedFunctionMatData { + Identity, + True, + False, + Allow, + Deny, + Pass, + InternalPolicy, + ContextCheck { key: String, value: ContextCheckX }, +} + +#[derive(Serialize)] +struct PredefinedFunctionMatDataRaw { + name: String, + param: Option, +} + +impl PredefinedFunctionMatData { + pub fn from_raw(name: String, param: Option) -> Result { + let param = param + .map(|p| serde_json::from_str(&p)) + .transpose() + .context("invalid predefined function materializer parameter")?; + let value = serde_json::to_value(&PredefinedFunctionMatDataRaw { name, param })?; + serde_json::from_value(value) + .map_err(|e| anyhow!("invalid predefined function materializer: {e:?}")) + } +} + #[derive(Serialize, Deserialize, Clone, Debug)] #[serde(rename_all = "camelCase")] pub struct ModuleMatData { diff --git a/src/typegate/src/runtimes/deno/deno.ts b/src/typegate/src/runtimes/deno/deno.ts index 6fde122009..5a698ebd8b 100644 --- a/src/typegate/src/runtimes/deno/deno.ts +++ b/src/typegate/src/runtimes/deno/deno.ts @@ -30,15 +30,42 @@ import { WorkerManager } from "./worker_manager.ts"; const logger = getLogger(import.meta); -const predefinedFuncs: Record>> = { - identity: ({ _, ...args }) => args, - true: () => true, - false: () => false, - allow: () => "ALLOW" as PolicyResolverOutput, - deny: () => "DENY" as PolicyResolverOutput, - pass: () => "PASS" as PolicyResolverOutput, - internal_policy: ({ _: { context } }) => +const predefinedFuncs: Record< + string, + (param: any) => Resolver> +> = { + identity: () => ({ _, ...args }) => args, + true: () => () => true, + false: () => () => false, + allow: () => () => "ALLOW" as PolicyResolverOutput, + deny: () => () => "DENY" as PolicyResolverOutput, + pass: () => () => "PASS" as PolicyResolverOutput, + internal_policy: () => ({ _: { context } }) => context.provider === "internal" ? "ALLOW" : "PASS" as PolicyResolverOutput, + context_check: ({ key, value }) => { + let check: (value: any) => boolean; + switch (value.type) { + case "not_null": + check = (v) => v != null; + break; + case "value": + check = (v) => v === value.value; + break; + case "pattern": + check = (v) => v != null && new RegExp(value.value).test(v); + break; + default: + throw new Error("unreachable"); + } + const path = key.split("."); + return ({ _: { context } }) => { + let value: any = context; + for (const segment of path) { + value = value?.[segment]; + } + return check(value) ? "PASS" : "DENY" as PolicyResolverOutput; + }; + }, }; export class DenoRuntime extends Runtime { @@ -229,7 +256,7 @@ export class DenoRuntime extends Runtime { if (!func) { throw new Error(`predefined function ${mat.data.name} not found`); } - return func; + return func(mat.data.param); } if (mat.name === "static") { diff --git a/src/typegate/src/typegraphs/typegate.json b/src/typegate/src/typegraphs/typegate.json index 828df86a18..d56a6fced0 100644 --- a/src/typegate/src/typegraphs/typegate.json +++ b/src/typegate/src/typegraphs/typegate.json @@ -807,15 +807,21 @@ "data": {} }, { - "name": "function", + "name": "predefined_function", "runtime": 0, "effect": { "effect": "read", "idempotent": true }, "data": { - "script": "var _my_lambda = (_args, { context }) => context.username === 'admin' ? 'ALLOW' : 'DENY' ", - "secrets": [] + "name": "context_check", + "param": { + "key": "username", + "value": { + "type": "value", + "value": "admin" + } + } } }, { @@ -942,7 +948,7 @@ ], "policies": [ { - "name": "admin_only", + "name": "__ctx_username_admin", "materializer": 1 } ], diff --git a/src/typegate/src/typegraphs/typegate.py b/src/typegate/src/typegraphs/typegate.py index 5008c472e9..a47547314e 100644 --- a/src/typegate/src/typegraphs/typegate.py +++ b/src/typegate/src/typegraphs/typegate.py @@ -1,12 +1,11 @@ # Copyright Metatype OÜ, licensed under the Mozilla Public License Version 2.0. # SPDX-License-Identifier: MPL-2.0 -from typegraph import Graph, fx, t, typegraph +from typegraph import Graph, fx, t, typegraph, Policy from typegraph.gen.exports.runtimes import TypegateOperation from typegraph.gen.types import Err from typegraph.graph.params import Auth, Cors, Rate from typegraph.runtimes.base import Materializer -from typegraph.runtimes.deno import DenoRuntime from typegraph.wit import runtimes, store ### Prisma query (Json protocol): @@ -95,11 +94,7 @@ ), ) def typegate(g: Graph): - deno = DenoRuntime() - admin_only = deno.policy( - "admin_only", - code="(_args, { context }) => context.username === 'admin' ? 'ALLOW' : 'DENY' ", - ) + admin_only = Policy.context("username", "admin") g.auth(Auth.basic(["admin"])) diff --git a/src/typegraph/core/src/conversion/runtimes.rs b/src/typegraph/core/src/conversion/runtimes.rs index 76c7dc89ef..537bd4e7d2 100644 --- a/src/typegraph/core/src/conversion/runtimes.rs +++ b/src/typegraph/core/src/conversion/runtimes.rs @@ -128,10 +128,7 @@ impl MaterializerConverter for DenoMaterializer { ("import_function".to_string(), data) } Predefined(predef) => { - let data = serde_json::from_value(json!({ - "name": predef.name, - })) - .unwrap(); + let data = serde_json::from_value(serde_json::to_value(predef).unwrap()).unwrap(); ("predefined_function".to_string(), data) } }; diff --git a/src/typegraph/core/src/errors.rs b/src/typegraph/core/src/errors.rs index ef33a1421c..0f8804b1f0 100644 --- a/src/typegraph/core/src/errors.rs +++ b/src/typegraph/core/src/errors.rs @@ -143,10 +143,6 @@ pub fn object_not_found(kind: &str, id: u32) -> TgError { // .into() // } -pub fn unknown_predefined_function(name: &str, runtime: &str) -> TgError { - format!("unknown predefined function {name} for runtime {runtime}").into() -} - pub fn duplicate_policy_name(name: &str) -> TgError { format!("duplicate policy name '{name}'").into() } diff --git a/src/typegraph/core/src/global_store.rs b/src/typegraph/core/src/global_store.rs index b09b4a3a69..305ccdf3d9 100644 --- a/src/typegraph/core/src/global_store.rs +++ b/src/typegraph/core/src/global_store.rs @@ -14,6 +14,7 @@ use crate::wit::utils::Auth as WitAuth; #[allow(unused)] use crate::wit::runtimes::{Effect, MaterializerDenoPredefined, MaterializerId}; +use common::typegraph::runtimes::deno::PredefinedFunctionMatData; use graphql_parser::parse_query; use indexmap::IndexMap; use std::rc::Rc; @@ -55,7 +56,7 @@ pub struct Store { pub policies: Vec, deno_runtime: RuntimeId, - predefined_deno_functions: HashMap, + predefined_deno_functions: HashMap, deno_modules: IndexMap, public_policy_id: PolicyId, @@ -88,9 +89,7 @@ impl Store { runtime_id: deno_runtime, effect: Effect::Read, data: MaterializerData::Deno(Rc::new(DenoMaterializer::Predefined( - crate::wit::runtimes::MaterializerDenoPredefined { - name: "pass".to_string(), - }, + PredefinedFunctionMatData::Pass, ))), }], @@ -104,8 +103,6 @@ impl Store { } } -const PREDEFINED_DENO_FUNCTIONS: &[&str] = &["identity", "true"]; - thread_local! { pub static STORE: RefCell = RefCell::new(Store::new()); pub static SDK_VERSION: String = "0.5.0-rc.9".to_owned(); @@ -350,25 +347,24 @@ impl Store { with_store(|s| s.public_policy_id) } - pub fn get_predefined_deno_function(name: String) -> Result { - if let Some(mat) = with_store(|s| s.predefined_deno_functions.get(&name).cloned()) { - Ok(mat) - } else if !PREDEFINED_DENO_FUNCTIONS.iter().any(|n| n == &name) { - Err(errors::unknown_predefined_function(&name, "deno")) + pub fn get_predefined_deno_function( + name: String, + param: Option, + ) -> Result { + let mat = PredefinedFunctionMatData::from_raw(name, param)?; + if let Some(mat_id) = with_store(|s| s.predefined_deno_functions.get(&mat).cloned()) { + Ok(mat_id) } else { let runtime_id = Store::get_deno_runtime(); - let mat = Store::register_materializer(Materializer { + let mat_id = Store::register_materializer(Materializer { runtime_id, effect: Effect::Read, - data: Rc::new(DenoMaterializer::Predefined(MaterializerDenoPredefined { - name: name.clone(), - })) - .into(), + data: Rc::new(DenoMaterializer::Predefined(mat.clone())).into(), }); with_store_mut(|s| { - s.predefined_deno_functions.insert(name, mat); + s.predefined_deno_functions.insert(mat, mat_id); }); - Ok(mat) + Ok(mat_id) } } diff --git a/src/typegraph/core/src/lib.rs b/src/typegraph/core/src/lib.rs index 8c33565b5c..a3816d3132 100644 --- a/src/typegraph/core/src/lib.rs +++ b/src/typegraph/core/src/lib.rs @@ -19,10 +19,10 @@ mod test_utils; use std::collections::HashSet; +use common::typegraph::runtimes::deno::{ContextCheckX, PredefinedFunctionMatData}; use common::typegraph::Injection; use errors::{Result, TgError}; use global_store::Store; -use indoc::formatdoc; use params::apply; use regex::Regex; use runtimes::{DenoMaterializer, Materializer}; @@ -38,7 +38,7 @@ use wit::core::{ TypeEither, TypeFile, TypeFloat, TypeFunc, TypeId as CoreTypeId, TypeInteger, TypeList, TypeOptional, TypeString, TypeStruct, TypeUnion, TypegraphInitParams, }; -use wit::runtimes::{Guest, MaterializerDenoFunc}; +use wit::runtimes::{Guest as _, MaterializerDenoPredefined}; pub mod wit { wit_bindgen::generate!({ @@ -52,6 +52,17 @@ pub mod wit { pub struct Lib {} +impl From for ContextCheckX { + fn from(check: ContextCheck) -> Self { + use ContextCheck as CC; + match check { + CC::NotNull => ContextCheckX::NotNull, + CC::Value(v) => ContextCheckX::Value(v), + CC::Pattern(p) => ContextCheckX::Pattern(p), + } + } +} + impl wit::core::Guest for Lib { fn init_typegraph(params: TypegraphInitParams) -> Result<()> { typegraph::init(params) @@ -185,9 +196,7 @@ impl wit::core::Guest for Lib { } fn get_internal_policy() -> Result<(PolicyId, String)> { - let deno_mat = DenoMaterializer::Predefined(wit::runtimes::MaterializerDenoPredefined { - name: "internal_policy".to_string(), - }); + let deno_mat = DenoMaterializer::Predefined(PredefinedFunctionMatData::InternalPolicy); let mat = Materializer::deno(deno_mat, crate::wit::runtimes::Effect::Read); let policy_id = Store::register_policy( Policy { @@ -213,42 +222,19 @@ impl wit::core::Guest for Lib { .replace_all(&name, "_") .to_string(); - let check = match check { - ContextCheck::NotNull => "value != null ? 'PASS' : 'DENY'".to_string(), - ContextCheck::Value(val) => { - format!( - "value === {} ? 'PASS' : 'DENY'", - serde_json::to_string(&val).unwrap() - ) - } - ContextCheck::Pattern(pattern) => { - format!( - "new RegExp({}).test(value) ? 'PASS' : 'DENY' ", - serde_json::to_string(&pattern).unwrap() - ) - } - }; + let check: ContextCheckX = check.into(); + let check = serde_json::json!({ + "key": key, + "value": check + }); - let key = serde_json::to_string(&key).unwrap(); - - let code = formatdoc! {r#" - (_, {{ context }}) => {{ - const chunks = {key}.split("."); - let value = context; - for (const chunk of chunks) {{ - value = value?.[chunk]; - }} - return {check}; - }} - "# }; - - let mat_id = Lib::register_deno_func( - MaterializerDenoFunc { - code, - secrets: vec![], - }, - wit::runtimes::Effect::Read, - )?; + let mat_id = Lib::get_predefined_deno_func(MaterializerDenoPredefined { + name: "context_check".to_string(), + param: Some( + serde_json::to_string(&check) + .map_err(|e| format!("Error while serializing context check: {e:?}"))?, + ), + })?; Lib::register_policy(Policy { name: name.clone(), diff --git a/src/typegraph/core/src/runtimes/deno.rs b/src/typegraph/core/src/runtimes/deno.rs index fe34beae5c..eb89f6a0fa 100644 --- a/src/typegraph/core/src/runtimes/deno.rs +++ b/src/typegraph/core/src/runtimes/deno.rs @@ -1,6 +1,8 @@ // Copyright Metatype OÜ, licensed under the Mozilla Public License Version 2.0. // SPDX-License-Identifier: MPL-2.0 +use common::typegraph::runtimes::deno::PredefinedFunctionMatData; + use crate::wit::runtimes as wit; #[derive(Debug)] @@ -25,7 +27,7 @@ pub struct MaterializerDenoStatic { pub enum DenoMaterializer { Static(MaterializerDenoStatic), Inline(wit::MaterializerDenoFunc), - Predefined(wit::MaterializerDenoPredefined), + Predefined(PredefinedFunctionMatData), Module(MaterializerDenoModule), Import(MaterializerDenoImport), } diff --git a/src/typegraph/core/src/runtimes/mod.rs b/src/typegraph/core/src/runtimes/mod.rs index 704b86296a..73606efb2b 100644 --- a/src/typegraph/core/src/runtimes/mod.rs +++ b/src/typegraph/core/src/runtimes/mod.rs @@ -295,7 +295,7 @@ impl crate::wit::runtimes::Guest for crate::Lib { fn get_predefined_deno_func( data: wit::MaterializerDenoPredefined, ) -> Result { - Store::get_predefined_deno_function(data.name) + Store::get_predefined_deno_function(data.name, data.param) } fn import_deno_function( diff --git a/src/typegraph/core/src/validation/errors.rs b/src/typegraph/core/src/validation/errors.rs index 41c4467f5e..63e11f7a25 100644 --- a/src/typegraph/core/src/validation/errors.rs +++ b/src/typegraph/core/src/validation/errors.rs @@ -17,7 +17,3 @@ pub fn invalid_output_type_predefined(name: &str, expected: &str, got: &str) -> ) .into() } - -pub fn unknown_predefined_function(name: &str) -> Error { - format!("unknown predefined function {}", name).into() -} diff --git a/src/typegraph/core/src/validation/materializers.rs b/src/typegraph/core/src/validation/materializers.rs index c05459aebe..1901efcf02 100644 --- a/src/typegraph/core/src/validation/materializers.rs +++ b/src/typegraph/core/src/validation/materializers.rs @@ -1,6 +1,8 @@ // Copyright Metatype OÜ, licensed under the Mozilla Public License Version 2.0. // SPDX-License-Identifier: MPL-2.0 +use common::typegraph::runtimes::deno::PredefinedFunctionMatData; + use crate::runtimes::{DenoMaterializer, MaterializerData, Runtime}; use crate::types::{AsTypeDefEx as _, TypeDef, TypeId}; use crate::wit::core::TypeFunc; @@ -26,43 +28,54 @@ impl Materializer { fn validate_deno_mat(mat_data: &DenoMaterializer, func: &TypeFunc) -> Result<()> { match mat_data { DenoMaterializer::Predefined(predef) => { - match predef.name.as_str() { - "identity" => { + use PredefinedFunctionMatData as P; + match predef { + P::Identity => { if !type_utils::is_equal(func.inp.into(), func.out.into())? { return Err(errors::invalid_output_type_predefined( - &predef.name, + "identity", &TypeId(func.inp).repr()?, &TypeId(func.out).repr()?, )); } } - - "true" | "false" => { + P::True | P::False => { if let Ok(xdef) = TypeId(func.out).as_xdef() { let TypeDef::Boolean(_) = xdef.type_def else { return Err(errors::invalid_output_type_predefined( - &predef.name, + match predef { + P::True => "true", + P::False => "false", + _ => unreachable!(), + }, "bool", &TypeId(func.out).repr()?, )); }; } } - - "allow" | "deny" | "pass" => { + P::Allow + | P::Deny + | P::Pass + | P::ContextCheck { .. } + | P::InternalPolicy { .. } => { if let Ok(xdef) = TypeId(func.out).as_xdef() { let TypeDef::String(_) = xdef.type_def else { return Err(errors::invalid_output_type_predefined( - &predef.name, + match predef { + P::Allow => "allow", + P::Deny => "deny", + P::Pass => "pass", + P::ContextCheck { .. } => "context_check", + P::InternalPolicy { .. } => "internal_policy", + _ => unreachable!(), + }, "string", &TypeId(func.out).repr()?, )); }; } } - _ => { - return Err(errors::unknown_predefined_function(&predef.name)); - } } Ok(()) } diff --git a/src/typegraph/core/wit/typegraph.wit b/src/typegraph/core/wit/typegraph.wit index cda10f336d..c99c057dcb 100644 --- a/src/typegraph/core/wit/typegraph.wit +++ b/src/typegraph/core/wit/typegraph.wit @@ -268,6 +268,7 @@ interface runtimes { record materializer-deno-predefined { name: string, + param: option, } record materializer-deno-import { diff --git a/src/typegraph/deno/src/runtimes/deno.ts b/src/typegraph/deno/src/runtimes/deno.ts index 9eb8210931..36b2dce64e 100644 --- a/src/typegraph/deno/src/runtimes/deno.ts +++ b/src/typegraph/deno/src/runtimes/deno.ts @@ -134,9 +134,9 @@ export class DenoRuntime extends Runtime { fetchContext(outputShape?: C): t.Func { const returnValue = outputShape ? `context` : "JSON.stringify(context)"; return this.func( - t.struct({}), + t.struct({}), outputShape ?? t.json(), - { code: `(_, { context }) => ${returnValue}` } + { code: `(_, { context }) => ${returnValue}` }, ); } diff --git a/src/typegraph/python/typegraph/runtimes/deno.py b/src/typegraph/python/typegraph/runtimes/deno.py index 08dc005126..b223c0e1ad 100644 --- a/src/typegraph/python/typegraph/runtimes/deno.py +++ b/src/typegraph/python/typegraph/runtimes/deno.py @@ -118,7 +118,7 @@ def identity(self, inp: "t.struct") -> "t.func": from typegraph import t res = runtimes.get_predefined_deno_func( - store, MaterializerDenoPredefined(name="identity") + store, MaterializerDenoPredefined(name="identity", param=None) ) if isinstance(res, Err): raise Exception(res.value)