diff --git a/src/backend/db/prisma/schema/custom-server/lambda.prisma b/src/backend/db/prisma/schema/custom-server/lambda.prisma index 86d47050..5f015358 100644 --- a/src/backend/db/prisma/schema/custom-server/lambda.prisma +++ b/src/backend/db/prisma/schema/custom-server/lambda.prisma @@ -1,6 +1,7 @@ enum LambdaServerInstanceRuntime { deno_deploy_v1 deno_self_hosted_v1 + python_self_hosted_v1 aws_lambda_nodejs_24_x aws_lambda_nodejs_22_x aws_lambda_python_3_9 @@ -13,6 +14,7 @@ enum LambdaServerInstanceRuntime { enum LambdaServerInstanceProvider { deno_deploy deno_self_hosted + python_self_hosted aws_lambda } diff --git a/src/backend/modules/custom-server/src/deployment/python-local/deployment/index.ts b/src/backend/modules/custom-server/src/deployment/python-local/deployment/index.ts new file mode 100644 index 00000000..ce927758 --- /dev/null +++ b/src/backend/modules/custom-server/src/deployment/python-local/deployment/index.ts @@ -0,0 +1,151 @@ +import { + CodeBucket, + CustomServer, + CustomServerDeployment, + db, + Instance, + LambdaServerInstance +} from '@metorial/db'; +import { delay } from '@metorial/delay'; +import { joinPaths } from '@metorial/join-paths'; +import axios from 'axios'; +import { env } from '../../../env'; +import { DeploymentError } from '../../base/error'; +import { getPythonFs } from '../fs'; + +axios.defaults.headers.common['Accept-Encoding'] = 'gzip'; + +// Determine deployment mode +export let isPythonLocalEnabled = () => env.python.PYTHON_RUNNER_ADDRESS; + +export let createPythonLambdaDeployment = async (config: { + lambdaServerInstance: LambdaServerInstance & { + immutableCodeBucket: CodeBucket; + instance: Instance; + }; + customServer: CustomServer; + deployment: CustomServerDeployment; +}) => { + if (!isPythonLocalEnabled()) { + throw new Error('Python Local deployment is not enabled in the environment variables.'); + } + + let lambdaServerInstance = config.lambdaServerInstance; + + let deployment = await Promise.race([ + delay(1000 * 60 * 2).then(() => { + throw new DeploymentError({ + code: 'deployment_timeout', + message: 'Python deployment timed out after 5 minutes' + }); + }), + (async () => { + let fs = await getPythonFs(lambdaServerInstance); + + let deploymentPayload = { + entryPointUrl: fs.entrypoint, + envVars: { + ...fs.env, + METORIAL_AUTH_TOKEN_SECRET: lambdaServerInstance.securityToken + }, + description: `CSRV ${config.customServer.id} / DEPL ${config.deployment.id}`, + permissions: { + net: ['*'] + }, + assets: Object.fromEntries( + Array.from(fs.files.entries()).map(([k, v]) => [ + k, + { + kind: 'file', + encoding: 'utf-8', + content: v + } + ]) + ) + }; + + let deploymentId: string; + let providerResourceAccessIdentifier: string; + + let runnerDeployment = await axios.post<{ id: string }>( + `${env.python.PYTHON_RUNNER_ADDRESS}/deployments`, + deploymentPayload + ); + + deploymentId = runnerDeployment.data.id; + providerResourceAccessIdentifier = `${env.python.PYTHON_RUNNER_ADDRESS}/${deploymentId}`; + + return await db.lambdaServerInstance.update({ + where: { oid: lambdaServerInstance.oid }, + data: { + status: 'deploying', + providerInfo: { id: deploymentId }, + providerResourceId: deploymentId, + providerResourceAccessIdentifier, + runtime: 'python_self_hosted_v1', + provider: 'python_self_hosted', + platform: 'metorial_stellar_v1', + protocol: 'metorial_stellar_over_websocket_v1' + } + }); + })() + ]); + + let serverUrl = { current: deployment.providerResourceAccessIdentifier || '' }; + + return { + pollDeploymentStatus: async () => { + return { + status: 'success' as const, + logs: [] as { type: 'info' | 'error'; lines: string[] }[] + }; + }, + + discoverServer: async () => { + let discoverUrl = new URL(serverUrl.current); + discoverUrl.pathname = joinPaths(discoverUrl.pathname, '/discover'); + let discoverRes = await axios.get(discoverUrl.toString(), { + headers: { + 'metorial-stellar-token': lambdaServerInstance.securityToken + }, + timeout: 5000 + }); + + let oauthUrl = new URL(serverUrl.current); + oauthUrl.pathname = joinPaths(oauthUrl.pathname, '/oauth'); + let oauthRes = await axios.get<{ enabled: boolean; hasForm: boolean }>( + oauthUrl.toString(), + { + headers: { + 'metorial-stellar-token': lambdaServerInstance.securityToken + }, + timeout: 5000 + } + ); + + let callbacksUrl = new URL(serverUrl.current); + callbacksUrl.pathname = joinPaths(callbacksUrl.pathname, '/callbacks'); + let callbacksRes = await axios.get<{ + enabled: boolean; + type: 'webhook' | 'polling' | 'manual'; + }>(callbacksUrl.toString(), { + headers: { + 'metorial-stellar-token': lambdaServerInstance.securityToken + }, + timeout: 5000 + }); + + return { + capabilities: discoverRes.data, + oauth: oauthRes.data, + callbacks: callbacksRes.data + }; + }, + + get httpEndpoint() { + return serverUrl.current; + } + }; +}; + +export type PythonDeployment = Awaited>; diff --git a/src/backend/modules/custom-server/src/deployment/python-local/fs/index.ts b/src/backend/modules/custom-server/src/deployment/python-local/fs/index.ts new file mode 100644 index 00000000..7c025bec --- /dev/null +++ b/src/backend/modules/custom-server/src/deployment/python-local/fs/index.ts @@ -0,0 +1,99 @@ +import { CodeBucket, Instance, LambdaServerInstance } from '@metorial/db'; +import { codeBucketService } from '@metorial/module-code-bucket'; +import { DeploymentError } from '../../base/error'; + +let commonEntryPoints = ['index', 'app', 'main', 'server', 'boot', 'mcp'].flatMap(name => [ + `${name}.ts`, + `${name}.js`, + `${name}.cjs`, + `${name}.mjs` +]); + +export let getPythonFs = async ( + lambda: LambdaServerInstance & { + instance: Instance; + immutableCodeBucket: CodeBucket; + } +) => { + let files = new Map( + Object.entries({ + // TODO: @RahmeKarim add boot loader files for python here + // 'boot.ts': bootTs, + // 'delay.ts': delayTs, + // 'discover.ts': discoverTs, + // 'error.ts': errorTs, + // 'logs.ts': logsTs, + // 'promise.ts': promiseTs, + // 'server.ts': serverTs, + // 'transport.ts': transportTs, + // 'lib/index.ts': libIndexTs, + // 'lib/args.ts': libArgsTs, + // 'lib/oauth.ts': libOauthTs, + // 'lib/callbacks.ts': libCallbacksTs, + // 'config.ts': configTs, + // 'oauth.ts': oauthTs, + // 'callbacks.ts': callbacksTs + }) + ); + + let bucketFiles = await codeBucketService.getCodeBucketFilesWithContent({ + codeBucket: lambda.immutableCodeBucket + }); + + for (let file of bucketFiles) { + let path = `app/${file.path}`; + if (files.has(path)) { + throw new DeploymentError({ + code: 'invalid_file', + message: `File ${file.path} is reserved and cannot be used in the code bucket` + }); + } + files.set(path, new TextDecoder().decode(file.content)); + } + + let entrypoint: string | undefined; + if (!entrypoint) { + let found = commonEntryPoints.find(name => bucketFiles.some(f => f.path === name)); + if (found) entrypoint = found; + } + if (!entrypoint) { + throw new DeploymentError({ + code: 'missing_entry_point', + message: `Could not determine entry point. Please specify a "main" field in your package.json file or add one of the following files to your code bucket: ${commonEntryPoints.join( + ', ' + )}` + }); + } + + let metorialDeploymentContent = JSON.stringify( + { + entrypoint, + lambda: { + id: lambda.id, + createdAt: lambda.createdAt + }, + immutableCodeBucket: { + id: lambda.immutableCodeBucket.id, + createdAt: lambda.immutableCodeBucket.createdAt + }, + instance: { + id: lambda.instance.id, + slug: lambda.instance.slug, + name: lambda.instance.name, + type: lambda.instance.type, + createdAt: lambda.instance.createdAt + } + }, + null, + 2 + ); + files.set('mtdpl.json', metorialDeploymentContent); + + return { + entrypoint: 'boot.py', // TODO: @RahmeKarim change to what the entrypoint should be + env: { + CUSTOM_SERVER_ENTRYPOINT: entrypoint + }, + files + }; +}; diff --git a/src/backend/modules/custom-server/src/deployment/python-local/impl/callbacks.ts b/src/backend/modules/custom-server/src/deployment/python-local/impl/callbacks.ts new file mode 100644 index 00000000..f2110982 --- /dev/null +++ b/src/backend/modules/custom-server/src/deployment/python-local/impl/callbacks.ts @@ -0,0 +1,190 @@ +import { Callback, CallbackEvent } from '@metorial/db'; +import { getAxiosSsrfFilter } from '@metorial/ssrf'; +import axios from 'axios'; +import { CallbackHandler } from '../../base/callbackHandler'; + +export class PythonCallbackHandler extends CallbackHandler { + async handleCallback(d: { + events: CallbackEvent[]; + callback: Callback; + }): Promise< + ({ event: CallbackEvent } & ( + | { success: false; error: { code: string; message: string } } + | { success: true; result: any; type: string } + ))[] + > { + let url = this.lambda.providerResourceAccessIdentifier; + if (!url) throw new Error('WTF - no url for lambda server instance'); + + try { + let res = await axios.post<{ + results: { success: boolean; eventId: string; error?: string; result?: any }[]; + }>( + `${url}/callbacks/handle`, + { + callbackId: d.callback.id, + events: d.events.map(e => ({ + eventId: e.id, + payload: JSON.parse(e.payloadIncoming) + })) + }, + { + ...getAxiosSsrfFilter(url), + headers: { + 'metorial-stellar-token': this.lambda.securityToken + } + } + ); + + let eventMap = new Map(res.data.results.map(r => [r.eventId, r])); + + return d.events.map(e => { + let r = eventMap.get(e.id); + if (!r) { + return { + event: e, + success: false as const, + error: { + code: 'execution_error', + message: 'Unable to run callback on server (no result returned)' + } + }; + } + + if (r.success) { + let fullResult = r.result; + if (fullResult === null) { + return { + event: e, + result: null, + type: 'noop', + success: true as const + }; + } + + if ( + typeof fullResult != 'object' || + fullResult === null || + Array.isArray(fullResult) || + typeof fullResult.type !== 'string' || + !('result' in fullResult) + ) { + return { + event: e, + success: false as const, + error: { + code: 'invalid_result', + message: 'Invalid result format returned from server' + } + }; + } + + return { + event: e, + result: fullResult.result, + type: fullResult.type, + success: true as const + }; + } else { + return { + event: e, + success: false as const, + error: { + code: 'server_error', + message: r.error ?? 'Unknown server error' + } + }; + } + }); + } catch (e) { + return d.events.map(e => ({ + event: e, + success: false as const, + error: { + code: 'execution_error', + message: 'Unable to run callback on server' + } + })); + } + } + + async installCallback(d: { + callback: Callback; + url: string; + }): Promise< + { success: true } | { success: false; error: { code: string; message: string } } + > { + let url = this.lambda.providerResourceAccessIdentifier; + if (!url) throw new Error('WTF - no url for lambda server instance'); + + try { + await axios.post( + `${url}/callbacks/install`, + { + callbackId: d.callback.id, + callbackUrl: d.url + }, + { + ...getAxiosSsrfFilter(url), + headers: { + 'metorial-stellar-token': this.lambda.securityToken + } + } + ); + + return { success: true as const }; + } catch (e) { + return { + success: false as const, + error: { + code: 'installation_error', + message: 'Unable to install callback on server' + } + }; + } + } + + async pollCallback(d: { + callback: Callback; + state: any; + }): Promise< + | { success: true; events: CallbackEvent[]; newState: any } + | { success: false; error: { code: string; message: string } } + > { + let url = this.lambda.providerResourceAccessIdentifier; + if (!url) throw new Error('WTF - no url for lambda server instance'); + + try { + let res = await axios.post<{ + events: any[]; + newState: any; + }>( + `${url}/callbacks/poll`, + { + callbackId: d.callback.id, + state: d.state + }, + { + ...getAxiosSsrfFilter(url), + headers: { + 'metorial-stellar-token': this.lambda.securityToken + } + } + ); + + return { + success: true as const, + events: res.data.events, + newState: res.data.newState + }; + } catch (e: any) { + return { + success: false as const, + error: { + code: 'poll_error', + message: e.response?.data?.message || 'Unable to poll callback' + } + }; + } + } +} diff --git a/src/backend/modules/custom-server/src/deployment/python-local/impl/oauth.ts b/src/backend/modules/custom-server/src/deployment/python-local/impl/oauth.ts new file mode 100644 index 00000000..f84fd96a --- /dev/null +++ b/src/backend/modules/custom-server/src/deployment/python-local/impl/oauth.ts @@ -0,0 +1,158 @@ +import { createCachedFunction } from '@metorial/cache'; +import { + ProviderOAuthConfig, + ProviderOAuthConnection, + ProviderOAuthConnectionAuthAttempt +} from '@metorial/db'; +import { badRequestError, ServiceError } from '@metorial/error'; +import { getAxiosSsrfFilter } from '@metorial/ssrf'; +import axios from 'axios'; +import { OAuthHandler, OAuthResponse } from '../../base/oauthHandler'; + +let getFormCached = createCachedFunction({ + name: 'crv/lpyt/form1', + provider: async (i: { securityToken: string; httpEndpoint: string }) => { + let form = await axios.post<{ + authForm: { fields: any[] }; + }>( + `${i.httpEndpoint}/oauth/authorization-form`, + { input: {} }, + { + ...getAxiosSsrfFilter(i.httpEndpoint), + headers: { + 'metorial-stellar-token': i.securityToken + } + } + ); + return form.data.authForm; + }, + ttlSeconds: 60 * 5, + getHash: i => i.httpEndpoint +}); + +export class PythonOAuthHandler extends OAuthHandler { + async getOAuthForm(d: {}): Promise<{ fields: any[] }> { + let endpoint = this.lambda.providerResourceAccessIdentifier; + if (!endpoint) throw new Error('WTF - no endpoint for lambda server instance'); + + return getFormCached({ + securityToken: this.lambda.securityToken, + httpEndpoint: endpoint + }); + } + + async getAuthorizationUrl(d: { + connection: ProviderOAuthConnection & { config: ProviderOAuthConfig }; + authAttempt: ProviderOAuthConnectionAuthAttempt; + fields: Record; + redirectUri: string; + }): Promise<{ authorizationUrl: string; codeVerifier: string }> { + let endpoint = this.lambda.providerResourceAccessIdentifier; + if (!endpoint) throw new Error('WTF - no endpoint for lambda server instance'); + + let authUrlRes = await axios.post<{ + authorizationUrl: string; + codeVerifier: string; + success: boolean; + }>( + `${endpoint}/oauth/authorization-url`, + { + input: { + fields: d.fields ?? {}, + clientId: d.connection.clientId, + clientSecret: d.connection.clientSecret, + state: d.authAttempt.stateIdentifier, + redirectUri: d.redirectUri + } + }, + { + ...getAxiosSsrfFilter(endpoint), + headers: { + 'metorial-stellar-token': this.lambda.securityToken + } + } + ); + if (authUrlRes.status !== 200 || !authUrlRes.data.success) { + throw new ServiceError( + badRequestError({ + message: 'Failed to fetch authorization URL from remote server' + }) + ); + } + + return { + authorizationUrl: authUrlRes.data.authorizationUrl, + codeVerifier: authUrlRes.data.codeVerifier + }; + } + + async handleOAuthCallback(d: { + fullUrl: string; + redirectUri: string; + response: OAuthResponse; + connection: ProviderOAuthConnection & { config: ProviderOAuthConfig }; + authAttempt: ProviderOAuthConnectionAuthAttempt; + }): Promise> { + let endpoint = this.lambda.providerResourceAccessIdentifier; + if (!endpoint) throw new Error('WTF - no endpoint for lambda server instance'); + + let tokenRes = await axios.post<{ + authData: Record; + }>( + `${endpoint}/oauth/callback`, + { + input: { + fields: d.authAttempt.additionalValues || {}, + code: d.response.code!, + state: d.response.state!, + clientId: d.connection.clientId!, + clientSecret: d.connection.clientSecret, + redirectUri: d.redirectUri, + fullUrl: d.fullUrl, + codeVerifier: d.authAttempt.codeVerifier + } + }, + { + ...getAxiosSsrfFilter(endpoint!), + headers: { + 'metorial-stellar-token': this.lambda.securityToken + } + } + ); + + return tokenRes.data.authData; + } + + async refreshOAuthToken(d: { + connection: ProviderOAuthConnection & { config: ProviderOAuthConfig }; + refreshToken: string; + redirectUri: string; + additionalAuthData: Record; + }): Promise> { + let endpoint = this.lambda.providerResourceAccessIdentifier; + if (!endpoint) throw new Error('WTF - no endpoint for lambda server instance'); + + let tokenRes = await axios.post<{ + authData: Record; + }>( + `${endpoint}/oauth/refresh`, + { + input: { + redirectUri: d.redirectUri, + refreshToken: d.refreshToken, + clientId: d.connection.clientId!, + clientSecret: d.connection.clientSecret, + fields: d.additionalAuthData + } + }, + { + ...getAxiosSsrfFilter(endpoint!), + headers: { + 'metorial-stellar-token': this.lambda.securityToken + } + } + ); + + return tokenRes.data.authData; + } +} diff --git a/src/backend/modules/custom-server/src/deployment/python-local/queues/main.ts b/src/backend/modules/custom-server/src/deployment/python-local/queues/main.ts new file mode 100644 index 00000000..f263d8a3 --- /dev/null +++ b/src/backend/modules/custom-server/src/deployment/python-local/queues/main.ts @@ -0,0 +1,308 @@ +import { db, ID, ServerVersion, withTransaction } from '@metorial/db'; +import { delay } from '@metorial/delay'; +import { providerOauthConfigService } from '@metorial/module-provider-oauth'; +import { createQueue } from '@metorial/queue'; +import { getSentry } from '@metorial/sentry'; +import { useDeploymentQueue } from '../../../lib/useDeploymentQueue'; +import { customServerVersionService } from '../../../services'; +import { DeploymentError } from '../../base/error'; +import { createPythonLambdaDeployment, PythonDeployment } from '../deployment'; + +let Sentry = getSentry(); + +export let pythonDeployMainQueue = createQueue<{ + lambdaId: string; + serverVersionData: Omit; +}>({ + name: 'csrv/python/main', + jobOpts: { + attempts: 10 + }, + workerOpts: { + concurrency: 1, + limiter: { + max: 5, + duration: 30 * 1000 + } + } +}); + +export let pythonDeployMainQueueProcessor = pythonDeployMainQueue.process(async data => { + let { failDeployment, stepManager, lambda, customServerVersion, deployment } = + await useDeploymentQueue({ + lambdaId: data.lambdaId, + serverVersionData: data.serverVersionData + }); + + let checkStep = await stepManager.createDeploymentStep({ + type: 'lambda_deploy_create', + status: 'running', + log: [ + { + type: 'info', + lines: [`Preparing deployment for managed server...`] + } + ] + }); + + let python: PythonDeployment; + + try { + python = await createPythonLambdaDeployment({ + lambdaServerInstance: lambda, + customServer: customServerVersion.customServer, + deployment + }); + + checkStep.complete([]); + } catch (error: any) { + console.error('Error during managed server deployment setup:', error); + + if (error instanceof DeploymentError) { + await checkStep.fail([ + { + type: 'error', + lines: [error.message] + } + ]); + await failDeployment(); + return; + } + + if (error.response && error.response.data && error.response.data.message) { + await checkStep.fail([ + { + type: 'error', + lines: [error.response.data.message] + } + ]); + } + Sentry.captureException(error); + await failDeployment(); + return; + } + + let buildStep = await stepManager.createDeploymentStep({ + type: 'lambda_deploy_build', + status: 'running', + log: [ + { + type: 'info', + lines: [`Building and deploying managed server...`] + } + ] + }); + + try { + while (true) { + await delay(2000); + let status = await python.pollDeploymentStatus(); + + for (let log of status.logs) { + buildStep.addLog(log.lines, log.type); + } + + if (status.status == 'success') { + buildStep.complete([ + { + type: 'info', + lines: ['Managed server deployed successfully.'] + } + ]); + break; + } else if (status.status == 'failed') { + buildStep.fail([ + { + type: 'info', + lines: ['Deployment failed.'] + } + ]); + await failDeployment(); + return; + } + } + } catch (error: any) { + Sentry.captureException(error); + await buildStep.fail(); + await failDeployment(); + return; + } + + let discoverStep = await stepManager.createDeploymentStep({ + type: 'discovering', + status: 'running', + log: [ + { + type: 'info', + lines: [`Discovering server capabilities...`] + } + ] + }); + + try { + let { capabilities, oauth, callbacks } = await python.discoverServer(); + data.serverVersionData.tools = capabilities.tools ?? []; + data.serverVersionData.resourceTemplates = capabilities.resourceTemplates ?? []; + data.serverVersionData.prompts = capabilities.prompts ?? []; + data.serverVersionData.serverCapabilities = capabilities.capabilities ?? []; + data.serverVersionData.serverInfo = capabilities.implementation ?? []; + data.serverVersionData.serverInstructions = capabilities.instructions || null; + await discoverStep.addLog([`Server capabilities discovered successfully.`], 'info'); + await discoverStep.addLog(JSON.stringify(capabilities, null, 2).split('\n'), 'info'); + + if (oauth.enabled) { + let config = await providerOauthConfigService.createConfig({ + instance: lambda.instance, + implementation: { + type: 'managed_server_http', + httpEndpoint: python.httpEndpoint, + hasRemoteOauthForm: !!oauth.hasForm, + lambdaServerInstanceOid: lambda.oid + } + }); + + await db.lambdaServerInstance.updateMany({ + where: { id: lambda.id }, + data: { + providerOAuthConfigOid: config.oid + } + }); + + await discoverStep.addLog( + ['Server implements custom OAuth. OAuth configuration created successfully.'], + 'info' + ); + } else if (lambda.providerOAuthConfigOid) { + let currentOauthConfig = await db.providerOAuthConfig.findFirstOrThrow({ + where: { oid: lambda.providerOAuthConfigOid } + }); + + // If the server used to be oauth enabled but isn't anymore, + // we remove the config from the lambda + if (currentOauthConfig.type == 'managed_server_http') { + await db.lambdaServerInstance.updateMany({ + where: { id: lambda.id }, + data: { + providerOAuthConfigOid: null + } + }); + } + } + + if (callbacks.enabled) { + let callbackTemplate = await db.callbackTemplate.create({ + data: { + id: await ID.generateId('callbackTemplate'), + eventType: callbacks.type + } + }); + + await db.lambdaServerInstance.updateMany({ + where: { id: lambda.id }, + data: { + callbackTemplateOid: callbackTemplate.oid + } + }); + + await discoverStep.addLog(['Discovered server callback support.'], 'info'); + } + + await discoverStep.complete(); + } catch (error: any) { + console.error('Error during managed server discovery:', error); + Sentry.captureException(error); + + if (error?.response?.data?.message) { + await discoverStep.addLog([error.response.data.message], 'error'); + } + + await discoverStep.fail([ + { + type: 'error', + lines: [`Managed server discovery failed.`] + } + ]); + await failDeployment(); + return; + } + + let deploymentStep = await stepManager.createDeploymentStep({ + type: 'deploying', + status: 'running', + log: [ + { + type: 'info', + lines: ['Deploying custom server to Metorial...'] + } + ] + }); + + try { + await withTransaction(async db => { + await deploymentStep.addLog(['Creating server version...']); + + let serverVersion = await db.serverVersion.create({ + data: { + ...data.serverVersionData, + lambdaOid: lambda.oid + } + }); + + let version = await db.customServerVersion.update({ + where: { id: customServerVersion.id }, + data: { + status: 'available', + serverVersionOid: serverVersion.oid + }, + include: { + serverVersion: true + } + }); + + await deploymentStep.addLog(['Updating current version...']); + + await customServerVersionService.setCurrentVersion({ + server: customServerVersion.customServer, + isEphemeralUpdate: true, + version + }); + + await db.customServerDeployment.updateMany({ + where: { id: deployment.id }, + data: { + status: 'completed', + endedAt: new Date() + } + }); + + await db.customServerDeploymentStep.updateMany({ + where: { deploymentOid: deployment.oid, status: 'running' }, + data: { status: 'completed', endedAt: new Date() } + }); + }); + + await deploymentStep.complete(); + + await stepManager.createDeploymentStep({ + type: 'deployed', + status: 'completed', + log: [ + { + type: 'info', + lines: [`Managed server deployed to Metorial successfully.`] + } + ] + }); + } catch (error: any) { + console.error('Error during managed server deployment:', error); + Sentry.captureException(error); + await deploymentStep.fail([ + { + type: 'error', + lines: [`Managed server deployment failed.`] + } + ]); + await failDeployment(); + return; + } +}); diff --git a/src/backend/modules/custom-server/src/env.ts b/src/backend/modules/custom-server/src/env.ts index 32a3d1ff..db2f1ff0 100644 --- a/src/backend/modules/custom-server/src/env.ts +++ b/src/backend/modules/custom-server/src/env.ts @@ -9,6 +9,10 @@ export let env = createValidatedEnv({ DENO_RUNNER_ADDRESS: v.optional(v.string()) }, + python: { + PYTHON_RUNNER_ADDRESS: v.optional(v.string()) + }, + aws: { AWS_ACCESS_KEY_ID: v.optional(v.string()), AWS_SECRET_ACCESS_KEY: v.optional(v.string()), diff --git a/src/backend/modules/custom-server/src/queues/initializeLambda.ts b/src/backend/modules/custom-server/src/queues/initializeLambda.ts index 2c3ddb58..c92bb8cd 100644 --- a/src/backend/modules/custom-server/src/queues/initializeLambda.ts +++ b/src/backend/modules/custom-server/src/queues/initializeLambda.ts @@ -6,6 +6,7 @@ import { isAwsLambdaEnabled } from '../deployment/aws-lambda/lib/aws'; import { lambdaDeployMainQueue } from '../deployment/aws-lambda/queues'; import { isDenoDeployEnabled } from '../deployment/deno/deployment'; import { denoDeployMainQueue } from '../deployment/deno/queues/main'; +import { pythonDeployMainQueue } from '../deployment/python-local/queues/main'; import { useDeploymentQueue } from '../lib/useDeploymentQueue'; let Sentry = getSentry(); @@ -70,8 +71,13 @@ export let initializeLambdaQueueProcessor = initializeLambdaQueue.process(async lang = 'ts'; } else if (content.runtime == 'python') { - provider = 'aws_lambda'; - lang = 'python'; + if (isAwsLambdaEnabled()) { + provider = 'aws_lambda'; + lang = 'python'; + } else { + provider = 'python_self_hosted'; + lang = 'python'; + } } } catch { deploymentStep.addLog(['Unable to find metorial.json']); @@ -117,6 +123,13 @@ export let initializeLambdaQueueProcessor = initializeLambdaQueue.process(async }); break; + case 'python_self_hosted': + await pythonDeployMainQueue.add({ + lambdaId: data.lambdaId, + serverVersionData: data.serverVersionData + }); + break; + default: Sentry.captureException( new Error(`Unsupported lambda provider: ${(lambda as any).provider}`)