From bcb9938100f351424e7bef0a09a759fdacfd39ee Mon Sep 17 00:00:00 2001 From: Maxime David Date: Tue, 16 May 2023 22:02:09 -0400 Subject: [PATCH 01/41] add tiered param --- function-deployer/app.js | 9 +++++++-- manifest.json | 25 ++++++++++++++++++++----- 2 files changed, 27 insertions(+), 7 deletions(-) diff --git a/function-deployer/app.js b/function-deployer/app.js index 0d16efddea..b44037f71b 100644 --- a/function-deployer/app.js +++ b/function-deployer/app.js @@ -60,7 +60,8 @@ const createFunction = async ( functionName, singleFunction, memorySize, - architecture + architecture, + environment ) => { const sanitizedRuntime = singleFunction.path ? singleFunction.path @@ -77,6 +78,9 @@ const createFunction = async ( ...(singleFunction.snapStart && singleFunction.snapStart), MemorySize: memorySize, Architectures: [architecture], + Environment: { + Variables: environment, + }, }; try { console.log( @@ -232,7 +236,8 @@ const deploy = async ( functionName, singleFunction, memorySize, - architecture + architecture, + singleFunction.environment ); await deleteLogGroup(cloudWatchLogsClient, functionName); await createLogGroup(cloudWatchLogsClient, functionName); diff --git a/manifest.json b/manifest.json index 26916c640b..7b97622298 100644 --- a/manifest.json +++ b/manifest.json @@ -84,14 +84,20 @@ "runtime": "java11", "handler": "io.github.maxday.Handler", "path": "java11", - "architectures": ["x86_64", "arm64"] + "architectures": ["x86_64", "arm64"], + "environment": { + "JAVA_TOOL_OPTIONS":"-XX:+TieredCompilation -XX:TieredStopAtLevel=1" + } }, { "displayName": "java8", "runtime": "java8.al2", "handler": "io.github.maxday.Handler", "path": "java8", - "architectures": ["x86_64", "arm64"] + "architectures": ["x86_64", "arm64"], + "environment": { + "JAVA_TOOL_OPTIONS":"-XX:+TieredCompilation -XX:TieredStopAtLevel=1" + } }, { "displayName": "ruby2.7", @@ -131,14 +137,20 @@ "ApplyOn": "PublishedVersions" } }, - "architectures": ["x86_64"] + "architectures": ["x86_64"], + "environment": { + "JAVA_TOOL_OPTIONS":"-XX:+TieredCompilation -XX:TieredStopAtLevel=1" + } }, { "displayName": "java17", "runtime": "java17", "handler": "io.github.maxday.Handler", "path": "java17", - "architectures": ["x86_64", "arm64"] + "architectures": ["x86_64", "arm64"], + "environment": { + "JAVA_TOOL_OPTIONS":"-XX:+TieredCompilation -XX:TieredStopAtLevel=1" + } }, { "displayName": "java17 snapstart", @@ -150,7 +162,10 @@ "ApplyOn": "PublishedVersions" } }, - "architectures": ["x86_64"] + "architectures": ["x86_64"], + "environment": { + "JAVA_TOOL_OPTIONS":"-XX:+TieredCompilation -XX:TieredStopAtLevel=1" + } }, { "displayName": "quarkus (prov.al2)", From 385ffdb021afc68f56fa67f3291b6f36c1ad4f71 Mon Sep 17 00:00:00 2001 From: Maxime David Date: Tue, 16 May 2023 22:19:53 -0400 Subject: [PATCH 02/41] fix manifest --- manifest.json | 57 +++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 51 insertions(+), 6 deletions(-) diff --git a/manifest.json b/manifest.json index 7b97622298..b75c53c5c4 100644 --- a/manifest.json +++ b/manifest.json @@ -66,7 +66,7 @@ "architectures": ["x86_64", "arm64"] }, { - "displayName": "dotnet7 aot (prov.al2)", + "displayName": "dotnet7 aot prov.al2", "runtime": "provided.al2", "handler": "bootstrap", "path": "dotnet7_aot_on_provided_al2", @@ -84,6 +84,13 @@ "runtime": "java11", "handler": "io.github.maxday.Handler", "path": "java11", + "architectures": ["x86_64", "arm64"] + }, + { + "displayName": "java11 TieredCompil", + "runtime": "java11", + "handler": "io.github.maxday.Handler", + "path": "java11", "architectures": ["x86_64", "arm64"], "environment": { "JAVA_TOOL_OPTIONS":"-XX:+TieredCompilation -XX:TieredStopAtLevel=1" @@ -94,6 +101,13 @@ "runtime": "java8.al2", "handler": "io.github.maxday.Handler", "path": "java8", + "architectures": ["x86_64", "arm64"] + }, + { + "displayName": "java8 TieredCompil", + "runtime": "java8.al2", + "handler": "io.github.maxday.Handler", + "path": "java8", "architectures": ["x86_64", "arm64"], "environment": { "JAVA_TOOL_OPTIONS":"-XX:+TieredCompilation -XX:TieredStopAtLevel=1" @@ -107,21 +121,21 @@ "architectures": ["x86_64", "arm64"] }, { - "displayName": "go (provided)", + "displayName": "go provided", "runtime": "provided", "handler": "bootstrap", "path": "go_on_provided", "architectures": ["x86_64"] }, { - "displayName": "go (prov.al2)", + "displayName": "go prov.al2", "runtime": "provided.al2", "handler": "bootstrap", "path": "go_on_provided_al2", "architectures": ["x86_64", "arm64"] }, { - "displayName": "rust (prov.al2)", + "displayName": "rust prov.al2", "runtime": "provided.al2", "handler": "bootstrap", "path": "rust_on_provided_al2", @@ -137,6 +151,18 @@ "ApplyOn": "PublishedVersions" } }, + "architectures": ["x86_64"] + }, + { + "displayName": "java11 snap TieredCompil", + "runtime": "java11", + "handler": "io.github.maxday.Handler", + "path": "java11_snapstart", + "snapStart": { + "SnapStart": { + "ApplyOn": "PublishedVersions" + } + }, "architectures": ["x86_64"], "environment": { "JAVA_TOOL_OPTIONS":"-XX:+TieredCompilation -XX:TieredStopAtLevel=1" @@ -147,6 +173,13 @@ "runtime": "java17", "handler": "io.github.maxday.Handler", "path": "java17", + "architectures": ["x86_64", "arm64"] + }, + { + "displayName": "java17 TieredCompil", + "runtime": "java17", + "handler": "io.github.maxday.Handler", + "path": "java17", "architectures": ["x86_64", "arm64"], "environment": { "JAVA_TOOL_OPTIONS":"-XX:+TieredCompilation -XX:TieredStopAtLevel=1" @@ -162,20 +195,32 @@ "ApplyOn": "PublishedVersions" } }, + "architectures": ["x86_64"] + }, + { + "displayName": "java17 snap TieredCompil", + "runtime": "java17", + "handler": "io.github.maxday.Handler", + "path": "java17_snapstart", + "snapStart": { + "SnapStart": { + "ApplyOn": "PublishedVersions" + } + }, "architectures": ["x86_64"], "environment": { "JAVA_TOOL_OPTIONS":"-XX:+TieredCompilation -XX:TieredStopAtLevel=1" } }, { - "displayName": "quarkus (prov.al2)", + "displayName": "quarkus prov.al2", "runtime": "provided.al2", "handler": "bootstrap", "path": "quarkus_native_on_provided_al2", "architectures": ["x86_64"] }, { - "displayName": "graalvm java17 (prov.al2)", + "displayName": "graalvm java17 prov.al2", "runtime": "provided.al2", "handler": "com.xavierbouclet.OkHandler", "path": "graalvm_java17_on_provided_al2", From 975c264565948a3c74aa77a1d279a44c7e4effe3 Mon Sep 17 00:00:00 2001 From: Maxime David Date: Tue, 16 May 2023 22:38:59 -0400 Subject: [PATCH 03/41] add slug --- function-deployer/app.js | 8 ++++---- function-invoker/app.js | 9 ++++++--- manifest.json | 5 +++++ 3 files changed, 15 insertions(+), 7 deletions(-) diff --git a/function-deployer/app.js b/function-deployer/app.js index b44037f71b..a7659363cf 100644 --- a/function-deployer/app.js +++ b/function-deployer/app.js @@ -17,7 +17,7 @@ const { const REGION = process.env.AWS_REGION; const ROLE_ARN = process.env.ROLE_ARN; const LOG_PROCESSOR_ARN = process.env.LOG_PROCESSOR_ARN; -const PROJECT = `lambda-perf`; +const PROJECT = "lambda-perf"; const delay = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); @@ -225,9 +225,9 @@ const deploy = async ( const runtimes = require("../manifest.json").runtimes; for (const singleFunction of runtimes) { if (singleFunction.architectures.includes(architecture)) { - const functionSufix = singleFunction.path - ? singleFunction.path - : singleFunction.runtime.replace(".", ""); + const functionSufix = singleFunction.slug + ? singleFunction.slug + : singleFunction.path; const functionName = `${PROJECT}-${functionSufix}-${memorySize}-${architecture}`; try { await deleteFunction(lambdaClient, functionName); diff --git a/function-invoker/app.js b/function-invoker/app.js index 8092343741..ae8b2adf85 100644 --- a/function-invoker/app.js +++ b/function-invoker/app.js @@ -6,7 +6,7 @@ const { } = require("@aws-sdk/client-lambda"); const REGION = process.env.AWS_REGION; -const PREFIX = "lambda-perf-"; +const PROJECT = "lambda-perf"; const NB_INVOKE = 10; const DELAY = 10000; @@ -57,10 +57,13 @@ function isSnapStart(path) { exports.handler = async (_, context) => { try { - const path = context.clientContext.path; + const functionSufix = context.clientContext.slug + ? context.clientContext.slug + : context.clientContext.path; + const architecture = context.clientContext.architecture; const memorySize = context.clientContext.memorySize; - const functionName = `${PREFIX}${path}-${memorySize}-${architecture}`; + const functionName = `${PROJECT}-${functionSufix}-${memorySize}-${architecture}`; const lambdaClient = new LambdaClient({ region: REGION }); let versions = []; diff --git a/manifest.json b/manifest.json index b75c53c5c4..5af08d457c 100644 --- a/manifest.json +++ b/manifest.json @@ -91,6 +91,7 @@ "runtime": "java11", "handler": "io.github.maxday.Handler", "path": "java11", + "slug": "java11_tiered", "architectures": ["x86_64", "arm64"], "environment": { "JAVA_TOOL_OPTIONS":"-XX:+TieredCompilation -XX:TieredStopAtLevel=1" @@ -108,6 +109,7 @@ "runtime": "java8.al2", "handler": "io.github.maxday.Handler", "path": "java8", + "slug": "java8_tiered", "architectures": ["x86_64", "arm64"], "environment": { "JAVA_TOOL_OPTIONS":"-XX:+TieredCompilation -XX:TieredStopAtLevel=1" @@ -158,6 +160,7 @@ "runtime": "java11", "handler": "io.github.maxday.Handler", "path": "java11_snapstart", + "slug": "java11_snapstart_tiered", "snapStart": { "SnapStart": { "ApplyOn": "PublishedVersions" @@ -180,6 +183,7 @@ "runtime": "java17", "handler": "io.github.maxday.Handler", "path": "java17", + "slug": "java17_tiered", "architectures": ["x86_64", "arm64"], "environment": { "JAVA_TOOL_OPTIONS":"-XX:+TieredCompilation -XX:TieredStopAtLevel=1" @@ -202,6 +206,7 @@ "runtime": "java17", "handler": "io.github.maxday.Handler", "path": "java17_snapstart", + "slug": "java17_snapstart_tiered", "snapStart": { "SnapStart": { "ApplyOn": "PublishedVersions" From 068b4b302cbdb599ab62f4126980412361def844 Mon Sep 17 00:00:00 2001 From: Maxime David Date: Wed, 17 May 2023 09:34:44 -0400 Subject: [PATCH 04/41] simplify invoker --- function-deployer/app.js | 62 ++++++++++++++++---------------- function-trigger-deployer/app.js | 20 +++++++---- 2 files changed, 44 insertions(+), 38 deletions(-) diff --git a/function-deployer/app.js b/function-deployer/app.js index a7659363cf..e81581579a 100644 --- a/function-deployer/app.js +++ b/function-deployer/app.js @@ -219,52 +219,50 @@ const addPermission = async (client, functionName) => { const deploy = async ( lambdaClient, cloudWatchLogsClient, + path, + slug, memorySize, architecture ) => { - const runtimes = require("../manifest.json").runtimes; - for (const singleFunction of runtimes) { - if (singleFunction.architectures.includes(architecture)) { - const functionSufix = singleFunction.slug - ? singleFunction.slug - : singleFunction.path; - const functionName = `${PROJECT}-${functionSufix}-${memorySize}-${architecture}`; - try { - await deleteFunction(lambdaClient, functionName); - await createFunction( - lambdaClient, - functionName, - singleFunction, - memorySize, - architecture, - singleFunction.environment - ); - await deleteLogGroup(cloudWatchLogsClient, functionName); - await createLogGroup(cloudWatchLogsClient, functionName); - await createSubscriptionFilter(cloudWatchLogsClient, functionName); - } catch (e) { - console.error(e); - throw e; - } - await delay(5000); - } else { - console.log( - `Skipping ${singleFunction.path} as it's not available for ${architecture}` - ); - } + const functionSufix = slug ? slug : path; + const functionName = `${PROJECT}-${functionSufix}-${memorySize}-${architecture}`; + try { + await deleteFunction(lambdaClient, functionName); + await createFunction( + lambdaClient, + functionName, + singleFunction, + memorySize, + architecture, + singleFunction.environment + ); + await deleteLogGroup(cloudWatchLogsClient, functionName); + await createLogGroup(cloudWatchLogsClient, functionName); + await createSubscriptionFilter(cloudWatchLogsClient, functionName); + } catch (e) { + console.error(e); + throw e; } + await delay(5000); }; exports.handler = async (_, context) => { try { console.log("clientContext = ", context.clientContext); - const { memorySize, architecture } = context.clientContext; + const { memorySize, architecture, path, slug } = context.clientContext; const lambdaClient = new LambdaClient({ region: REGION }); const cloudWatchLogsClient = new CloudWatchLogsClient({ region: REGION, }); await addPermission(lambdaClient, LOG_PROCESSOR_ARN); - await deploy(lambdaClient, cloudWatchLogsClient, memorySize, architecture); + await deploy( + lambdaClient, + cloudWatchLogsClient, + path, + slug, + memorySize, + architecture + ); return { statusCode: 200, body: JSON.stringify("success"), diff --git a/function-trigger-deployer/app.js b/function-trigger-deployer/app.js index 9c4d4f1803..907948609e 100644 --- a/function-trigger-deployer/app.js +++ b/function-trigger-deployer/app.js @@ -3,11 +3,11 @@ const { LambdaClient, InvokeCommand } = require("@aws-sdk/client-lambda"); const DEPLOYER = process.env.DEPLOYER; const REGION = process.env.AWS_REGION; -const invokeFunction = async (client, memorySize, architecture) => { +const invokeFunction = async (client, path, slug, memorySize, architecture) => { const params = { FunctionName: DEPLOYER, ClientContext: Buffer.from( - JSON.stringify({ memorySize, architecture }) + JSON.stringify({ path, slug, memorySize, architecture }) ).toString("base64"), }; try { @@ -27,10 +27,18 @@ exports.handler = async (_, context) => { const lambdaClient = new LambdaClient({ region: REGION }); for (memorySize of manifest.memorySizes) { - for (architecture of manifest.architectures) { - allPromises.push( - invokeFunction(lambdaClient, memorySize, architecture) - ); + for (runtime of manifest.runtimes) { + for (architecture of runtime.architectures) { + allPromises.push( + invokeFunction( + lambdaClient, + runtime.path, + runtime.slug, + memorySize, + architecture + ) + ); + } } } await Promise.all(allPromises); From 6b0b6a166faa1a31db25656975196860141e32fa Mon Sep 17 00:00:00 2001 From: Maxime David Date: Wed, 17 May 2023 09:59:58 -0400 Subject: [PATCH 05/41] prevent tooManyRequestException --- function-deployer/app.js | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/function-deployer/app.js b/function-deployer/app.js index e81581579a..51a9fc8046 100644 --- a/function-deployer/app.js +++ b/function-deployer/app.js @@ -200,7 +200,10 @@ const createSubscriptionFilter = async (client, functionName) => { } }; -const addPermission = async (client, functionName) => { +const addPermission = async (client, functionName, nbRetry) => { + if (nbRetry > 5) { + throw "max retries exceeded"; + } const params = { FunctionName: functionName, Action: "lambda:InvokeFunction", @@ -213,6 +216,8 @@ const addPermission = async (client, functionName) => { console.log(`permission added to ${functionName}`); } catch (e) { console.error(e); + await delay(2000); + await addPermission(client, functionName, nbRetry + 1); } }; From 28f36076488ee98b999fc1accc483552b3f403ee Mon Sep 17 00:00:00 2001 From: Maxime David Date: Wed, 17 May 2023 10:19:05 -0400 Subject: [PATCH 06/41] skip add permission --- function-deployer/app.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/function-deployer/app.js b/function-deployer/app.js index 51a9fc8046..a2fb9883dd 100644 --- a/function-deployer/app.js +++ b/function-deployer/app.js @@ -259,7 +259,7 @@ exports.handler = async (_, context) => { const cloudWatchLogsClient = new CloudWatchLogsClient({ region: REGION, }); - await addPermission(lambdaClient, LOG_PROCESSOR_ARN); + //await addPermission(lambdaClient, LOG_PROCESSOR_ARN); await deploy( lambdaClient, cloudWatchLogsClient, From f1b380d91ad0696bde4845ca035e15c03f9bc8a8 Mon Sep 17 00:00:00 2001 From: Maxime David Date: Wed, 17 May 2023 10:25:15 -0400 Subject: [PATCH 07/41] skip add permission --- function-deployer/app.js | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/function-deployer/app.js b/function-deployer/app.js index a2fb9883dd..703bad63c0 100644 --- a/function-deployer/app.js +++ b/function-deployer/app.js @@ -227,8 +227,12 @@ const deploy = async ( path, slug, memorySize, - architecture + architecture, + nbRetry ) => { + if (nbRetry > 5) { + throw "max retries exceeded"; + } const functionSufix = slug ? slug : path; const functionName = `${PROJECT}-${functionSufix}-${memorySize}-${architecture}`; try { @@ -245,10 +249,17 @@ const deploy = async ( await createLogGroup(cloudWatchLogsClient, functionName); await createSubscriptionFilter(cloudWatchLogsClient, functionName); } catch (e) { - console.error(e); - throw e; + await delay(5000); + await deploy( + lambdaClient, + cloudWatchLogsClient, + path, + slug, + memorySize, + architecture, + nbRetry + 1 + ); } - await delay(5000); }; exports.handler = async (_, context) => { From b864955591f42d1035faf36d0804554cea00d313 Mon Sep 17 00:00:00 2001 From: Maxime David Date: Wed, 17 May 2023 10:34:30 -0400 Subject: [PATCH 08/41] skip add permission --- function-deployer/app.js | 42 ++++++++++++++++++++++------------------ 1 file changed, 23 insertions(+), 19 deletions(-) diff --git a/function-deployer/app.js b/function-deployer/app.js index 703bad63c0..2b70e12373 100644 --- a/function-deployer/app.js +++ b/function-deployer/app.js @@ -21,7 +21,10 @@ const PROJECT = "lambda-perf"; const delay = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); -const deleteFunction = async (client, functionName) => { +const deleteFunction = async (client, functionName, nbRetry) => { + if (nbRetry > 5) { + throw "max retries exceeded"; + } const params = { FunctionName: functionName, }; @@ -34,7 +37,8 @@ const deleteFunction = async (client, functionName) => { console.log(`function ${functionName} does not exist, skipping deletion`); } else { console.error(e); - throw e; + await delay(5000); + await deleteFunction(client, functionName, nbRetry + 1); } } }; @@ -61,8 +65,12 @@ const createFunction = async ( singleFunction, memorySize, architecture, - environment + environment, + nbRetry ) => { + if (nbRetry > 5) { + throw "max retries exceeded"; + } const sanitizedRuntime = singleFunction.path ? singleFunction.path : singleFunction.runtime.replace(".", ""); @@ -102,7 +110,16 @@ const createFunction = async ( } } catch (e) { console.error(e); - throw e; + await delay(5000); + await createFunction( + client, + functionName, + singleFunction, + memorySize, + architecture, + environment, + nbRetry + 1 + ); } }; @@ -227,12 +244,8 @@ const deploy = async ( path, slug, memorySize, - architecture, - nbRetry + architecture ) => { - if (nbRetry > 5) { - throw "max retries exceeded"; - } const functionSufix = slug ? slug : path; const functionName = `${PROJECT}-${functionSufix}-${memorySize}-${architecture}`; try { @@ -249,16 +262,7 @@ const deploy = async ( await createLogGroup(cloudWatchLogsClient, functionName); await createSubscriptionFilter(cloudWatchLogsClient, functionName); } catch (e) { - await delay(5000); - await deploy( - lambdaClient, - cloudWatchLogsClient, - path, - slug, - memorySize, - architecture, - nbRetry + 1 - ); + throw e; } }; From 92addb51dd2f56bc4228ed8794626648f904f43f Mon Sep 17 00:00:00 2001 From: Maxime David Date: Wed, 17 May 2023 10:37:47 -0400 Subject: [PATCH 09/41] skip add permission --- serverless.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/serverless.yaml b/serverless.yaml index 09a8dee692..85b4a0d40c 100644 --- a/serverless.yaml +++ b/serverless.yaml @@ -124,6 +124,7 @@ functions: - "!report-log-processor" - "!result-builder" handler: function-deployer/app.handler + timeout: 120 functionInvoker: package: From a59ac98223c8b9f7e3a03749a38102cb20c83e84 Mon Sep 17 00:00:00 2001 From: Maxime David Date: Wed, 17 May 2023 10:55:47 -0400 Subject: [PATCH 10/41] better deployer --- function-deployer/app.js | 48 +++++++++++++++++++++----------- function-trigger-deployer/app.js | 26 +++++++++++++++-- 2 files changed, 55 insertions(+), 19 deletions(-) diff --git a/function-deployer/app.js b/function-deployer/app.js index 2b70e12373..1779b683c0 100644 --- a/function-deployer/app.js +++ b/function-deployer/app.js @@ -62,28 +62,27 @@ async function updateFunction(client, functionName) { const createFunction = async ( client, functionName, - singleFunction, + runtime, memorySize, architecture, environment, + snapStart, nbRetry ) => { if (nbRetry > 5) { throw "max retries exceeded"; } - const sanitizedRuntime = singleFunction.path - ? singleFunction.path - : singleFunction.runtime.replace(".", ""); + const sanitizedRuntime = runtime.replace(".", ""); const params = { FunctionName: functionName, - Handler: singleFunction.handler, - Runtime: singleFunction.runtime, + Handler: handler, + Runtime: runtime, Code: { S3Bucket: `${PROJECT}-${REGION}`, S3Key: `${sanitizedRuntime}/code_${architecture}.zip`, }, Role: ROLE_ARN, - ...(singleFunction.snapStart && singleFunction.snapStart), + ...snapStart, MemorySize: memorySize, Architectures: [architecture], Environment: { @@ -97,7 +96,7 @@ const createFunction = async ( const command = new CreateFunctionCommand(params); await client.send(command); - if (singleFunction.snapStart) { + if (snapStart) { await waitForActive(client, functionName); //publish 10 versions @@ -105,7 +104,7 @@ const createFunction = async ( //update variables for publishing new version await updateFunction(client, functionName); await delay(10000); - const resp = await publishVersion(client, functionName, 0); + await publishVersion(client, functionName, 0); } } } catch (e) { @@ -114,10 +113,11 @@ const createFunction = async ( await createFunction( client, functionName, - singleFunction, + runtime, memorySize, architecture, environment, + snapStart, nbRetry + 1 ); } @@ -243,20 +243,25 @@ const deploy = async ( cloudWatchLogsClient, path, slug, + runtime, memorySize, - architecture + architecture, + environment, + snapStart ) => { const functionSufix = slug ? slug : path; const functionName = `${PROJECT}-${functionSufix}-${memorySize}-${architecture}`; try { - await deleteFunction(lambdaClient, functionName); + await deleteFunction(lambdaClient, functionName, 0); await createFunction( lambdaClient, functionName, - singleFunction, + runtime, memorySize, architecture, - singleFunction.environment + environment, + snapStart, + 0 ); await deleteLogGroup(cloudWatchLogsClient, functionName); await createLogGroup(cloudWatchLogsClient, functionName); @@ -269,7 +274,15 @@ const deploy = async ( exports.handler = async (_, context) => { try { console.log("clientContext = ", context.clientContext); - const { memorySize, architecture, path, slug } = context.clientContext; + const { + memorySize, + architecture, + path, + slug, + runtime, + environment, + snapStart, + } = context.clientContext; const lambdaClient = new LambdaClient({ region: REGION }); const cloudWatchLogsClient = new CloudWatchLogsClient({ region: REGION, @@ -280,8 +293,11 @@ exports.handler = async (_, context) => { cloudWatchLogsClient, path, slug, + runtime, memorySize, - architecture + architecture, + environment, + snapStart ); return { statusCode: 200, diff --git a/function-trigger-deployer/app.js b/function-trigger-deployer/app.js index 907948609e..36e76dc1aa 100644 --- a/function-trigger-deployer/app.js +++ b/function-trigger-deployer/app.js @@ -3,11 +3,28 @@ const { LambdaClient, InvokeCommand } = require("@aws-sdk/client-lambda"); const DEPLOYER = process.env.DEPLOYER; const REGION = process.env.AWS_REGION; -const invokeFunction = async (client, path, slug, memorySize, architecture) => { +const invokeFunction = async ( + client, + path, + slug, + memorySize, + architecture, + runtime, + environment, + snapStart +) => { const params = { FunctionName: DEPLOYER, ClientContext: Buffer.from( - JSON.stringify({ path, slug, memorySize, architecture }) + JSON.stringify({ + path, + slug, + memorySize, + architecture, + runtime, + environment, + snapStart, + }) ).toString("base64"), }; try { @@ -35,7 +52,10 @@ exports.handler = async (_, context) => { runtime.path, runtime.slug, memorySize, - architecture + architecture, + runtime.runtime, + runtime.environment, + runtime.snapStart ) ); } From 2dd1940aa9a81d8df2d50977edf5bbffeb070e99 Mon Sep 17 00:00:00 2001 From: Maxime David Date: Wed, 17 May 2023 11:01:47 -0400 Subject: [PATCH 11/41] better deployer --- function-deployer/app.js | 6 ++++++ function-trigger-deployer/app.js | 3 +++ 2 files changed, 9 insertions(+) diff --git a/function-deployer/app.js b/function-deployer/app.js index 1779b683c0..c9d8127e33 100644 --- a/function-deployer/app.js +++ b/function-deployer/app.js @@ -63,6 +63,7 @@ const createFunction = async ( client, functionName, runtime, + handler, memorySize, architecture, environment, @@ -113,6 +114,7 @@ const createFunction = async ( await createFunction( client, functionName, + handler, runtime, memorySize, architecture, @@ -243,6 +245,7 @@ const deploy = async ( cloudWatchLogsClient, path, slug, + handler, runtime, memorySize, architecture, @@ -256,6 +259,7 @@ const deploy = async ( await createFunction( lambdaClient, functionName, + handler, runtime, memorySize, architecture, @@ -279,6 +283,7 @@ exports.handler = async (_, context) => { architecture, path, slug, + handler, runtime, environment, snapStart, @@ -293,6 +298,7 @@ exports.handler = async (_, context) => { cloudWatchLogsClient, path, slug, + handler, runtime, memorySize, architecture, diff --git a/function-trigger-deployer/app.js b/function-trigger-deployer/app.js index 36e76dc1aa..22e7dc3808 100644 --- a/function-trigger-deployer/app.js +++ b/function-trigger-deployer/app.js @@ -6,6 +6,7 @@ const REGION = process.env.AWS_REGION; const invokeFunction = async ( client, path, + handler, slug, memorySize, architecture, @@ -19,6 +20,7 @@ const invokeFunction = async ( JSON.stringify({ path, slug, + handler, memorySize, architecture, runtime, @@ -50,6 +52,7 @@ exports.handler = async (_, context) => { invokeFunction( lambdaClient, runtime.path, + runtime.handler, runtime.slug, memorySize, architecture, From c7e0279f80ad02361c48aef39594f06c81892431 Mon Sep 17 00:00:00 2001 From: Maxime David Date: Wed, 17 May 2023 11:14:19 -0400 Subject: [PATCH 12/41] better deployer --- function-deployer/app.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/function-deployer/app.js b/function-deployer/app.js index c9d8127e33..cdf3a0be53 100644 --- a/function-deployer/app.js +++ b/function-deployer/app.js @@ -62,8 +62,8 @@ async function updateFunction(client, functionName) { const createFunction = async ( client, functionName, - runtime, handler, + runtime, memorySize, architecture, environment, From 4fa91cf809c6f026647738bffa88955839a5ac24 Mon Sep 17 00:00:00 2001 From: Maxime David Date: Wed, 17 May 2023 11:19:52 -0400 Subject: [PATCH 13/41] better deployer --- function-deployer/app.js | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/function-deployer/app.js b/function-deployer/app.js index cdf3a0be53..5f9f1ab9e7 100644 --- a/function-deployer/app.js +++ b/function-deployer/app.js @@ -62,6 +62,7 @@ async function updateFunction(client, functionName) { const createFunction = async ( client, functionName, + path, handler, runtime, memorySize, @@ -73,14 +74,13 @@ const createFunction = async ( if (nbRetry > 5) { throw "max retries exceeded"; } - const sanitizedRuntime = runtime.replace(".", ""); const params = { FunctionName: functionName, Handler: handler, Runtime: runtime, Code: { S3Bucket: `${PROJECT}-${REGION}`, - S3Key: `${sanitizedRuntime}/code_${architecture}.zip`, + S3Key: `${path}/code_${architecture}.zip`, }, Role: ROLE_ARN, ...snapStart, @@ -114,6 +114,7 @@ const createFunction = async ( await createFunction( client, functionName, + path, handler, runtime, memorySize, @@ -259,6 +260,7 @@ const deploy = async ( await createFunction( lambdaClient, functionName, + path, handler, runtime, memorySize, From 00aac4a4df574a31244a1c5aadc19289a551769f Mon Sep 17 00:00:00 2001 From: Maxime David Date: Wed, 17 May 2023 11:30:50 -0400 Subject: [PATCH 14/41] better deployer --- function-invoker/app.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/function-invoker/app.js b/function-invoker/app.js index ae8b2adf85..f40050f41d 100644 --- a/function-invoker/app.js +++ b/function-invoker/app.js @@ -85,8 +85,9 @@ exports.handler = async (_, context) => { statusCode: 200, body: JSON.stringify("success"), }; - } catch (_) { - throw "failure"; + } catch (e) { + console.error(e); + context.fail(); } }; From a5429837b0a7138d7f4c24ed9ee02a500ffdcd8a Mon Sep 17 00:00:00 2001 From: Maxime David Date: Wed, 17 May 2023 11:38:10 -0400 Subject: [PATCH 15/41] better deployer --- function-invoker/app.js | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/function-invoker/app.js b/function-invoker/app.js index f40050f41d..417b9c042d 100644 --- a/function-invoker/app.js +++ b/function-invoker/app.js @@ -57,12 +57,10 @@ function isSnapStart(path) { exports.handler = async (_, context) => { try { - const functionSufix = context.clientContext.slug - ? context.clientContext.slug - : context.clientContext.path; + const { path, slug, architecture, memorySize } = context.clientContext; + + const functionSufix = slug ? slug : path; - const architecture = context.clientContext.architecture; - const memorySize = context.clientContext.memorySize; const functionName = `${PROJECT}-${functionSufix}-${memorySize}-${architecture}`; const lambdaClient = new LambdaClient({ region: REGION }); From f07f17cd11e46fd760406dcd6ac14131abe2b1e2 Mon Sep 17 00:00:00 2001 From: Maxime David Date: Wed, 17 May 2023 11:49:17 -0400 Subject: [PATCH 16/41] better deployer --- function-trigger-invoker/app.js | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/function-trigger-invoker/app.js b/function-trigger-invoker/app.js index b015b6bba2..1674f09436 100644 --- a/function-trigger-invoker/app.js +++ b/function-trigger-invoker/app.js @@ -66,11 +66,12 @@ const invokeFunction = async (client, runtime, architecture, memorySize) => { ClientContext: Buffer.from(clientContext).toString("base64"), }; try { - const command = new InvokeCommand(params); - await client.send(command); console.log( - `function ${params.FunctionName} invoked with clientContext = ${clientContext}` + `function ${params.FunctionName} invoking with clientContext = ${clientContext}` ); + const command = new InvokeCommand(params); + await client.send(command); + console.log(`function ${params.FunctionName} invoked succeeded`); } catch (e) { console.error(e); throw e; From 407b8efe0805a5a0706767b794447765919c31e6 Mon Sep 17 00:00:00 2001 From: Maxime David Date: Wed, 17 May 2023 11:58:51 -0400 Subject: [PATCH 17/41] better deployer --- function-deployer/app.js | 1 + function-trigger-deployer/app.js | 27 ++++++++++++++------------- 2 files changed, 15 insertions(+), 13 deletions(-) diff --git a/function-deployer/app.js b/function-deployer/app.js index 5f9f1ab9e7..95bcdec8a7 100644 --- a/function-deployer/app.js +++ b/function-deployer/app.js @@ -29,6 +29,7 @@ const deleteFunction = async (client, functionName, nbRetry) => { FunctionName: functionName, }; try { + console.log(`function ${functionName} is about to be deleted`); const command = new DeleteFunctionCommand(params); await client.send(command); console.log(`function ${functionName} deleted`); diff --git a/function-trigger-deployer/app.js b/function-trigger-deployer/app.js index 22e7dc3808..752c82c5ff 100644 --- a/function-trigger-deployer/app.js +++ b/function-trigger-deployer/app.js @@ -14,25 +14,26 @@ const invokeFunction = async ( environment, snapStart ) => { + const clientContext = JSON.stringify({ + path, + slug, + handler, + memorySize, + architecture, + runtime, + environment, + snapStart, + }); const params = { FunctionName: DEPLOYER, - ClientContext: Buffer.from( - JSON.stringify({ - path, - slug, - handler, - memorySize, - architecture, - runtime, - environment, - snapStart, - }) - ).toString("base64"), + ClientContext: Buffer.from(clientContext).toString("base64"), }; try { const command = new InvokeCommand(params); await client.send(command); - console.log(`function ${params.FunctionName} invoked`); + console.log( + `function ${params.FunctionName} invoked, client context = ${clientContext}` + ); } catch (e) { console.error(e); throw e; From 7e3e30724faab1f3fb070e95b2ec0ebfadcec647 Mon Sep 17 00:00:00 2001 From: Maxime David Date: Wed, 17 May 2023 11:59:08 -0400 Subject: [PATCH 18/41] better deployer --- serverless.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/serverless.yaml b/serverless.yaml index 85b4a0d40c..09a8dee692 100644 --- a/serverless.yaml +++ b/serverless.yaml @@ -124,7 +124,6 @@ functions: - "!report-log-processor" - "!result-builder" handler: function-deployer/app.handler - timeout: 120 functionInvoker: package: From fd686ba087530d46c61018dde6f95ed44a2d194f Mon Sep 17 00:00:00 2001 From: Maxime David Date: Wed, 17 May 2023 12:44:32 -0400 Subject: [PATCH 19/41] better deployer --- function-trigger-invoker/app.js | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/function-trigger-invoker/app.js b/function-trigger-invoker/app.js index 1674f09436..febf3a684d 100644 --- a/function-trigger-invoker/app.js +++ b/function-trigger-invoker/app.js @@ -71,8 +71,13 @@ const invokeFunction = async (client, runtime, architecture, memorySize) => { ); const command = new InvokeCommand(params); await client.send(command); - console.log(`function ${params.FunctionName} invoked succeeded`); + console.log( + `function ${params.FunctionName} invoked succeeded with clientContext = ${clientContext}` + ); } catch (e) { + console.error( + `function ${params.FunctionName} invoked failure with clientContext = ${clientContext}` + ); console.error(e); throw e; } From 6956b41b189c58dcfee262eb1e51b7e32631221b Mon Sep 17 00:00:00 2001 From: Maxime David Date: Wed, 17 May 2023 13:00:49 -0400 Subject: [PATCH 20/41] better deployer --- function-trigger-invoker/app.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/function-trigger-invoker/app.js b/function-trigger-invoker/app.js index febf3a684d..d54545367c 100644 --- a/function-trigger-invoker/app.js +++ b/function-trigger-invoker/app.js @@ -104,7 +104,8 @@ exports.handler = async () => { } } } - await Promise.all(allPromises); + const r = await Promise.allSettled(allPromises); + console.log(r); return { statusCode: 200, body: JSON.stringify("success"), From 8670935fe00863acdeacb33228e8e018466399de Mon Sep 17 00:00:00 2001 From: Maxime David Date: Wed, 17 May 2023 13:08:13 -0400 Subject: [PATCH 21/41] better deployer --- function-trigger-invoker/app.js | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/function-trigger-invoker/app.js b/function-trigger-invoker/app.js index d54545367c..24f0809e1d 100644 --- a/function-trigger-invoker/app.js +++ b/function-trigger-invoker/app.js @@ -94,18 +94,25 @@ exports.handler = async () => { await createTable(dynamoDbClient, TABLE); await delay(DELAY); const lambdaClient = new LambdaClient({ region: REGION }); - const allPromises = []; + let allPromises = []; + + // we need to batch for 50 functions to avoid the invalid signature error + // see more details: https://repost.aws/knowledge-center/lambda-sdk-signature for (runtime of manifest.runtimes) { for (architecture of runtime.architectures) { for (memorySize of manifest.memorySizes) { + if (allPromises.length === 50) { + console.log("array is full, invoking the current batch"); + await Promise.all(allPromises); + allPromises = []; + } allPromises.push( invokeFunction(lambdaClient, runtime, architecture, memorySize) ); } } } - const r = await Promise.allSettled(allPromises); - console.log(r); + await Promise.all(allPromises); return { statusCode: 200, body: JSON.stringify("success"), From f1911bb009add324368b2259dd92343e41bb742a Mon Sep 17 00:00:00 2001 From: Maxime David Date: Wed, 17 May 2023 13:08:17 -0400 Subject: [PATCH 22/41] better deployer --- function-trigger-invoker/app.js | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/function-trigger-invoker/app.js b/function-trigger-invoker/app.js index 24f0809e1d..e63ef532a7 100644 --- a/function-trigger-invoker/app.js +++ b/function-trigger-invoker/app.js @@ -9,6 +9,7 @@ const REGION = process.env.AWS_REGION; const INVOKER = process.env.INVOKER; const TABLE = "report-log"; const DELAY = 5000; +const MAX_LAMBDA_QUEUE = 50; const deleteTable = async (client, table) => { const params = { @@ -96,13 +97,13 @@ exports.handler = async () => { const lambdaClient = new LambdaClient({ region: REGION }); let allPromises = []; - // we need to batch for 50 functions to avoid the invalid signature error + // we need to batch for MAX_LAMBDA_QUEUE functions to avoid the invalid signature error // see more details: https://repost.aws/knowledge-center/lambda-sdk-signature for (runtime of manifest.runtimes) { for (architecture of runtime.architectures) { for (memorySize of manifest.memorySizes) { - if (allPromises.length === 50) { - console.log("array is full, invoking the current batch"); + if (allPromises.length === MAX_LAMBDA_QUEUE) { + console.log("array is full, resolving the current batch"); await Promise.all(allPromises); allPromises = []; } From 655b748dd2d19d4a03fb275b20c863482914489d Mon Sep 17 00:00:00 2001 From: Maxime David Date: Wed, 17 May 2023 15:54:29 -0400 Subject: [PATCH 23/41] add env --- function-invoker/app.js | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/function-invoker/app.js b/function-invoker/app.js index 417b9c042d..589caa5761 100644 --- a/function-invoker/app.js +++ b/function-invoker/app.js @@ -28,14 +28,14 @@ const invokeFunction = async (client, functionName, nbRetry) => { } }; -const updateFunction = async (client, functionName, nbRetry) => { +const updateFunction = async (client, functionName, environment, nbRetry) => { if (nbRetry > 5) { throw "max retries exceeded in updateFunction"; } const params = { FunctionName: functionName, Environment: { - Variables: { coldStart: `${Math.random()}` }, + Variables: { coldStart: `${Math.random()}`, ...environment }, }, }; try { @@ -57,7 +57,8 @@ function isSnapStart(path) { exports.handler = async (_, context) => { try { - const { path, slug, architecture, memorySize } = context.clientContext; + const { path, slug, architecture, memorySize, environment } = + context.clientContext; const functionSufix = slug ? slug : path; @@ -75,7 +76,7 @@ exports.handler = async (_, context) => { await invokeFunction(lambdaClient, functionNameWithVersion, 0); } else { await invokeFunction(lambdaClient, functionName, 0); - await updateFunction(lambdaClient, functionName, 0); + await updateFunction(lambdaClient, functionName, environment, 0); } await delay(DELAY); } From 934ab77b7821539f5888d033d0cb04e5ea776192 Mon Sep 17 00:00:00 2001 From: Maxime David Date: Wed, 17 May 2023 15:58:41 -0400 Subject: [PATCH 24/41] add env --- report-log-processor/app.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/report-log-processor/app.js b/report-log-processor/app.js index ce4d07a1e0..e93ecdabf0 100644 --- a/report-log-processor/app.js +++ b/report-log-processor/app.js @@ -15,6 +15,7 @@ exports.handler = async (input, context) => { const functionName = fromLambda.replace("lambda-perf-", ""); const tokens = functionName.split("-"); if (tokens.length !== 3) { + console.error(`token error (${token})`); context.fail(); } const name = tokens[0]; @@ -22,6 +23,7 @@ exports.handler = async (input, context) => { const filter = runtimes.filter((e) => e.path === name); if (filter.length !== 1) { // could not find the display name + console.error(`filter mismatch (${filter})`); context.fail(); } const displayName = filter[0].displayName; From 0cefb88e7da3ba01de6508b3cb881f396baef639 Mon Sep 17 00:00:00 2001 From: Maxime David Date: Wed, 17 May 2023 16:01:25 -0400 Subject: [PATCH 25/41] add env --- report-log-processor/app.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/report-log-processor/app.js b/report-log-processor/app.js index e93ecdabf0..84a7a5f6f2 100644 --- a/report-log-processor/app.js +++ b/report-log-processor/app.js @@ -14,12 +14,16 @@ exports.handler = async (input, context) => { const fromLambda = result.logGroup.replace("/aws/lambda/", ""); const functionName = fromLambda.replace("lambda-perf-", ""); const tokens = functionName.split("-"); + console.log("functionName", functionName); + console.log("tokens", tokens); if (tokens.length !== 3) { console.error(`token error (${token})`); context.fail(); } const name = tokens[0]; const architecture = tokens[2]; + console.log("name", name); + console.log("architecture", architecture); const filter = runtimes.filter((e) => e.path === name); if (filter.length !== 1) { // could not find the display name From e0a167f1a17f8b6e3c3e6d4e177c47d252a0e8b4 Mon Sep 17 00:00:00 2001 From: Maxime David Date: Wed, 17 May 2023 17:33:10 -0400 Subject: [PATCH 26/41] add env --- function-deployer/app.js | 55 ++++++++++++++++++++++--------------- report-log-processor/app.js | 2 +- 2 files changed, 34 insertions(+), 23 deletions(-) diff --git a/function-deployer/app.js b/function-deployer/app.js index 95bcdec8a7..a6a3e00619 100644 --- a/function-deployer/app.js +++ b/function-deployer/app.js @@ -18,11 +18,13 @@ const REGION = process.env.AWS_REGION; const ROLE_ARN = process.env.ROLE_ARN; const LOG_PROCESSOR_ARN = process.env.LOG_PROCESSOR_ARN; const PROJECT = "lambda-perf"; +const MAX_RETRY = 10; +const RETRY_DELAY = 20000; const delay = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); const deleteFunction = async (client, functionName, nbRetry) => { - if (nbRetry > 5) { + if (nbRetry > MAX_RETRY) { throw "max retries exceeded"; } const params = { @@ -38,7 +40,7 @@ const deleteFunction = async (client, functionName, nbRetry) => { console.log(`function ${functionName} does not exist, skipping deletion`); } else { console.error(e); - await delay(5000); + await delay(RETRY_DELAY); await deleteFunction(client, functionName, nbRetry + 1); } } @@ -72,7 +74,7 @@ const createFunction = async ( snapStart, nbRetry ) => { - if (nbRetry > 5) { + if (nbRetry > MAX_RETRY) { throw "max retries exceeded"; } const params = { @@ -105,13 +107,13 @@ const createFunction = async ( for (let i = 0; i < 10; i++) { //update variables for publishing new version await updateFunction(client, functionName); - await delay(10000); + await delay(RETRY_DELAY); await publishVersion(client, functionName, 0); } } } catch (e) { console.error(e); - await delay(5000); + await delay(RETRY_DELAY); await createFunction( client, functionName, @@ -133,7 +135,7 @@ const waitForActive = async (client, functionName) => { console.log(`waiting for function ${functionName} to be active`, func); while (func.Configuration.State !== "Active") { console.log(`waiting for function ${functionName} to be active`); - await delay(1000); + await delay(RETRY_DELAY); func = await getFunction(client, functionName); } }; @@ -151,7 +153,7 @@ const getFunction = async (client, functionName) => { } }; const publishVersion = async (client, functionName, nbRetry) => { - if (nbRetry > 5) { + if (nbRetry > MAX_RETRY) { throw "max retries exceeded"; } const params = { @@ -165,12 +167,15 @@ const publishVersion = async (client, functionName, nbRetry) => { ); } catch (e) { console.error(e); - await delay(20000); + await delay(RETRY_DELAY); await publishVersion(client, functionName, nbRetry + 1); } }; -const deleteLogGroup = async (client, functionName) => { +const deleteLogGroup = async (client, functionName, nbRetry) => { + if (nbRetry > MAX_RETRY) { + throw "max retries exceeded"; + } const params = { logGroupName: `/aws/lambda/${functionName}`, }; @@ -184,13 +189,16 @@ const deleteLogGroup = async (client, functionName) => { `loggroup ${params.logGroupName} does not exist, skipping deletion` ); } else { - console.error(e); - throw e; + await delay(RETRY_DELAY); + await deleteLogGroup(client, functionName, nbRetry + 1); } } }; -const createLogGroup = async (client, functionName) => { +const createLogGroup = async (client, functionName, nbRetry) => { + if (nbRetry > MAX_RETRY) { + throw "max retries exceeded"; + } const params = { logGroupName: `/aws/lambda/${functionName}`, }; @@ -199,12 +207,15 @@ const createLogGroup = async (client, functionName) => { await client.send(command); console.log(`log group ${params.logGroupName} created`); } catch (e) { - console.error(e); - throw e; + await delay(RETRY_DELAY); + await createLogGroup(client, functionName, nbRetry + 1); } }; -const createSubscriptionFilter = async (client, functionName) => { +const createSubscriptionFilter = async (client, functionName, nbRetry) => { + if (nbRetry > MAX_RETRY) { + throw "max retries exceeded"; + } const params = { destinationArn: LOG_PROCESSOR_ARN, filterName: `report-log-from-${functionName}`, @@ -216,13 +227,13 @@ const createSubscriptionFilter = async (client, functionName) => { await client.send(command); console.log(`subscription ${params.filterName} created`); } catch (e) { - console.error(e); - throw e; + await delay(RETRY_DELAY); + await createSubscriptionFilter(client, functionName, nbRetry + 1); } }; const addPermission = async (client, functionName, nbRetry) => { - if (nbRetry > 5) { + if (nbRetry > MAX_RETRY) { throw "max retries exceeded"; } const params = { @@ -237,7 +248,7 @@ const addPermission = async (client, functionName, nbRetry) => { console.log(`permission added to ${functionName}`); } catch (e) { console.error(e); - await delay(2000); + await delay(RETRY_DELAY); await addPermission(client, functionName, nbRetry + 1); } }; @@ -270,9 +281,9 @@ const deploy = async ( snapStart, 0 ); - await deleteLogGroup(cloudWatchLogsClient, functionName); - await createLogGroup(cloudWatchLogsClient, functionName); - await createSubscriptionFilter(cloudWatchLogsClient, functionName); + await deleteLogGroup(cloudWatchLogsClient, functionName, 0); + await createLogGroup(cloudWatchLogsClient, functionName, 0); + await createSubscriptionFilter(cloudWatchLogsClient, functionName, 0); } catch (e) { throw e; } diff --git a/report-log-processor/app.js b/report-log-processor/app.js index 84a7a5f6f2..be42b28b99 100644 --- a/report-log-processor/app.js +++ b/report-log-processor/app.js @@ -24,7 +24,7 @@ exports.handler = async (input, context) => { const architecture = tokens[2]; console.log("name", name); console.log("architecture", architecture); - const filter = runtimes.filter((e) => e.path === name); + const filter = runtimes.filter((e) => (e.slug ? e.slug : e.path) === name); if (filter.length !== 1) { // could not find the display name console.error(`filter mismatch (${filter})`); From 9cc97457c4a54de09714864cbb5ba17e596b942a Mon Sep 17 00:00:00 2001 From: Maxime David Date: Wed, 17 May 2023 17:54:52 -0400 Subject: [PATCH 27/41] add env --- function-deployer/app.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/function-deployer/app.js b/function-deployer/app.js index a6a3e00619..24ef9a94dd 100644 --- a/function-deployer/app.js +++ b/function-deployer/app.js @@ -19,7 +19,7 @@ const ROLE_ARN = process.env.ROLE_ARN; const LOG_PROCESSOR_ARN = process.env.LOG_PROCESSOR_ARN; const PROJECT = "lambda-perf"; const MAX_RETRY = 10; -const RETRY_DELAY = 20000; +const RETRY_DELAY = 10000; const delay = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); From 231d88eddf2cb22f3cea4f12d3c3671fc7ac18fc Mon Sep 17 00:00:00 2001 From: Maxime David Date: Thu, 18 May 2023 08:18:10 -0400 Subject: [PATCH 28/41] update delay --- function-deployer/app.js | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/function-deployer/app.js b/function-deployer/app.js index 24ef9a94dd..0ac2d574c7 100644 --- a/function-deployer/app.js +++ b/function-deployer/app.js @@ -19,7 +19,8 @@ const ROLE_ARN = process.env.ROLE_ARN; const LOG_PROCESSOR_ARN = process.env.LOG_PROCESSOR_ARN; const PROJECT = "lambda-perf"; const MAX_RETRY = 10; -const RETRY_DELAY = 10000; +const SHORT_DELAY = 5000; +const RETRY_DELAY = 15000; const delay = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); @@ -46,11 +47,11 @@ const deleteFunction = async (client, functionName, nbRetry) => { } }; -async function updateFunction(client, functionName) { +async function updateFunction(client, functionName, environment) { const params = { FunctionName: functionName, Environment: { - Variables: { coldStart: `${Math.random()}` }, + Variables: { coldStart: `${Math.random()}`, ...environment }, }, }; try { @@ -99,18 +100,6 @@ const createFunction = async ( ); const command = new CreateFunctionCommand(params); await client.send(command); - - if (snapStart) { - await waitForActive(client, functionName); - - //publish 10 versions - for (let i = 0; i < 10; i++) { - //update variables for publishing new version - await updateFunction(client, functionName); - await delay(RETRY_DELAY); - await publishVersion(client, functionName, 0); - } - } } catch (e) { console.error(e); await delay(RETRY_DELAY); @@ -135,7 +124,7 @@ const waitForActive = async (client, functionName) => { console.log(`waiting for function ${functionName} to be active`, func); while (func.Configuration.State !== "Active") { console.log(`waiting for function ${functionName} to be active`); - await delay(RETRY_DELAY); + await delay(SHORT_DELAY); func = await getFunction(client, functionName); } }; @@ -153,7 +142,7 @@ const getFunction = async (client, functionName) => { } }; const publishVersion = async (client, functionName, nbRetry) => { - if (nbRetry > MAX_RETRY) { + if (nbRetry > 2 * MAX_RETRY) { throw "max retries exceeded"; } const params = { @@ -281,6 +270,17 @@ const deploy = async ( snapStart, 0 ); + if (snapStart) { + await waitForActive(client, functionName); + + //publish 10 versions + for (let i = 0; i < 10; i++) { + //update variables for publishing new version + await updateFunction(client, functionName, environment); + await delay(SHORT_DELAY); + await publishVersion(client, functionName, 0); + } + } await deleteLogGroup(cloudWatchLogsClient, functionName, 0); await createLogGroup(cloudWatchLogsClient, functionName, 0); await createSubscriptionFilter(cloudWatchLogsClient, functionName, 0); From 66a0b7927e21aadbd624e781e8da45e52525ee18 Mon Sep 17 00:00:00 2001 From: Maxime David Date: Thu, 18 May 2023 08:24:10 -0400 Subject: [PATCH 29/41] update delay --- function-deployer/app.js | 4 ++-- function-trigger-deployer/app.js | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/function-deployer/app.js b/function-deployer/app.js index 0ac2d574c7..d7f1f4c042 100644 --- a/function-deployer/app.js +++ b/function-deployer/app.js @@ -18,9 +18,9 @@ const REGION = process.env.AWS_REGION; const ROLE_ARN = process.env.ROLE_ARN; const LOG_PROCESSOR_ARN = process.env.LOG_PROCESSOR_ARN; const PROJECT = "lambda-perf"; -const MAX_RETRY = 10; +const MAX_RETRY = 20; const SHORT_DELAY = 5000; -const RETRY_DELAY = 15000; +const RETRY_DELAY = 30000; const delay = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); diff --git a/function-trigger-deployer/app.js b/function-trigger-deployer/app.js index 752c82c5ff..7d0b4352c0 100644 --- a/function-trigger-deployer/app.js +++ b/function-trigger-deployer/app.js @@ -27,6 +27,7 @@ const invokeFunction = async ( const params = { FunctionName: DEPLOYER, ClientContext: Buffer.from(clientContext).toString("base64"), + InvocationType: "Event", }; try { const command = new InvokeCommand(params); From a1be0474e4e80aa23cd52ebf4fcd1d6dcb806e2f Mon Sep 17 00:00:00 2001 From: Maxime David Date: Thu, 18 May 2023 08:35:46 -0400 Subject: [PATCH 30/41] update delay --- function-deployer/app.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/function-deployer/app.js b/function-deployer/app.js index d7f1f4c042..2546014141 100644 --- a/function-deployer/app.js +++ b/function-deployer/app.js @@ -289,8 +289,10 @@ const deploy = async ( } }; -exports.handler = async (_, context) => { +exports.handler = async (event, context) => { try { + console.log("event = ", event); + console.log(JSON.stringify(event)); console.log("clientContext = ", context.clientContext); const { memorySize, From 72ebc37aa4469283c27cda66d0ac9fced0344379 Mon Sep 17 00:00:00 2001 From: Maxime David Date: Thu, 18 May 2023 08:39:34 -0400 Subject: [PATCH 31/41] update delay --- function-deployer/app.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/function-deployer/app.js b/function-deployer/app.js index 2546014141..6c624ae739 100644 --- a/function-deployer/app.js +++ b/function-deployer/app.js @@ -293,6 +293,8 @@ exports.handler = async (event, context) => { try { console.log("event = ", event); console.log(JSON.stringify(event)); + console.log("context = ", context); + console.log(JSON.stringify(context)); console.log("clientContext = ", context.clientContext); const { memorySize, From e6ba67fec6d9bf6e1099d72ff7087f7fec9f3b24 Mon Sep 17 00:00:00 2001 From: Maxime David Date: Thu, 18 May 2023 08:46:21 -0400 Subject: [PATCH 32/41] update delay --- function-trigger-deployer/app.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/function-trigger-deployer/app.js b/function-trigger-deployer/app.js index 7d0b4352c0..4269d15492 100644 --- a/function-trigger-deployer/app.js +++ b/function-trigger-deployer/app.js @@ -24,10 +24,12 @@ const invokeFunction = async ( environment, snapStart, }); + const buffer = Buffer.from(clientContext).toString("base64"); + let enc = new TextEncoder(); // always utf-8 const params = { FunctionName: DEPLOYER, - ClientContext: Buffer.from(clientContext).toString("base64"), InvocationType: "Event", + Payload: enc.encode(buffer), }; try { const command = new InvokeCommand(params); From 183a310ef2e8d4986e73ba5a7e64efe6a9c2d3fb Mon Sep 17 00:00:00 2001 From: Maxime David Date: Thu, 18 May 2023 08:51:17 -0400 Subject: [PATCH 33/41] update delay --- function-trigger-deployer/app.js | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/function-trigger-deployer/app.js b/function-trigger-deployer/app.js index 4269d15492..716d5e57ee 100644 --- a/function-trigger-deployer/app.js +++ b/function-trigger-deployer/app.js @@ -14,7 +14,7 @@ const invokeFunction = async ( environment, snapStart ) => { - const clientContext = JSON.stringify({ + const clientContext = { path, slug, handler, @@ -23,13 +23,11 @@ const invokeFunction = async ( runtime, environment, snapStart, - }); - const buffer = Buffer.from(clientContext).toString("base64"); - let enc = new TextEncoder(); // always utf-8 + }; const params = { FunctionName: DEPLOYER, InvocationType: "Event", - Payload: enc.encode(buffer), + Payload: clientContext, }; try { const command = new InvokeCommand(params); From babd690fc8b79a590bd4b5c91a2e304bc84ea8db Mon Sep 17 00:00:00 2001 From: Maxime David Date: Thu, 18 May 2023 08:56:13 -0400 Subject: [PATCH 34/41] update delay --- function-trigger-deployer/app.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/function-trigger-deployer/app.js b/function-trigger-deployer/app.js index 716d5e57ee..d510fd4248 100644 --- a/function-trigger-deployer/app.js +++ b/function-trigger-deployer/app.js @@ -27,7 +27,7 @@ const invokeFunction = async ( const params = { FunctionName: DEPLOYER, InvocationType: "Event", - Payload: clientContext, + Payload: Buffer.from(clientContext), }; try { const command = new InvokeCommand(params); From 8ba76d2ec1ae87858c2bdd95c2a5bb45592fa318 Mon Sep 17 00:00:00 2001 From: Maxime David Date: Thu, 18 May 2023 09:01:23 -0400 Subject: [PATCH 35/41] update delay --- function-trigger-deployer/app.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/function-trigger-deployer/app.js b/function-trigger-deployer/app.js index d510fd4248..a3dc3c2a20 100644 --- a/function-trigger-deployer/app.js +++ b/function-trigger-deployer/app.js @@ -27,7 +27,7 @@ const invokeFunction = async ( const params = { FunctionName: DEPLOYER, InvocationType: "Event", - Payload: Buffer.from(clientContext), + Payload: Buffer.from(JSON.stringify(clientContext)), }; try { const command = new InvokeCommand(params); From fc859ab6cb53a40637f0d97f2f03304fe53cef01 Mon Sep 17 00:00:00 2001 From: Maxime David Date: Thu, 18 May 2023 09:04:58 -0400 Subject: [PATCH 36/41] update delay --- function-deployer/app.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/function-deployer/app.js b/function-deployer/app.js index 6c624ae739..71e0908ffc 100644 --- a/function-deployer/app.js +++ b/function-deployer/app.js @@ -305,7 +305,7 @@ exports.handler = async (event, context) => { runtime, environment, snapStart, - } = context.clientContext; + } = event; const lambdaClient = new LambdaClient({ region: REGION }); const cloudWatchLogsClient = new CloudWatchLogsClient({ region: REGION, From 17e137e810709c3004453aeff6b63147458904bc Mon Sep 17 00:00:00 2001 From: Maxime David Date: Thu, 18 May 2023 10:02:12 -0400 Subject: [PATCH 37/41] update delay --- function-deployer/app.js | 6 +----- function-trigger-deployer/app.js | 7 +++---- 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/function-deployer/app.js b/function-deployer/app.js index 71e0908ffc..c09b0244ae 100644 --- a/function-deployer/app.js +++ b/function-deployer/app.js @@ -291,10 +291,6 @@ const deploy = async ( exports.handler = async (event, context) => { try { - console.log("event = ", event); - console.log(JSON.stringify(event)); - console.log("context = ", context); - console.log(JSON.stringify(context)); console.log("clientContext = ", context.clientContext); const { memorySize, @@ -305,7 +301,7 @@ exports.handler = async (event, context) => { runtime, environment, snapStart, - } = event; + } = context.clientContext; const lambdaClient = new LambdaClient({ region: REGION }); const cloudWatchLogsClient = new CloudWatchLogsClient({ region: REGION, diff --git a/function-trigger-deployer/app.js b/function-trigger-deployer/app.js index a3dc3c2a20..752c82c5ff 100644 --- a/function-trigger-deployer/app.js +++ b/function-trigger-deployer/app.js @@ -14,7 +14,7 @@ const invokeFunction = async ( environment, snapStart ) => { - const clientContext = { + const clientContext = JSON.stringify({ path, slug, handler, @@ -23,11 +23,10 @@ const invokeFunction = async ( runtime, environment, snapStart, - }; + }); const params = { FunctionName: DEPLOYER, - InvocationType: "Event", - Payload: Buffer.from(JSON.stringify(clientContext)), + ClientContext: Buffer.from(clientContext).toString("base64"), }; try { const command = new InvokeCommand(params); From 5208d4068d30cf1b9a8a532aa2a1c45d81c27c60 Mon Sep 17 00:00:00 2001 From: Maxime David Date: Thu, 18 May 2023 10:34:55 -0400 Subject: [PATCH 38/41] update delay --- function-deployer/app.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/function-deployer/app.js b/function-deployer/app.js index c09b0244ae..b885a2e149 100644 --- a/function-deployer/app.js +++ b/function-deployer/app.js @@ -271,14 +271,14 @@ const deploy = async ( 0 ); if (snapStart) { - await waitForActive(client, functionName); + await waitForActive(lambdaClient, functionName); //publish 10 versions for (let i = 0; i < 10; i++) { //update variables for publishing new version - await updateFunction(client, functionName, environment); + await updateFunction(lambdaClient, functionName, environment); await delay(SHORT_DELAY); - await publishVersion(client, functionName, 0); + await publishVersion(lambdaClient, functionName, 0); } } await deleteLogGroup(cloudWatchLogsClient, functionName, 0); From 8e6fad25f54d23dd20434f8f8d7f67ff29897426 Mon Sep 17 00:00:00 2001 From: Maxime David Date: Thu, 18 May 2023 10:37:37 -0400 Subject: [PATCH 39/41] update delay --- function-deployer/app.js | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/function-deployer/app.js b/function-deployer/app.js index b885a2e149..0eda20c57d 100644 --- a/function-deployer/app.js +++ b/function-deployer/app.js @@ -19,8 +19,13 @@ const ROLE_ARN = process.env.ROLE_ARN; const LOG_PROCESSOR_ARN = process.env.LOG_PROCESSOR_ARN; const PROJECT = "lambda-perf"; const MAX_RETRY = 20; -const SHORT_DELAY = 5000; -const RETRY_DELAY = 30000; + +const randomIntFromInterval = (min, max) => { + return Math.floor(Math.random() * (max - min + 1) + min); +}; + +const SHORT_DELAY = randomIntFromInterval(3000, 5000); +const RETRY_DELAY = randomIntFromInterval(20000, 30000); const delay = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); From 1590fce21af27068d9f2c4d980fd1910240b86a0 Mon Sep 17 00:00:00 2001 From: Maxime David Date: Thu, 18 May 2023 10:56:37 -0400 Subject: [PATCH 40/41] update delay --- function-deployer/app.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/function-deployer/app.js b/function-deployer/app.js index 0eda20c57d..bb204abf5a 100644 --- a/function-deployer/app.js +++ b/function-deployer/app.js @@ -18,7 +18,7 @@ const REGION = process.env.AWS_REGION; const ROLE_ARN = process.env.ROLE_ARN; const LOG_PROCESSOR_ARN = process.env.LOG_PROCESSOR_ARN; const PROJECT = "lambda-perf"; -const MAX_RETRY = 20; +const MAX_RETRY = 5; const randomIntFromInterval = (min, max) => { return Math.floor(Math.random() * (max - min + 1) + min); @@ -147,7 +147,7 @@ const getFunction = async (client, functionName) => { } }; const publishVersion = async (client, functionName, nbRetry) => { - if (nbRetry > 2 * MAX_RETRY) { + if (nbRetry > MAX_RETRY) { throw "max retries exceeded"; } const params = { From 3cae6c9574ddd583fef97965cb012f67c37e9736 Mon Sep 17 00:00:00 2001 From: Maxime David Date: Thu, 18 May 2023 14:09:05 -0400 Subject: [PATCH 41/41] use event --- function-deployer/app.js | 8 ++++---- function-trigger-deployer/app.js | 7 ++++--- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/function-deployer/app.js b/function-deployer/app.js index bb204abf5a..cff4e8d1d5 100644 --- a/function-deployer/app.js +++ b/function-deployer/app.js @@ -18,14 +18,14 @@ const REGION = process.env.AWS_REGION; const ROLE_ARN = process.env.ROLE_ARN; const LOG_PROCESSOR_ARN = process.env.LOG_PROCESSOR_ARN; const PROJECT = "lambda-perf"; -const MAX_RETRY = 5; +const MAX_RETRY = 20; const randomIntFromInterval = (min, max) => { return Math.floor(Math.random() * (max - min + 1) + min); }; const SHORT_DELAY = randomIntFromInterval(3000, 5000); -const RETRY_DELAY = randomIntFromInterval(20000, 30000); +const RETRY_DELAY = randomIntFromInterval(25000, 30000); const delay = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); @@ -296,7 +296,7 @@ const deploy = async ( exports.handler = async (event, context) => { try { - console.log("clientContext = ", context.clientContext); + console.log("event = ", event); const { memorySize, architecture, @@ -306,7 +306,7 @@ exports.handler = async (event, context) => { runtime, environment, snapStart, - } = context.clientContext; + } = event; const lambdaClient = new LambdaClient({ region: REGION }); const cloudWatchLogsClient = new CloudWatchLogsClient({ region: REGION, diff --git a/function-trigger-deployer/app.js b/function-trigger-deployer/app.js index 752c82c5ff..089d15070b 100644 --- a/function-trigger-deployer/app.js +++ b/function-trigger-deployer/app.js @@ -14,7 +14,7 @@ const invokeFunction = async ( environment, snapStart ) => { - const clientContext = JSON.stringify({ + const payload = JSON.stringify({ path, slug, handler, @@ -26,13 +26,14 @@ const invokeFunction = async ( }); const params = { FunctionName: DEPLOYER, - ClientContext: Buffer.from(clientContext).toString("base64"), + InvocationType: "Event", + Payload: Buffer.from(payload), }; try { const command = new InvokeCommand(params); await client.send(command); console.log( - `function ${params.FunctionName} invoked, client context = ${clientContext}` + `function ${params.FunctionName} invoked, payload = ${payload}` ); } catch (e) { console.error(e);