Heap profiling

cdk/lib/dotcom-components.ts

@@ -13,6 +13,7 @@ import {
 	GuDynamoDBReadPolicy,
 	GuGetS3ObjectsPolicy,
 	GuPutCloudwatchMetricsPolicy,
+	GuPutS3ObjectsPolicy,
 } from '@guardian/cdk/lib/constructs/iam';
 import type { Alarms } from '@guardian/cdk/lib/patterns/ec2-app/base';
 import type { GuAsgCapacity } from '@guardian/cdk/lib/types';
@@ -224,6 +225,16 @@ sudo amazon-cloudwatch-agent-ctl -a fetch-config -m ec2 -c file:/opt/aws/amazon-
 					],
 				},
 			),
+			new GuPutS3ObjectsPolicy(
+				this,
+				'S3PutPolicyGuReaderRevenuePrivate',
+				{
+					bucketName: 'gu-reader-revenue-private',
+					paths: [
+						`support-dotcom-components/heapSnapshots/${this.stage}/*`,
+					],
+				},
+			),
 			new GuDynamoDBReadPolicy(this, 'DynamoReadPolicy', {
 				tableName: `super-mode-calculator-${this.stage}`,
 			}),
@@ -262,7 +273,7 @@ sudo amazon-cloudwatch-agent-ctl -a fetch-config -m ec2 -c file:/opt/aws/amazon-
 						},
 						unhealthyInstancesAlarm: true,
 						snsTopicName,
-				  }
+					}
 				: { noMonitoring: true };
 
 		const ec2App = new GuEc2App(this, {

src/server/server.ts

@@ -27,6 +27,7 @@ import { buildEpicLiveblogTestsReloader, buildEpicTestsReloader } from './tests/
 import { buildGutterLiveblogTestsReloader } from './tests/gutters/gutterTests';
 import { buildHeaderTestsReloader } from './tests/headers/headerTests';
 import { logError } from './utils/logging';
+import { Profiler } from './utils/profiling';
 
 const buildApp = async (): Promise<Express> => {
     const app = express();
@@ -137,6 +138,8 @@ const buildApp = async (): Promise<Express> => {
         res.send('OK');
     });
 
+    new Profiler();
+
     return Promise.resolve(app);
 };
 

src/server/utils/S3.ts

@@ -1,4 +1,5 @@
 import readline from 'readline';
+import type { Readable } from 'stream';
 import AWS from 'aws-sdk';
 import type { GetObjectOutput } from 'aws-sdk/clients/s3';
 import { isDev } from '../lib/env';
@@ -59,3 +60,12 @@ export const streamS3DataByLine = ({ bucket, key, onLine, onComplete }: S3Stream
         logError(`Error streaming from S3 for ${bucket}/${key}: ${error}`),
     );
 };
+
+export const streamToS3 = (bucket: string, key: string, stream: Readable) => {
+    const s3 = getS3();
+    return s3.upload({
+        Bucket: bucket,
+        Key: key,
+        Body: stream,
+    });
+};

src/server/utils/awsMetadata.ts

@@ -0,0 +1,18 @@
+import AWS from 'aws-sdk';
+import { isDev } from '../lib/env';
+
+export const getInstanceId = (): Promise<string> =>
+    new Promise((resolve, reject) => {
+        if (isDev) {
+            resolve('local');
+        }
+
+        const service = new AWS.MetadataService();
+        service.request('/latest/meta-data/instance-id', function (err, data) {
+            // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition -- this is how you check for error
+            if (err) {
+                reject(err);
+            }
+            resolve(data);
+        });
+    });

src/server/utils/profiling.ts

@@ -0,0 +1,67 @@
+import { Session } from 'node:inspector/promises';
+import { Readable } from 'stream';
+import { formatISO } from 'date-fns';
+import { getInstanceId } from './awsMetadata';
+import { logError, logInfo } from './logging';
+import { streamToS3 } from './S3';
+import { getSsmValue } from './ssm';
+
+const bucket = 'gu-reader-revenue-private';
+const stage = process.env.stage ?? 'CODE';
+const intervalInMilliseconds = 60 * 5 * 1000; // 5 mins
+
+interface ProfilerConfig {
+    enableHeapProfiling: boolean;
+}
+
+// Periodically checks config in Parameter Store to see if profiling is enabled, and if so takes a heap snapshot and writes it to S3
+export class Profiler {
+    constructor() {
+        this.run();
+    }
+
+    run() {
+        // first check if profiling is enabled
+        void getSsmValue(stage, 'profiling')
+            .then((ssmValue) => {
+                if (ssmValue) {
+                    const config = JSON.parse(ssmValue) as ProfilerConfig;
+                    if (config.enableHeapProfiling) {
+                        return this.takeHeapSnapshot();
+                    }
+                }
+            })
+            .catch((error) => {
+                logError(`Error profiling: ${error}`);
+            })
+            .finally(() => {
+                setTimeout(() => {
+                    void this.run();
+                }, intervalInMilliseconds);
+            });
+    }
+
+    // Takes a heap snapshot and streams the result to S3
+    async takeHeapSnapshot() {
+        const session = new Session();
+        const stream = new Readable();
+        const instanceId = await getInstanceId();
+        const key = `support-dotcom-components/heapSnapshots/${stage}/${instanceId}-${formatISO(new Date())}.heapsnapshot`;
+
+        const upload = streamToS3(bucket, key, stream);
+
+        session.connect();
+        session.on('HeapProfiler.addHeapSnapshotChunk', (m) => {
+            stream.push(m.params.chunk);
+        });
+
+        logInfo('Taking heap snapshot');
+        await session.post('HeapProfiler.takeHeapSnapshot');
+
+        stream.push(null); // end the stream
+        session.disconnect(); // end the profiling session
+
+        await upload.promise();
+        logInfo(`Finished uploading heap snapshot to S3 with key: ${key}`);
+    }
+}

src/server/utils/ssm.ts

@@ -7,6 +7,7 @@ export async function getSsmValue(stage: string, id: string): Promise<string | u
     const response = await client
         .getParameter({
             Name: name,
+            WithDecryption: true,
         })
         .promise();