feat(scripts): add automated ODD memory usage analysis

.github/workflows/odd-memory-usage-test.yaml

@@ -0,0 +1,43 @@
+name: 'ODD Memory Usage Test'
+
+on:
+  schedule:
+    - cron: '30 12 * * *'
+  workflow_dispatch:
+
+jobs:
+  analyze-memory:
+    name: 'ODD Memory Usage Test'
+    runs-on: 'ubuntu-latest'
+    timeout-minutes: 15
+
+    steps:
+      - name: Checkout repository
+        uses: 'actions/checkout@v4'
+
+      - name: Setup Node.js
+        uses: 'actions/setup-node@v3'
+        with:
+          node-version: '18.19.0'
+          cache: 'yarn'
+
+      - name: Install dependencies
+        run: |
+          yarn install
+
+      - name: Run memory analysis
+        env:
+          MIXPANEL_INGEST_USER: ${{ secrets.MIXPANEL_INGEST_USER }}
+          MIXPANEL_INGEST_SECRET: ${{ secrets.MIXPANEL_INGEST_SECRET }}
+          MIXPANEL_PROJECT_ID: ${{ secrets.OT_APP_MIXPANEL_ID }}
+        run: |
+          OUTPUT=$(node ./scripts/resource-monitor/perform-memory-analysis "$MIXPANEL_INGEST_USER" "$MIXPANEL_INGEST_SECRET" "$MIXPANEL_PROJECT_ID")
+
+          echo "::group::ODD Memory Usage Results"
+          echo "$OUTPUT"
+          echo "::endgroup::"
+
+          echo "## ODD Memory Usage Results" >> $GITHUB_STEP_SUMMARY
+          echo '```' >> $GITHUB_STEP_SUMMARY
+          echo "$OUTPUT" >> $GITHUB_STEP_SUMMARY
+          echo '```' >> $GITHUB_STEP_SUMMARY

scripts/resource-monitor/lib/analysis.js

@@ -0,0 +1,84 @@
+// Analysis is based on one-tailed, Pearson's correlation coefficient.
+
+const { MINIMUM_VALID_SAMPLE_SIZE } = require('./constants')
+
+/**
+ *
+ * @param x An array of a numbers.
+ * @param y An array of numbers.
+ * @return {number} The Pearson Correlation.
+ */
+function calculatePearsonCorrelation(x, y) {
+  const n = x.length
+  let sum_x = 0
+  let sum_y = 0
+  let sum_xy = 0
+  let sum_x2 = 0
+  let sum_y2 = 0
+
+  for (let i = 0; i < n; i++) {
+    sum_x += x[i]
+    sum_y += y[i]
+    sum_xy += x[i] * y[i]
+    sum_x2 += x[i] * x[i]
+    sum_y2 += y[i] * y[i]
+  }
+
+  const numerator = n * sum_xy - sum_x * sum_y
+  const denominator = Math.sqrt(
+    (n * sum_x2 - sum_x * sum_x) * (n * sum_y2 - sum_y * sum_y)
+  )
+
+  return denominator === 0 ? 0 : numerator / denominator
+}
+
+/**
+ * @description Calculate p-value using t-distribution approximation for a one-tailed test.
+ * If there are too few samples, assume no correlation.
+ * For positive correlations only.
+ * @param correlation The Pearson Correlation
+ * @param sampleSize The total number of samples.
+ * @return {number} The p-value.
+ */
+function calculatePValueOneTailed(correlation, sampleSize) {
+  if (sampleSize < MINIMUM_VALID_SAMPLE_SIZE) {
+    return 1
+  }
+
+  // The t-statistic
+  const t =
+    correlation * Math.sqrt((sampleSize - 2) / (1 - correlation * correlation))
+
+  // Approximate p-value using t-distribution (one-tailed test)
+  const degreesOfFreedom = sampleSize - 2
+  return 1 - tDistributionCDF(t, degreesOfFreedom)
+}
+
+// t-distribution CDF approximation
+function tDistributionCDF(t, df) {
+  const x = df / (df + t * t)
+  return 1 - 0.5 * Math.pow(x, df / 2)
+}
+
+function interpretResults(result) {
+  if (!result.isSignificant) {
+    return 'No significant correlation found'
+  }
+
+  const strength = Math.abs(result.correlation)
+  const direction = result.correlation > 0 ? 'positive' : 'negative'
+
+  if (strength > 0.7) {
+    return `Strong ${direction} correlation (>0.7)`
+  } else if (strength > 0.3) {
+    return `Moderate ${direction} correlation (>0.3 and <0.7)`
+  } else {
+    return `Weak ${direction} correlation (>=0.3)`
+  }
+}
+
+module.exports = {
+  calculatePearsonCorrelation,
+  calculatePValueOneTailed,
+  interpretResults,
+}

scripts/resource-monitor/lib/constants.js

@@ -0,0 +1,41 @@
+/**
+ * @description Several processes we care about execute with a lot of unique sub args determined at
+ * runtime. These processes are aggregated using a regex pattern.
+ */
+const AGGREGATED_PROCESSES = [
+  {
+    pattern: /^\/opt\/opentrons-app\/opentrons --type=renderer/,
+    key: 'app-renderer-processes',
+  },
+  {
+    pattern: /^\/opt\/opentrons-app\/opentrons --type=zygote/,
+    key: 'app-zygote-processes',
+  },
+  {
+    pattern: /^python3 -m uvicorn/,
+    key: 'robot-server-uvicorn-processes',
+  },
+  {
+    pattern: /^\/opt\/opentrons-app\/opentrons --type=utility/,
+    key: 'app-utility-processes',
+  },
+]
+
+/**
+ * @description Generally don't include any variation of external processes in analysis.
+ */
+const BLACKLISTED_PROCESSES = [/^nmcli/, /^\/usr\/bin\/python3/]
+
+/**
+ * @description For Pearson's, it's generally recommended to use a sample size of at least n=30.
+ */
+const MINIMUM_VALID_SAMPLE_SIZE = 30
+
+const P_VALUE_SIGNIFICANCE_THRESHOLD = 0.05
+
+module.exports = {
+  AGGREGATED_PROCESSES,
+  BLACKLISTED_PROCESSES,
+  MINIMUM_VALID_SAMPLE_SIZE,
+  P_VALUE_SIGNIFICANCE_THRESHOLD,
+}

scripts/resource-monitor/lib/helpers/date.js

@@ -0,0 +1,22 @@
+/**
+ * @description Get ISO date strings for the past month from yesterday.
+ */
+function getISODatesForPastMonth() {
+  const now = new Date()
+  // Don't use today's data, because the Mixpanel API seemingly doesn't use UTC timestamps, and
+  // it's easy to fail a request depending on the time of day it's made.
+  const yesterday = new Date(now.setDate(now.getDate() - 1))
+  const formatDate = date => date.toISOString().split('T')[0]
+
+  const monthAgo = new Date(yesterday)
+  monthAgo.setMonth(yesterday.getMonth() - 1)
+
+  return {
+    from: formatDate(monthAgo),
+    to: formatDate(yesterday),
+  }
+}
+
+module.exports = {
+  getISODatesForPastMonth,
+}

scripts/resource-monitor/lib/helpers/index.js

@@ -0,0 +1,5 @@
+module.exports = {
+  ...require('./date'),
+  ...require('./manifest'),
+  ...require('./mixpanel'),
+}

scripts/resource-monitor/lib/helpers/manifest.js

@@ -0,0 +1,52 @@
+const fetch = require('node-fetch')
+
+const APP_MANIFEST = 'https://builds.opentrons.com/app/releases.json'
+
+async function downloadAppManifest() {
+  const response = await fetch(APP_MANIFEST)
+  return await response.json()
+}
+
+/**
+ * @description Get the most recent app version that is not revoked.
+ * @param manifest The app manifest
+ */
+function latestValidVersionFromManifest(manifest) {
+  const versions = Object.keys(manifest.production)
+  const latestValidVersion = versions.findLast(
+    version => manifest.production[version].revoked === false
+  )
+
+  if (latestValidVersion != null) {
+    return latestValidVersion
+  } else {
+    throw new Error('No valid versions found')
+  }
+}
+
+/**
+ * @description Get `count` latest, previous non revoked versions relative to the latest version.
+ * @param manifest The app manifest
+ * @param latestVersion The latest valid version
+ * @param count Number of previous versions to return
+ * @returns {string[]} Array of version strings, ordered from newest to oldest
+ */
+function getPrevValidVersions(manifest, latestVersion, count) {
+  const versions = Object.keys(manifest.production)
+  const latestIndex = versions.indexOf(latestVersion)
+
+  if (latestIndex === -1) {
+    throw new Error('Latest version not found in manifest')
+  }
+
+  return versions
+    .slice(0, latestIndex)
+    .filter(version => !manifest.production[version].revoked)
+    .slice(-count)
+    .reverse()
+}
+module.exports = {
+  downloadAppManifest,
+  latestValidVersionFromManifest,
+  getPrevValidVersions,
+}

scripts/resource-monitor/lib/helpers/mixpanel.js

@@ -0,0 +1,65 @@
+const fetch = require('node-fetch')
+
+const MIXPANEL_URL = 'https://data.mixpanel.com/api/2.0/export'
+
+/**
+ * @description Base64 encode a username and password in
+ * @param uname Mixpanel service account username.
+ * @param pwd Mixpanel service account password.
+ * @return {string}
+ */
+function encodeCredentialsForMixpanel(uname, pwd) {
+  return Buffer.from(`${uname}:${pwd}`).toString('base64')
+}
+
+/**
+ * @description Cleans up Mixpanel data for post-processing.
+ * @param data Mixpanel data
+ */
+function parseMixpanelData(data) {
+  const lines = data.split('\n').filter(line => line.trim())
+  return lines.map(line => JSON.parse(line))
+}
+
+/**
+ * @description Make the network request to Mixpanel.
+ */
+async function getMixpanelResourceMonitorDataFor({
+  uname,
+  pwd,
+  projectId,
+  fromDate,
+  toDate,
+  where,
+}) {
+  const params = new URLSearchParams({
+    project_id: projectId,
+    from_date: fromDate,
+    to_date: toDate,
+    event: '["resourceMonitorReport"]',
+    where,
+  })
+
+  const options = {
+    method: 'GET',
+    headers: {
+      'Accept-Encoding': 'gzip',
+      accept: 'text/plain',
+      authorization: `Basic ${encodeCredentialsForMixpanel(uname, pwd)}`,
+    },
+  }
+
+  const response = await fetch(`${MIXPANEL_URL}?${params}`, options)
+  const text = await response.text()
+  if (!response.ok) {
+    throw new Error(
+      `Mixpanel request failed: ${response.status}, ${response.statusText}, ${text}`
+    )
+  }
+  return text
+}
+
+module.exports = {
+  getMixpanelResourceMonitorDataFor,
+  parseMixpanelData,
+}