Autotune memory

Public runners have ~10G of ram available.

XL runners have >50G of ram available.

It's nice to be able to run tests on public
runners.

Introduce an action that:
* (Linux) frees some memory
* (Linux) adds some swap
* (Linux and macOS) calculates available memory
* (Windows) suggests 10G of available memory

Author: jsoref

.github/actions/reasonable-memory/action.yml

@@ -0,0 +1,84 @@
+name: Make more memory available
+description: Stops services and adds swap
+
+inputs:
+  max:
+    description: Maximum memory to suggest using
+    default: 50000
+
+outputs:
+  available:
+    description: Available memory
+    value: ${{ steps.memory.outputs.available }}
+  suggested:
+    description: Memory suggested for use (constrained by max and available)
+    value: ${{ steps.memory.outputs.suggested }}
+
+runs:
+  using: composite
+  steps:
+    - name: Stop services
+      if: env.RUNNER_OS == 'Linux'
+      shell: bash
+      run: |
+        sudo systemctl disable php8.1-fpm mono-xsp4 walinuxagent multipathd walinuxagent chrony cron getty@tty1 networkd-dispatcher rsyslog serial-getty@ttyS0 snapd multipathd.socket snapd.socket
+        sudo systemctl stop php8.1-fpm mono-xsp4 walinuxagent multipathd walinuxagent chrony cron getty@tty1 networkd-dispatcher rsyslog serial-getty@ttyS0 snapd
+        sudo killall mono
+
+    - name: enable swap
+      if: env.RUNNER_OS == 'Linux'
+      shell: bash
+      run: |
+        sudo fallocate -l 10G /mnt/big-swapfile
+        sudo chmod 600 /mnt/big-swapfile
+        sudo mkswap /mnt/big-swapfile
+        sudo swapon /mnt/big-swapfile
+
+    - name: Report Available Memory (Linux)
+      id: memory-linux
+      if: runner.os == 'Linux'
+      shell: bash
+      run: |
+        free |
+        perl -ne '
+          next unless /Mem:.*\s(\d+)$/;
+          my $m = int($1 / 102400)*100;
+          print "available=$m\n";
+        ' >> "$GITHUB_OUTPUT"
+
+    - name: Report Available Memory (macOS)
+      id: memory-mac
+      if: runner.os == 'macOS'
+      shell: bash
+      run: |
+        sysctl -a |
+        perl -e '
+          my $m;
+          while (<>) {
+            next unless /.*memsize.*:\s*(\d+)/;
+            my $m0 = int($1 >> 20);
+            $m = $m0 unless ($m && $m > $m0);
+          }
+          print "available=$m\n";
+        ' >> "$GITHUB_OUTPUT"
+
+    - name: Report Available Memory (Windows)
+      id: memory-windows
+      if: runner.os == 'Windows'
+      shell: bash
+      run: |
+        echo "available=10240" >> "$GITHUB_OUTPUT"
+
+    - name: Report Memory
+      id: memory
+      shell: bash
+      env:
+        max: ${{ inputs.max }}
+        available: ${{ steps.memory-linux.outputs.available || steps.memory-mac.outputs.available  || steps.memory-windows.outputs.available }}
+      run: |
+        perl -e '
+          my $available = $ENV{available};
+          my $max = $ENV{max} || $available;
+          my $s = ($m < $max ? $available : $max);
+          print "suggested=$s\n"."available=$available\n";
+        ' >> "$GITHUB_OUTPUT"

.github/workflows/csharp-qltest.yml

@@ -5,7 +5,11 @@ on:
     paths:
       - "csharp/**"
       - "shared/**"
+      - .github/workflows/csharp-qltest.yml
       - .github/actions/fetch-codeql/action.yml
+      - .github/actions/cache-query-compilation/action.yml
+      - .github/actions/reasonable-memory/action.yml
+      - csharp/actions/create-extractor-pack/action.yml
       - codeql-workspace.yml
     branches:
       - main
@@ -16,6 +20,9 @@ on:
       - "shared/**"
       - .github/workflows/csharp-qltest.yml
       - .github/actions/fetch-codeql/action.yml
+      - .github/actions/cache-query-compilation/action.yml
+      - .github/actions/reasonable-memory/action.yml
+      - csharp/actions/create-extractor-pack/action.yml
       - codeql-workspace.yml
     branches:
       - main
@@ -53,6 +60,9 @@ jobs:
         slice: ["1/2", "2/2"]
     steps:
       - uses: actions/checkout@v4
+      - name: Allocate memory
+        id: memory
+        uses: ./.github/actions/reasonable-memory
       - uses: ./csharp/actions/create-extractor-pack
       - name: Cache compilation cache
         id: query-cache
@@ -61,9 +71,10 @@ jobs:
           key: csharp-qltest-${{ matrix.slice }}
       - name: Run QL tests
         run: |
-          codeql test run --threads=0 --ram 50000 --slice ${{ matrix.slice }} --search-path extractor-pack --check-databases --check-undefined-labels --check-repeated-labels --check-redefined-labels --consistency-queries ql/consistency-queries ql/test --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}"
+          codeql test run --threads=0 --ram $memory --slice ${{ matrix.slice }} --search-path extractor-pack --check-databases --check-undefined-labels --check-repeated-labels --check-redefined-labels --consistency-queries ql/consistency-queries ql/test --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}"
         env:
           GITHUB_TOKEN: ${{ github.token }}
+          memory: ${{ steps.memory.outputs.suggested }}
   unit-tests:
     strategy:
       matrix:

.github/workflows/ruby-qltest.yml

@@ -5,8 +5,11 @@ on:
     paths:
       - "ruby/**"
       - "shared/**"
-      - .github/workflows/ruby-build.yml
+      - .github/workflows/ruby-qltest.yml
       - .github/actions/fetch-codeql/action.yml
+      - .github/actions/cache-query-compilation/action.yml
+      - .github/actions/reasonable-memory/action.yml
+      - csharp/actions/create-extractor-pack/action.yml
       - codeql-workspace.yml
     branches:
       - main
@@ -17,10 +20,14 @@ on:
       - "shared/**"
       - .github/workflows/ruby-qltest.yml
       - .github/actions/fetch-codeql/action.yml
+      - .github/actions/cache-query-compilation/action.yml
+      - .github/actions/reasonable-memory/action.yml
+      - csharp/actions/create-extractor-pack/action.yml
       - codeql-workspace.yml
     branches:
       - main
       - "rc/*"
+  workflow_dispatch:
 
 env:
   CARGO_TERM_COLOR: always
@@ -55,6 +62,9 @@ jobs:
       fail-fast: false
     steps:
       - uses: actions/checkout@v4
+      - name: Allocate memory
+        id: memory
+        uses: ./.github/actions/reasonable-memory
       - uses: ./.github/actions/fetch-codeql
       - uses: ./ruby/actions/create-extractor-pack
       - name: Cache compilation cache
@@ -64,6 +74,7 @@ jobs:
           key: ruby-qltest
       - name: Run QL tests
         run: |
-          codeql test run --threads=0 --ram 50000 --search-path "${{ github.workspace }}/ruby/extractor-pack" --check-databases --check-undefined-labels --check-unused-labels --check-repeated-labels --check-redefined-labels --check-use-before-definition --consistency-queries ql/consistency-queries ql/test --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}"
+          codeql test run --threads=0 --ram $memory --search-path "${{ github.workspace }}/ruby/extractor-pack" --check-databases --check-undefined-labels --check-unused-labels --check-repeated-labels --check-redefined-labels --check-use-before-definition --consistency-queries ql/consistency-queries ql/test --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}"
         env:
           GITHUB_TOKEN: ${{ github.token }}
+          memory: ${{ steps.memory.outputs.suggested }}

.github/workflows/swift.yml

@@ -32,6 +32,7 @@ on:
       - main
       - rc/*
       - codeql-cli-*
+  workflow_dispatch:
 
 jobs:
   # not using a matrix as you cannot depend on a specific job in a matrix, and we want to start linux checks

swift/actions/run-ql-tests/action.yml

@@ -8,6 +8,9 @@ inputs:
 runs:
   using: composite
   steps:
+    - name: Allocate memory
+      id: memory
+      uses: ./.github/actions/reasonable-memory
     - uses: ./swift/actions/share-extractor-pack
     - uses: ./.github/actions/fetch-codeql
     - id: query-cache
@@ -19,7 +22,7 @@ runs:
       run: |
         codeql test run \
           --threads=0 \
-          --ram 50000 \
+          --ram $memory \
           --search-path "${{ github.workspace }}/swift/extractor-pack" \
           --check-databases \
           --check-unused-labels \
@@ -32,3 +35,4 @@ runs:
           swift/ql/test
       env:
         GITHUB_TOKEN: ${{ github.token }}
+        memory: ${{ steps.memory.outputs.suggested }}