generate a fuzzing corpus by extracting code from testrunner

Author: firewave

.github/workflows/corpus.yml

@@ -0,0 +1,59 @@
+# Syntax reference https://help.github.com/en/actions/reference/workflow-syntax-for-github-actions
+# Environment reference https://help.github.com/en/actions/reference/virtual-environments-for-github-hosted-runners
+name: corpus
+
+on:
+  schedule:
+    - cron: '0 0 * * 0'
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  corpus:
+    runs-on: ubuntu-22.04
+    if: ${{ github.repository_owner == 'danmar' }}
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          persist-credentials: false
+
+      - name: ccache
+        uses: hendrikmuhs/ccache-action@v1.2
+        with:
+          key: ${{ github.workflow }}-${{ runner.os }}
+
+      - name: Install missing software on ubuntu
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y fdupes
+
+      - name: build testrunner
+        run: |
+          store_dir=$(pwd)/store
+          mkdir $store_dir
+          export PATH="/usr/lib/ccache:/usr/local/opt/ccache/libexec:$PATH"
+          make -s -j$(nproc) CXXFLAGS="-w -DSTORE_INPUT_DIR=\"\\\"$store_dir\\\"\"" testrunner
+
+      - name: run testrunner
+        run: |
+          ./testrunner -q
+
+      - name: de-duplicate files
+        run: |
+          ls -l ./store | wc -l
+          echo "removing duplicates"
+          fdupes -qdN ./store > /dev/null
+          ls -l ./store | wc -l
+          # print biggest size
+          ls -l ./store | cut -d' ' -f5 | sort -u -n -r | head -n1
+
+      # TODO: dedup with fuzzer?
+
+      - uses: actions/upload-artifact@v4
+        if: success()
+        with:
+          name: corpus
+          path: ./store

lib/tokenlist.cpp

@@ -328,8 +328,30 @@ bool TokenList::createTokens(std::istream &code)
 
 //---------------------------------------------------------------------------
 
+#ifdef STORE_INPUT_DIR
+#include <atomic>
+#include <fstream>
+#include <sstream>
+
+static void storeInput(std::istream &code)
+{
+    static std::atomic_uint64_t num(0);
+    {
+        std::ostringstream oss;
+        oss << code.rdbuf();
+        code.seekg(0);
+        std::ofstream out(STORE_INPUT_DIR "/" + std::to_string(num++));
+        out << oss.str();
+    }
+}
+#endif
+
 bool TokenList::createTokensInternal(std::istream &code, const std::string& file0)
 {
+#ifdef STORE_INPUT_DIR
+    storeInput(code);
+#endif
+
     simplecpp::OutputList outputList;
     simplecpp::TokenList tokens(code, mFiles, file0, &outputList);