solves issue #129

.github/workflows/benchmarks.yml

@@ -0,0 +1,250 @@
+name: Benchmarks
+
+on:
+  pull_request:
+    branches: [main]
+    types: [opened, synchronize, reopened, ready_for_review]
+  push:
+    branches: [main]
+  workflow_dispatch:
+
+# Add explicit permissions for the workflow
+permissions:
+  contents: read
+  pull-requests: write
+  actions: read
+
+jobs:
+  benchmark:
+    name: Run Performance Benchmarks
+    runs-on: ubuntu-latest
+    if: github.event.pull_request.draft == false
+
+    steps:
+    - name: Checkout Repository
+      uses: actions/checkout@v4
+
+    - name: Set up Python
+      uses: actions/setup-python@v4
+      with:
+        python-version: '3.10'
+        cache: 'pip'
+
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        python -m pip install -e ".[dev]"
+        python -m pip install perfplot matplotlib seaborn
+
+    - name: Try to set up GPU support (optional)
+      run: |
+        # Try to install CUDA dependencies for GPU-accelerated benchmarks
+        # This won't fail the workflow if it doesn't work, as most CI runners don't have GPUs
+        python -m pip install numba cupy-cuda11x || echo "GPU support not available (expected for most CI runners)"
+
+    - name: Run SugarScape Benchmark (Small Dataset)
+      run: |
+        cd examples/sugarscape_ig
+        python -c '
+        import sys
+        import time
+        from performance_comparison import SugarScapeSetup, mesa_frames_polars_numba_parallel, mesa_implementation
+
+        # Run a smaller subset for CI benchmarks (faster execution)
+        setup = SugarScapeSetup(50000)
+
+        print("Running mesa-frames implementation...")
+        start_time = time.time()
+        mf_model = mesa_frames_polars_numba_parallel(setup)
+        mf_time = time.time() - start_time
+        print(f"mesa-frames implementation completed in {mf_time:.2f} seconds")
+
+        print("Running mesa implementation...")
+        start_time = time.time()
+        mesa_model = mesa_implementation(setup)
+        mesa_time = time.time() - start_time
+        print(f"mesa implementation completed in {mesa_time:.2f} seconds")
+
+        print("Benchmark complete!")
+
+        # Save timing results for the PR comment
+        with open("sugarscape_results.txt", "w") as f:
+            f.write(f"mesa-frames: {mf_time:.2f}s\n")
+            f.write(f"mesa: {mesa_time:.2f}s\n")
+            f.write(f"speedup: {mesa_time/mf_time:.2f}x\n")
+        '
+
+    - name: Run Boltzmann Wealth Benchmark (Small Dataset)
+      run: |
+        cd examples/boltzmann_wealth
+        python -c '
+        import sys
+        import time
+        from performance_plot import mesa_frames_polars_concise, mesa_implementation
+
+        # Run a smaller subset for CI benchmarks (faster execution)
+        print("Running mesa-frames implementation...")
+        start_time = time.time()
+        mf_model = mesa_frames_polars_concise(10000)
+        mf_time = time.time() - start_time
+        print(f"mesa-frames implementation completed in {mf_time:.2f} seconds")
+
+        print("Running mesa implementation...")
+        start_time = time.time()
+        mesa_model = mesa_implementation(10000)
+        mesa_time = time.time() - start_time
+        print(f"mesa implementation completed in {mesa_time:.2f} seconds")
+
+        print("Benchmark complete!")
+
+        # Save timing results for the PR comment
+        with open("boltzmann_results.txt", "w") as f:
+            f.write(f"mesa-frames: {mf_time:.2f}s\n")
+            f.write(f"mesa: {mesa_time:.2f}s\n")
+            f.write(f"speedup: {mesa_time/mf_time:.2f}x\n")
+        '
+
+    - name: Generate Simple Benchmark Visualizations
+      run: |
+        python -c '
+        import matplotlib.pyplot as plt
+        import numpy as np
+        import os
+
+        # Function to read benchmark results
+        def read_results(filename):
+            results = {}
+            with open(filename, "r") as f:
+                for line in f:
+                    key, value = line.strip().split(": ")
+                    results[key] = value
+            return results
+
+        # Create visualization for Sugarscape benchmark
+        sugarscape_results = read_results("examples/sugarscape_ig/sugarscape_results.txt")
+        boltzmann_results = read_results("examples/boltzmann_wealth/boltzmann_results.txt")
+
+        # Create a simple bar chart comparing execution times
+        fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(12, 5))
+
+        # Sugarscape plot
+        sugarscape_mesa_time = float(sugarscape_results["mesa"].replace("s", ""))
+        sugarscape_mf_time = float(sugarscape_results["mesa-frames"].replace("s", ""))
+        ax1.bar(["mesa-frames", "mesa"], [sugarscape_mf_time, sugarscape_mesa_time])
+        ax1.set_title("SugarScape Benchmark (50k agents)")
+        ax1.set_ylabel("Execution time (s)")
+        ax1.text(0, sugarscape_mf_time/2, f"{sugarscape_mf_time:.2f}s",
+                ha="center", va="center", color="white", fontweight="bold")
+        ax1.text(1, sugarscape_mesa_time/2, f"{sugarscape_mesa_time:.2f}s",
+                ha="center", va="center", color="white", fontweight="bold")
+        ax1.text(0.5, max(sugarscape_mf_time, sugarscape_mesa_time) * 0.9,
+                f"Speedup: {sugarscape_results[\"speedup\"]}",
+                ha="center", va="center", bbox=dict(facecolor="white", alpha=0.8))
+
+        # Boltzmann plot
+        boltzmann_mesa_time = float(boltzmann_results["mesa"].replace("s", ""))
+        boltzmann_mf_time = float(boltzmann_results["mesa-frames"].replace("s", ""))
+        ax2.bar(["mesa-frames", "mesa"], [boltzmann_mf_time, boltzmann_mesa_time])
+        ax2.set_title("Boltzmann Wealth Benchmark (10k agents)")
+        ax2.set_ylabel("Execution time (s)")
+        ax2.text(0, boltzmann_mf_time/2, f"{boltzmann_mf_time:.2f}s",
+                ha="center", va="center", color="white", fontweight="bold")
+        ax2.text(1, boltzmann_mesa_time/2, f"{boltzmann_mesa_time:.2f}s",
+                ha="center", va="center", color="white", fontweight="bold")
+        ax2.text(0.5, max(boltzmann_mf_time, boltzmann_mesa_time) * 0.9,
+                f"Speedup: {boltzmann_results[\"speedup\"]}",
+                ha="center", va="center", bbox=dict(facecolor="white", alpha=0.8))
+
+        plt.tight_layout()
+        plt.savefig("benchmark_results.png", dpi=150)
+        print("Benchmark visualization saved as benchmark_results.png")
+        '
+
+    - name: Prepare Artifacts Directory
+      run: |
+        mkdir -p benchmark_artifacts
+        cp examples/sugarscape_ig/*.png benchmark_artifacts/ || true
+        cp examples/sugarscape_ig/*.txt benchmark_artifacts/ || true
+        cp examples/boltzmann_wealth/*.png benchmark_artifacts/ || true
+        cp examples/boltzmann_wealth/*.txt benchmark_artifacts/ || true
+        cp benchmark_results.png benchmark_artifacts/ || true
+
+    - name: Save Benchmark Results
+      if: always()
+      uses: actions/upload-artifact@v4
+      with:
+        name: benchmark-results
+        path: benchmark_artifacts
+
+    - name: Log Benchmark Results (Fallback)
+      if: always()
+      run: |
+        echo "===== BENCHMARK RESULTS (FALLBACK) ====="
+        echo "SugarScape Benchmark Results:"
+        if [ -f "examples/sugarscape_ig/sugarscape_results.txt" ]; then
+          cat examples/sugarscape_ig/sugarscape_results.txt
+        else
+          echo "Results file not found"
+        fi
+
+        echo "Boltzmann Wealth Benchmark Results:"
+        if [ -f "examples/boltzmann_wealth/boltzmann_results.txt" ]; then
+          cat examples/boltzmann_wealth/boltzmann_results.txt
+        else
+          echo "Results file not found"
+        fi
+
+    - name: Create Result Summary for PR Comment
+      if: github.event_name == 'pull_request'
+      id: result_summary
+      run: |
+        # Create summary file
+        echo '## 📊 Performance Benchmark Results' > summary.md
+        echo '' >> summary.md
+        echo 'The benchmarks have been executed.' >> summary.md
+        echo '' >> summary.md
+
+        echo '### SugarScape Model (50k agents, 100 steps)' >> summary.md
+        echo '```' >> summary.md
+        if [ -f "examples/sugarscape_ig/sugarscape_results.txt" ]; then
+          cat examples/sugarscape_ig/sugarscape_results.txt >> summary.md
+        else
+          echo "Results file not found" >> summary.md
+        fi
+        echo '```' >> summary.md
+        echo '' >> summary.md
+
+        echo '### Boltzmann Wealth Model (10k agents, 100 steps)' >> summary.md
+        echo '```' >> summary.md
+        if [ -f "examples/boltzmann_wealth/boltzmann_results.txt" ]; then
+          cat examples/boltzmann_wealth/boltzmann_results.txt >> summary.md
+        else
+          echo "Results file not found" >> summary.md
+        fi
+        echo '```' >> summary.md
+        echo '' >> summary.md
+
+        echo "[Click here to see full benchmark results](${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID})" >> summary.md
+        echo "Note: Benchmark results are now available as artifacts on the GitHub Actions run page." >> summary.md
+
+        # Set output
+        SUMMARY=$(cat summary.md)
+        echo "summary<<EOF" >> $GITHUB_OUTPUT
+        echo "$SUMMARY" >> $GITHUB_OUTPUT
+        echo "EOF" >> $GITHUB_OUTPUT
+
+    - name: Add Benchmark Comment
+      if: github.event_name == 'pull_request'
+      uses: actions/github-script@v6
+      with:
+        github-token: ${{ secrets.GITHUB_TOKEN }}
+        script: |
+          const fs = require('fs');
+
+          // Create a comment with benchmark results
+          github.rest.issues.createComment({
+            issue_number: context.issue.number,
+            owner: context.repo.owner,
+            repo: context.repo.repo,
+            body: `${{ steps.result_summary.outputs.summary }}`
+          });

docs/general/contributing/benchmarks.md

@@ -0,0 +1,76 @@
+# Benchmarking in mesa-frames
+
+As a library focused on performance improvements, it's crucial that mesa-frames maintains its speed advantages over time. To ensure this, we've implemented an automated benchmarking system that runs on every pull request targeting the main branch.
+
+## How the Benchmark Workflow Works
+
+The automated benchmark workflow runs on GitHub Actions and performs the following steps:
+
+1. Sets up a Python environment with all necessary dependencies
+2. Installs optional GPU dependencies (if available in the runner)
+3. Runs a small subset of our benchmark examples:
+   - SugarScape model (with 50,000 agents)
+   - Boltzmann Wealth model (with 10,000 agents)
+4. Generates timing results comparing mesa-frames to the original Mesa implementation
+5. Produces a visualization of the benchmark results
+6. Posts a comment on the PR with the benchmark results
+7. Uploads full benchmark artifacts for detailed inspection
+
+## Interpreting Benchmark Results
+
+When reviewing a PR with benchmark results, look for:
+
+1. **Successful execution**: The benchmarks should complete without errors
+2. **Performance impact**: Check if the PR introduces any performance regressions
+3. **Expected changes**: If the PR is aimed at improving performance, verify that the benchmarks show the expected improvements
+
+The benchmark comment will include:
+
+- Execution time for both mesa-frames and Mesa implementations
+- The speedup factor (how many times faster mesa-frames is compared to Mesa)
+- A visualization comparing the performance
+
+## Running Benchmarks Locally
+
+To run the same benchmarks locally and compare your changes to the current main branch:
+
+```bash
+# Clone the repository
+git clone https://github.com/projectmesa/mesa-frames.git
+cd mesa-frames
+
+# Install dependencies
+pip install -e ".[dev]"
+pip install perfplot matplotlib seaborn
+
+# Run the Sugarscape benchmark
+cd examples/sugarscape_ig
+python performance_comparison.py
+
+# Run the Boltzmann Wealth benchmark
+cd ../boltzmann_wealth
+python performance_plot.py
+```
+
+The full benchmarks will take longer to run than the CI version as they test with more agents.
+
+## Adding New Benchmarks
+
+When adding new models or features to mesa-frames, consider adding benchmark tests to ensure their performance:
+
+1. Create a benchmark script in the `examples` directory
+2. Implement both mesa-frames and Mesa versions of the model
+3. Use the `perfplot` library to measure and visualize performance
+4. Update the GitHub Actions workflow to include your new benchmark (with a small dataset for CI)
+
+## Tips for Performance Optimization
+
+When optimizing code in mesa-frames:
+
+1. **Always benchmark your changes**: Don't assume changes will improve performance without measuring
+2. **Focus on real-world use cases**: Optimize for patterns that users are likely to encounter
+3. **Balance readability and performance**: Code should remain maintainable even while being optimized
+4. **Document performance characteristics**: Note any trade-offs or specific usage patterns that affect performance
+5. **Test on different hardware**: If possible, verify improvements on both CPU and GPU environments
+
+Remember that consistent, predictable performance is often more valuable than squeezing out every last bit of speed at the cost of complexity or stability.

mkdocs.yml

@@ -115,4 +115,5 @@ nav:
   - API Reference: api/index.html
   - Contributing:
     - Contribution Guide: contributing.md
+    - Benchmarking: contributing/benchmarks.md
     - Roadmap: roadmap.md