diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml new file mode 100644 index 00000000..b4f629e5 --- /dev/null +++ b/.github/workflows/benchmarks.yml @@ -0,0 +1,250 @@ +name: Benchmarks + +on: + pull_request: + branches: [main] + types: [opened, synchronize, reopened, ready_for_review] + push: + branches: [main] + workflow_dispatch: + +# Add explicit permissions for the workflow +permissions: + contents: read + pull-requests: write + actions: read + +jobs: + benchmark: + name: Run Performance Benchmarks + runs-on: ubuntu-latest + if: github.event.pull_request.draft == false + + steps: + - name: Checkout Repository + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.10' + cache: 'pip' + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + python -m pip install -e ".[dev]" + python -m pip install perfplot matplotlib seaborn + + - name: Try to set up GPU support (optional) + run: | + # Try to install CUDA dependencies for GPU-accelerated benchmarks + # This won't fail the workflow if it doesn't work, as most CI runners don't have GPUs + python -m pip install numba cupy-cuda11x || echo "GPU support not available (expected for most CI runners)" + + - name: Run SugarScape Benchmark (Small Dataset) + run: | + cd examples/sugarscape_ig + python -c ' + import sys + import time + from performance_comparison import SugarScapeSetup, mesa_frames_polars_numba_parallel, mesa_implementation + + # Run a smaller subset for CI benchmarks (faster execution) + setup = SugarScapeSetup(50000) + + print("Running mesa-frames implementation...") + start_time = time.time() + mf_model = mesa_frames_polars_numba_parallel(setup) + mf_time = time.time() - start_time + print(f"mesa-frames implementation completed in {mf_time:.2f} seconds") + + print("Running mesa implementation...") + start_time = time.time() + mesa_model = mesa_implementation(setup) + mesa_time = time.time() - start_time + print(f"mesa implementation completed in {mesa_time:.2f} seconds") + + print("Benchmark complete!") + + # Save timing results for the PR comment + with open("sugarscape_results.txt", "w") as f: + f.write(f"mesa-frames: {mf_time:.2f}s\n") + f.write(f"mesa: {mesa_time:.2f}s\n") + f.write(f"speedup: {mesa_time/mf_time:.2f}x\n") + ' + + - name: Run Boltzmann Wealth Benchmark (Small Dataset) + run: | + cd examples/boltzmann_wealth + python -c ' + import sys + import time + from performance_plot import mesa_frames_polars_concise, mesa_implementation + + # Run a smaller subset for CI benchmarks (faster execution) + print("Running mesa-frames implementation...") + start_time = time.time() + mf_model = mesa_frames_polars_concise(10000) + mf_time = time.time() - start_time + print(f"mesa-frames implementation completed in {mf_time:.2f} seconds") + + print("Running mesa implementation...") + start_time = time.time() + mesa_model = mesa_implementation(10000) + mesa_time = time.time() - start_time + print(f"mesa implementation completed in {mesa_time:.2f} seconds") + + print("Benchmark complete!") + + # Save timing results for the PR comment + with open("boltzmann_results.txt", "w") as f: + f.write(f"mesa-frames: {mf_time:.2f}s\n") + f.write(f"mesa: {mesa_time:.2f}s\n") + f.write(f"speedup: {mesa_time/mf_time:.2f}x\n") + ' + + - name: Generate Simple Benchmark Visualizations + run: | + python -c ' + import matplotlib.pyplot as plt + import numpy as np + import os + + # Function to read benchmark results + def read_results(filename): + results = {} + with open(filename, "r") as f: + for line in f: + key, value = line.strip().split(": ") + results[key] = value + return results + + # Create visualization for Sugarscape benchmark + sugarscape_results = read_results("examples/sugarscape_ig/sugarscape_results.txt") + boltzmann_results = read_results("examples/boltzmann_wealth/boltzmann_results.txt") + + # Create a simple bar chart comparing execution times + fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(12, 5)) + + # Sugarscape plot + sugarscape_mesa_time = float(sugarscape_results["mesa"].replace("s", "")) + sugarscape_mf_time = float(sugarscape_results["mesa-frames"].replace("s", "")) + ax1.bar(["mesa-frames", "mesa"], [sugarscape_mf_time, sugarscape_mesa_time]) + ax1.set_title("SugarScape Benchmark (50k agents)") + ax1.set_ylabel("Execution time (s)") + ax1.text(0, sugarscape_mf_time/2, f"{sugarscape_mf_time:.2f}s", + ha="center", va="center", color="white", fontweight="bold") + ax1.text(1, sugarscape_mesa_time/2, f"{sugarscape_mesa_time:.2f}s", + ha="center", va="center", color="white", fontweight="bold") + ax1.text(0.5, max(sugarscape_mf_time, sugarscape_mesa_time) * 0.9, + f"Speedup: {sugarscape_results[\"speedup\"]}", + ha="center", va="center", bbox=dict(facecolor="white", alpha=0.8)) + + # Boltzmann plot + boltzmann_mesa_time = float(boltzmann_results["mesa"].replace("s", "")) + boltzmann_mf_time = float(boltzmann_results["mesa-frames"].replace("s", "")) + ax2.bar(["mesa-frames", "mesa"], [boltzmann_mf_time, boltzmann_mesa_time]) + ax2.set_title("Boltzmann Wealth Benchmark (10k agents)") + ax2.set_ylabel("Execution time (s)") + ax2.text(0, boltzmann_mf_time/2, f"{boltzmann_mf_time:.2f}s", + ha="center", va="center", color="white", fontweight="bold") + ax2.text(1, boltzmann_mesa_time/2, f"{boltzmann_mesa_time:.2f}s", + ha="center", va="center", color="white", fontweight="bold") + ax2.text(0.5, max(boltzmann_mf_time, boltzmann_mesa_time) * 0.9, + f"Speedup: {boltzmann_results[\"speedup\"]}", + ha="center", va="center", bbox=dict(facecolor="white", alpha=0.8)) + + plt.tight_layout() + plt.savefig("benchmark_results.png", dpi=150) + print("Benchmark visualization saved as benchmark_results.png") + ' + + - name: Prepare Artifacts Directory + run: | + mkdir -p benchmark_artifacts + cp examples/sugarscape_ig/*.png benchmark_artifacts/ || true + cp examples/sugarscape_ig/*.txt benchmark_artifacts/ || true + cp examples/boltzmann_wealth/*.png benchmark_artifacts/ || true + cp examples/boltzmann_wealth/*.txt benchmark_artifacts/ || true + cp benchmark_results.png benchmark_artifacts/ || true + + - name: Save Benchmark Results + if: always() + uses: actions/upload-artifact@v4 + with: + name: benchmark-results + path: benchmark_artifacts + + - name: Log Benchmark Results (Fallback) + if: always() + run: | + echo "===== BENCHMARK RESULTS (FALLBACK) =====" + echo "SugarScape Benchmark Results:" + if [ -f "examples/sugarscape_ig/sugarscape_results.txt" ]; then + cat examples/sugarscape_ig/sugarscape_results.txt + else + echo "Results file not found" + fi + + echo "Boltzmann Wealth Benchmark Results:" + if [ -f "examples/boltzmann_wealth/boltzmann_results.txt" ]; then + cat examples/boltzmann_wealth/boltzmann_results.txt + else + echo "Results file not found" + fi + + - name: Create Result Summary for PR Comment + if: github.event_name == 'pull_request' + id: result_summary + run: | + # Create summary file + echo '## 📊 Performance Benchmark Results' > summary.md + echo '' >> summary.md + echo 'The benchmarks have been executed.' >> summary.md + echo '' >> summary.md + + echo '### SugarScape Model (50k agents, 100 steps)' >> summary.md + echo '```' >> summary.md + if [ -f "examples/sugarscape_ig/sugarscape_results.txt" ]; then + cat examples/sugarscape_ig/sugarscape_results.txt >> summary.md + else + echo "Results file not found" >> summary.md + fi + echo '```' >> summary.md + echo '' >> summary.md + + echo '### Boltzmann Wealth Model (10k agents, 100 steps)' >> summary.md + echo '```' >> summary.md + if [ -f "examples/boltzmann_wealth/boltzmann_results.txt" ]; then + cat examples/boltzmann_wealth/boltzmann_results.txt >> summary.md + else + echo "Results file not found" >> summary.md + fi + echo '```' >> summary.md + echo '' >> summary.md + + echo "[Click here to see full benchmark results](${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID})" >> summary.md + echo "Note: Benchmark results are now available as artifacts on the GitHub Actions run page." >> summary.md + + # Set output + SUMMARY=$(cat summary.md) + echo "summary<> $GITHUB_OUTPUT + echo "$SUMMARY" >> $GITHUB_OUTPUT + echo "EOF" >> $GITHUB_OUTPUT + + - name: Add Benchmark Comment + if: github.event_name == 'pull_request' + uses: actions/github-script@v6 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const fs = require('fs'); + + // Create a comment with benchmark results + github.rest.issues.createComment({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + body: `${{ steps.result_summary.outputs.summary }}` + }); \ No newline at end of file diff --git a/docs/general/contributing/benchmarks.md b/docs/general/contributing/benchmarks.md new file mode 100644 index 00000000..07bf52b3 --- /dev/null +++ b/docs/general/contributing/benchmarks.md @@ -0,0 +1,76 @@ +# Benchmarking in mesa-frames + +As a library focused on performance improvements, it's crucial that mesa-frames maintains its speed advantages over time. To ensure this, we've implemented an automated benchmarking system that runs on every pull request targeting the main branch. + +## How the Benchmark Workflow Works + +The automated benchmark workflow runs on GitHub Actions and performs the following steps: + +1. Sets up a Python environment with all necessary dependencies +2. Installs optional GPU dependencies (if available in the runner) +3. Runs a small subset of our benchmark examples: + - SugarScape model (with 50,000 agents) + - Boltzmann Wealth model (with 10,000 agents) +4. Generates timing results comparing mesa-frames to the original Mesa implementation +5. Produces a visualization of the benchmark results +6. Posts a comment on the PR with the benchmark results +7. Uploads full benchmark artifacts for detailed inspection + +## Interpreting Benchmark Results + +When reviewing a PR with benchmark results, look for: + +1. **Successful execution**: The benchmarks should complete without errors +2. **Performance impact**: Check if the PR introduces any performance regressions +3. **Expected changes**: If the PR is aimed at improving performance, verify that the benchmarks show the expected improvements + +The benchmark comment will include: + +- Execution time for both mesa-frames and Mesa implementations +- The speedup factor (how many times faster mesa-frames is compared to Mesa) +- A visualization comparing the performance + +## Running Benchmarks Locally + +To run the same benchmarks locally and compare your changes to the current main branch: + +```bash +# Clone the repository +git clone https://github.com/projectmesa/mesa-frames.git +cd mesa-frames + +# Install dependencies +pip install -e ".[dev]" +pip install perfplot matplotlib seaborn + +# Run the Sugarscape benchmark +cd examples/sugarscape_ig +python performance_comparison.py + +# Run the Boltzmann Wealth benchmark +cd ../boltzmann_wealth +python performance_plot.py +``` + +The full benchmarks will take longer to run than the CI version as they test with more agents. + +## Adding New Benchmarks + +When adding new models or features to mesa-frames, consider adding benchmark tests to ensure their performance: + +1. Create a benchmark script in the `examples` directory +2. Implement both mesa-frames and Mesa versions of the model +3. Use the `perfplot` library to measure and visualize performance +4. Update the GitHub Actions workflow to include your new benchmark (with a small dataset for CI) + +## Tips for Performance Optimization + +When optimizing code in mesa-frames: + +1. **Always benchmark your changes**: Don't assume changes will improve performance without measuring +2. **Focus on real-world use cases**: Optimize for patterns that users are likely to encounter +3. **Balance readability and performance**: Code should remain maintainable even while being optimized +4. **Document performance characteristics**: Note any trade-offs or specific usage patterns that affect performance +5. **Test on different hardware**: If possible, verify improvements on both CPU and GPU environments + +Remember that consistent, predictable performance is often more valuable than squeezing out every last bit of speed at the cost of complexity or stability. diff --git a/mkdocs.yml b/mkdocs.yml index e7eb915b..da78c6c5 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -115,4 +115,5 @@ nav: - API Reference: api/index.html - Contributing: - Contribution Guide: contributing.md + - Benchmarking: contributing/benchmarks.md - Roadmap: roadmap.md \ No newline at end of file