*DO NOT MERGE* Doom hotfixes #1420
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: "Network fault tolerance" | |
| on: | |
| # We're using merge-chains; so this needs to run then. | |
| merge_group: | |
| push: | |
| branches: | |
| - master | |
| - release | |
| pull_request: | |
| workflow_dispatch: | |
| inputs: | |
| debug_enabled: | |
| type: boolean | |
| description: 'Run the build with tmate debugging enabled (https://github.com/marketplace/actions/debugging-with-tmate)' | |
| required: false | |
| default: false | |
| jobs: | |
| network-test: | |
| runs-on: ubuntu-latest | |
| strategy: | |
| # Hack: Minimise concurrency as this is blocking our GitHub runners. | |
| # This can be removed when the matrix mode itself is removed and the | |
| # network testing is done in pure Haskell. | |
| max-parallel: 2 | |
| matrix: | |
| # Note: At present we can only run for 3 peers; to configure this for | |
| # more we need to make the docker-compose spin-up dynamic across | |
| # however many we would like to configure here. | |
| # Currently this is just a label and does not have any functional impact. | |
| peers: [3] | |
| scaling_factor: [10, 50] | |
| # Note: We only put here the configuration values we _expected to pass_. | |
| netem_loss: [0, 1, 2, 3] | |
| name: "Peers: ${{ matrix.peers }}, scaling: ${{ matrix.scaling_factor }}, loss: ${{ matrix.netem_loss }}" | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| submodules: true | |
| - name: ❄ Prepare nix | |
| uses: cachix/install-nix-action@v30 | |
| with: | |
| extra_nix_config: | | |
| accept-flake-config = true | |
| log-lines = 1000 | |
| - name: ❄ Cachix cache of nix derivations | |
| uses: cachix/cachix-action@v15 | |
| with: | |
| name: cardano-scaling | |
| authToken: '${{ secrets.CACHIX_CARDANO_SCALING_AUTH_TOKEN }}' | |
| - name: Set up and use the "ci" devShell | |
| uses: nicknovitski/nix-develop@v1 | |
| with: | |
| arguments: ".#ci" | |
| - name: Build docker images for netem specifically | |
| run: | | |
| nix build .#docker-hydra-node-for-netem | |
| ./result | docker load | |
| - name: Setup containers for network testing | |
| run: | | |
| set -exo pipefail | |
| cd demo | |
| ./prepare-devnet.sh | |
| docker compose up -d cardano-node | |
| sleep 2 | |
| # :tear: socket permissions. | |
| sudo chmod a+w devnet/node.socket | |
| ./seed-devnet.sh "nix run .#cardano-cli --" "nix run .#hydra-node --" | |
| # Specify two docker compose yamls; the second one overrides the | |
| # images to use the netem ones specifically | |
| docker compose -f docker-compose.yaml -f docker-compose-netem.yaml up -d hydra-node-{1,2,3} | |
| docker ps | |
| - name: Build required nix and docker derivations | |
| run: | | |
| nix build .#legacyPackages.x86_64-linux.hydra-cluster.components.benchmarks.bench-e2e | |
| nix build github:noonio/pumba/noon/add-flake | |
| # Use tmate to get a shell onto the runner to do some temporary hacking | |
| # | |
| # <https://github.com/mxschmitt/action-tmate> | |
| # | |
| - name: Setup tmate session | |
| uses: mxschmitt/action-tmate@v3 | |
| if: ${{ github.event_name == 'workflow_dispatch' && inputs.debug_enabled }} | |
| with: | |
| limit-access-to-actor: true | |
| - name: Run pumba and the benchmarks | |
| run: | | |
| # Extract inputs with defaults for non-workflow_dispatch events | |
| percent="${{ matrix.netem_loss }}" | |
| scaling_factor="${{ matrix.scaling_factor }}" | |
| target_peer="hydra-node-1" | |
| other_peers="172.16.238.20 172.16.238.30" | |
| .github/workflows/network/run_pumba.sh $target_peer $percent $other_peers | |
| # Run benchmark on demo | |
| nix run .#legacyPackages.x86_64-linux.hydra-cluster.components.benchmarks.bench-e2e -- \ | |
| demo \ | |
| --output-directory=benchmarks \ | |
| --scaling-factor="$scaling_factor" \ | |
| --timeout=1000s \ | |
| --testnet-magic 42 \ | |
| --node-socket=demo/devnet/node.socket \ | |
| --hydra-client=localhost:4001 \ | |
| --hydra-client=localhost:4002 \ | |
| --hydra-client=localhost:4003 | |
| - name: Acquire logs | |
| if: always() | |
| run: | | |
| cd demo | |
| docker compose logs > docker-logs | |
| - name: 💾 Upload logs | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: "artifacts-netem-loss=${{ matrix.netem_loss }},scaling_factor=${{ matrix.scaling_factor }},peers=${{ matrix.peers }}" | |
| path: | | |
| demo/docker-logs | |
| benchmarks | |
| demo/devnet/protocol-parameters.json | |
| if-no-files-found: ignore |