From 95637f888cd858259de2b543f6f5dbff5b97a11f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ja=C3=ABl=20Champagne=20Gareau?= Date: Tue, 17 Jun 2025 19:50:18 -0400 Subject: [PATCH 01/10] Fix the `uniform_all` random generator - It always produced 'inf' because the range was greater than mt19937_64's limit - It makes more sense to use a logscale to better cover the range of possible floating values: uniform_all -> logspace_all --- benchmarks/benchmark.cpp | 2 +- benchmarks/random_generators.h | 39 ++++++++++++++++++++--------- scripts/generate_multiple_tables.py | 12 ++++----- 3 files changed, 34 insertions(+), 19 deletions(-) diff --git a/benchmarks/benchmark.cpp b/benchmarks/benchmark.cpp index 9319852..32ea36d 100644 --- a/benchmarks/benchmark.cpp +++ b/benchmarks/benchmark.cpp @@ -375,7 +375,7 @@ int main(int argc, char **argv) { fmt::println(" ./benchmark --file=data/canada.txt # Run benchmark using numbers from a file"); fmt::println(" ./benchmark --fixed=10 # Test fixed-point representation instead of shortest length"); fmt::println(" ./benchmark --test # Test correctness instead of performance"); - fmt::println(" ./benchmark --volume=1000 --model=uniform # Generate 1000 uniform random numbers"); + fmt::println(" ./benchmark --volume=1000 --model=uniform_01 # Generate 1000 uniform random numbers in [0, 1]"); fmt::println(" ./benchmark --algo-filter=ryu,grisu # Only test algorithms containing 'ryu' or 'grisu'"); fmt::println("\nFor full options list, run: ./benchmark --help"); return EXIT_FAILURE; diff --git a/benchmarks/random_generators.h b/benchmarks/random_generators.h index d6f37ac..7458156 100644 --- a/benchmarks/random_generators.h +++ b/benchmarks/random_generators.h @@ -30,6 +30,24 @@ struct uniform_generator : float_number_generator { T new_float() override { return dis(gen); } }; +template +struct logspace_generator : float_number_generator { + std::random_device rd; + std::mt19937_64 gen; + std::uniform_int_distribution exp; + std::uniform_real_distribution significand; + explicit logspace_generator() + : rd(), gen(rd()), + exp(std::numeric_limits::min_exponent + 1, // +1 skips subnormals + std::numeric_limits::max_exponent), + significand(-1, 1) {} + std::string describe() override { + return "Generate random numbers uniformly in log2 space, i.e. " + "magnitudes uniformly distributed in the interval [-2^max_exponent, 2^max_exponent]"; + } + T new_float() override { return significand(gen) * std::pow(2.0, exp(gen)); } +}; + enum centering { centered, non_centered }; template struct centered_generator : float_number_generator { @@ -112,29 +130,26 @@ struct one_over_rand : float_number_generator { }; constexpr std::array model_names = { - "uniform_01" , "uniform_all" , "integer_uniform" , - "centered" , "non_centered" , - "simple_uniform" , "simple_int" , - "one_over_rand" + "uniform_01" , "logspace_all" , + "centered" , "non_centered" , + "simple_uniform" , "simple_int" , + "one_over_rand" , "integer_uniform" , }; template inline std::unique_ptr> -get_generator_by_name(std::string name) { +get_generator_by_name(const std::string name) { std::cout << "available models (-m): "; - for (std::string name : model_names) { - std::cout << name << " "; + for (const auto& model : model_names) { + std::cout << model << " "; } std::cout << std::endl; // This is naive, but also not very important. if (name == "uniform_01") return std::unique_ptr>(new uniform_generator()); - if (name == "uniform_all") { - return std::unique_ptr>( - new uniform_generator(std::numeric_limits::lowest(), - std::numeric_limits::max()) - ); + if (name == "logspace_all") { + return std::unique_ptr>(new logspace_generator()); } if (name == "centered") return std::unique_ptr>(new centered_generator()); diff --git a/scripts/generate_multiple_tables.py b/scripts/generate_multiple_tables.py index efdd318..1f9e0ae 100755 --- a/scripts/generate_multiple_tables.py +++ b/scripts/generate_multiple_tables.py @@ -14,13 +14,13 @@ ] models = [ 'uniform_01', - 'uniform_all', - 'integer_uniform', - 'centered', - 'non_centered', + # 'logspace_all', + # 'integer_uniform', + # 'centered', + # 'non_centered', ] -runs_r = 1_000 -volume_v = 1_000_000 +runs_r = 100 +volume_v = 100_000 flag_combinations = [ [], ['-F6'], From 1ba42e8a584d21287a22ff54674f64ee85112f87 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ja=C3=ABl=20Champagne=20Gareau?= Date: Mon, 23 Jun 2025 18:03:02 -0400 Subject: [PATCH 02/10] add bash script to run tests on AWS instances --- scripts/aws_tests.bash | 73 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 73 insertions(+) create mode 100755 scripts/aws_tests.bash diff --git a/scripts/aws_tests.bash b/scripts/aws_tests.bash new file mode 100755 index 0000000..15dd96a --- /dev/null +++ b/scripts/aws_tests.bash @@ -0,0 +1,73 @@ +#!/bin/bash + +# Preliminary setup +# - Install AWS CLI v2 (perhaps it works on v1, untested): +# - on Arch Linux: paru -S aws-cli-v2 +# - Configure AWS Credentials: aws configure + +set -e # Exit script on first error + +AMI_ID_x86_64="ami-020cba7c55df1f615" +AMI_ID_aarch64="ami-07041441b708acbd6" + +INSTANCES_x86_64=() +INSTANCES_aarch64=("c6g.medium" "c7g.medium" "c8g.medium") + +KEY_NAME="openssh" +KEY_PATH="~/.ssh/openssh.pem" +SSH_COMMAND="ssh -i ${KEY_PATH} -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" +SECURITY_GROUP="sg-0315466a0d7fc99e1" +VOLUME_SIZE=10 # in GB + +PROJECT_DIR=$(basename $(pwd)) + +for INSTANCE_NAME in "${INSTANCES_x86_64[@]}" "${INSTANCES_aarch64[@]}"; do + if printf '%s\n' "${INSTANCES_aarch64[@]}" | grep -qx "${INSTANCE_NAME}"; then + AMI_ID=${AMI_ID_aarch64} + else + AMI_ID=${AMI_ID_x86_64} + fi + echo "Running instance for ${INSTANCE_NAME} with AMI ${AMI_ID}" + + INSTANCE_ID=$(aws ec2 run-instances \ + --image-id ${AMI_ID} \ + --instance-type ${INSTANCE_NAME} \ + --key-name ${KEY_NAME} \ + --block-device-mappings "DeviceName=/dev/sda1,Ebs={VolumeSize=${VOLUME_SIZE}}" \ + --associate-public-ip-address \ + --security-group-ids ${SECURITY_GROUP} \ + --count "1" --query 'Instances[0].InstanceId' --output text) + + aws ec2 wait instance-status-ok --instance-ids ${INSTANCE_ID} + echo "Started instance: ${INSTANCE_ID}" + + PUBLIC_IP=$(aws ec2 describe-instances \ + --instance-ids ${INSTANCE_ID} \ + --query "Reservations[0].Instances[0].PublicIpAddress" \ + --output text) + echo "Instance public IP: ${PUBLIC_IP}" + + rsync -avz --partial --progress --exclude ".git" --exclude "build" -e "${SSH_COMMAND}" \ + ${PROJECT_DIR}/ ubuntu@${PUBLIC_IP}:~/${PROJECT_DIR} + ${SSH_COMMAND} ubuntu@${PUBLIC_IP} << 'EOF' + set -e # Exit on error + + # Install dependencies + sudo apt update + sudo apt install -y linux-tools-common linux-tools-generic g++ cmake + + # Build the project + cmake -B build . && cmake --build build + + # Run the script to generate multiple tables + ./scripts/generate_multiple_tables.py +EOF + + echo "Script executed successfully" + mkdir -p "${PROJECT_DIR}/outputs/${INSTANCE_NAME}" + rsync -avz --partial --progress -e "${SSH_COMMAND}" \ + ubuntu@${PUBLIC_IP}:~/${PROJECT_DIR}/outputs/ ${PROJECT_DIR}/outputs/${INSTANCE_NAME}/ + + aws ec2 terminate-instances --instance-ids ${INSTANCE_ID} + echo "Terminated instance: ${INSTANCE_ID}" +done From 845697e359452152ae634abd213104b54d1cbc3f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ja=C3=ABl=20Champagne=20Gareau?= Date: Mon, 23 Jun 2025 18:25:00 -0400 Subject: [PATCH 03/10] make all AWS instances run in parallel --- scripts/aws_tests.bash | 33 +++++++++++++++++++++++---------- 1 file changed, 23 insertions(+), 10 deletions(-) diff --git a/scripts/aws_tests.bash b/scripts/aws_tests.bash index 15dd96a..fb03a27 100755 --- a/scripts/aws_tests.bash +++ b/scripts/aws_tests.bash @@ -21,12 +21,9 @@ VOLUME_SIZE=10 # in GB PROJECT_DIR=$(basename $(pwd)) -for INSTANCE_NAME in "${INSTANCES_x86_64[@]}" "${INSTANCES_aarch64[@]}"; do - if printf '%s\n' "${INSTANCES_aarch64[@]}" | grep -qx "${INSTANCE_NAME}"; then - AMI_ID=${AMI_ID_aarch64} - else - AMI_ID=${AMI_ID_x86_64} - fi +process_instance() { + INSTANCE_NAME=$1 + AMI_ID=$2 echo "Running instance for ${INSTANCE_NAME} with AMI ${AMI_ID}" INSTANCE_ID=$(aws ec2 run-instances \ @@ -38,6 +35,7 @@ for INSTANCE_NAME in "${INSTANCES_x86_64[@]}" "${INSTANCES_aarch64[@]}"; do --security-group-ids ${SECURITY_GROUP} \ --count "1" --query 'Instances[0].InstanceId' --output text) + echo "Waiting for instance ${INSTANCE_ID} to be ready..." aws ec2 wait instance-status-ok --instance-ids ${INSTANCE_ID} echo "Started instance: ${INSTANCE_ID}" @@ -45,10 +43,10 @@ for INSTANCE_NAME in "${INSTANCES_x86_64[@]}" "${INSTANCES_aarch64[@]}"; do --instance-ids ${INSTANCE_ID} \ --query "Reservations[0].Instances[0].PublicIpAddress" \ --output text) - echo "Instance public IP: ${PUBLIC_IP}" + echo "Instance ${INSTANCE_ID} public IP: ${PUBLIC_IP}" rsync -avz --partial --progress --exclude ".git" --exclude "build" -e "${SSH_COMMAND}" \ - ${PROJECT_DIR}/ ubuntu@${PUBLIC_IP}:~/${PROJECT_DIR} + ./ ubuntu@${PUBLIC_IP}:~/${PROJECT_DIR} ${SSH_COMMAND} ubuntu@${PUBLIC_IP} << 'EOF' set -e # Exit on error @@ -63,11 +61,26 @@ for INSTANCE_NAME in "${INSTANCES_x86_64[@]}" "${INSTANCES_aarch64[@]}"; do ./scripts/generate_multiple_tables.py EOF - echo "Script executed successfully" + echo "Script executed successfully on ${INSTANCE_NAME}" mkdir -p "${PROJECT_DIR}/outputs/${INSTANCE_NAME}" rsync -avz --partial --progress -e "${SSH_COMMAND}" \ - ubuntu@${PUBLIC_IP}:~/${PROJECT_DIR}/outputs/ ${PROJECT_DIR}/outputs/${INSTANCE_NAME}/ + ubuntu@${PUBLIC_IP}:~/${PROJECT_DIR}/outputs/ ./outputs/${INSTANCE_NAME}/ aws ec2 terminate-instances --instance-ids ${INSTANCE_ID} echo "Terminated instance: ${INSTANCE_ID}" +} + +echo "Launching ${#INSTANCES_aarch64[@]} aarch64 instances and ${#INSTANCES_x86_64[@]} x86_64 instances in parallel..." +for INSTANCE_NAME in "${INSTANCES_x86_64[@]}" "${INSTANCES_aarch64[@]}"; do + if printf '%s\n' "${INSTANCES_aarch64[@]}" | grep -qx "${INSTANCE_NAME}"; then + AMI_ID=${AMI_ID_aarch64} + else + AMI_ID=${AMI_ID_x86_64} + fi + + process_instance "${INSTANCE_NAME}" "${AMI_ID}" 2>&1 | tee "${INSTANCE_NAME}.log" & done + +# Wait for all background jobs to finish +wait +echo "All instances completed." From b9f7b61b32cd49abbcb0d54d59ab7e3751998113 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ja=C3=ABl=20Champagne=20Gareau?= Date: Mon, 23 Jun 2025 20:46:29 -0400 Subject: [PATCH 04/10] fix missing `cd` and remove warning in AWS script --- scripts/aws_tests.bash | 12 +++++++----- scripts/generate_multiple_tables.py | 5 ++--- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/scripts/aws_tests.bash b/scripts/aws_tests.bash index fb03a27..0dccda7 100755 --- a/scripts/aws_tests.bash +++ b/scripts/aws_tests.bash @@ -47,17 +47,19 @@ process_instance() { rsync -avz --partial --progress --exclude ".git" --exclude "build" -e "${SSH_COMMAND}" \ ./ ubuntu@${PUBLIC_IP}:~/${PROJECT_DIR} - ${SSH_COMMAND} ubuntu@${PUBLIC_IP} << 'EOF' + ${SSH_COMMAND} ubuntu@${PUBLIC_IP} << EOF set -e # Exit on error - # Install dependencies + echo "Updating and installing dependencies on ${INSTANCE_NAME}..." sudo apt update - sudo apt install -y linux-tools-common linux-tools-generic g++ cmake + sudo DEBIAN_FRONTEND=noninteractive \ + apt install -y linux-tools-common linux-tools-generic g++ cmake python3 - # Build the project + echo "Building project..." + cd ~/${PROJECT_DIR} cmake -B build . && cmake --build build - # Run the script to generate multiple tables + echo "Running the python script to generate tests..." ./scripts/generate_multiple_tables.py EOF diff --git a/scripts/generate_multiple_tables.py b/scripts/generate_multiple_tables.py index 1f9e0ae..2b8d08d 100755 --- a/scripts/generate_multiple_tables.py +++ b/scripts/generate_multiple_tables.py @@ -6,7 +6,6 @@ # Configuration benchmark_executable = './build/benchmarks/benchmark' -latex_script = './scripts/latex_table.py' output_dir = './outputs' input_files = [ 'data/canada.txt', @@ -60,7 +59,7 @@ def run_cmd(cmd): def process_job(label, cmd_args, flags): # Run the benchmark cmd = [benchmark_executable] + cmd_args + flags - print(f"Running: {' '.join(cmd)}") + print(f"Running: {' '.join(cmd)}", flush=True) output = run_cmd(cmd) # Build output file name @@ -73,7 +72,7 @@ def process_job(label, cmd_args, flags): tex_content = generate_latex_table(output) with open(out_path, 'w') as f: f.write(tex_content) - print(f"Written: {out_path}\n") + print(f"Written: {out_path}\n", flush=True) if __name__ == '__main__': From 4f8e13c970f85ccc733a84d4918dc7f5a1bf7e24 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ja=C3=ABl=20Champagne=20Gareau?= Date: Mon, 23 Jun 2025 22:12:52 -0400 Subject: [PATCH 05/10] fix perf permissions in AWS script --- scripts/aws_tests.bash | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/aws_tests.bash b/scripts/aws_tests.bash index 0dccda7..5ad1660 100755 --- a/scripts/aws_tests.bash +++ b/scripts/aws_tests.bash @@ -45,7 +45,7 @@ process_instance() { --output text) echo "Instance ${INSTANCE_ID} public IP: ${PUBLIC_IP}" - rsync -avz --partial --progress --exclude ".git" --exclude "build" -e "${SSH_COMMAND}" \ + git ls-files -z | rsync -avz --partial --progress --from0 --files-from=- -e "${SSH_COMMAND}" \ ./ ubuntu@${PUBLIC_IP}:~/${PROJECT_DIR} ${SSH_COMMAND} ubuntu@${PUBLIC_IP} << EOF set -e # Exit on error @@ -60,6 +60,7 @@ process_instance() { cmake -B build . && cmake --build build echo "Running the python script to generate tests..." + echo -1 | sudo tee /proc/sys/kernel/perf_event_paranoid ./scripts/generate_multiple_tables.py EOF From 0b6d47386e9c1b6602cdb9f3166c3a8c781db142 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ja=C3=ABl=20Champagne=20Gareau?= Date: Mon, 23 Jun 2025 23:28:01 -0400 Subject: [PATCH 06/10] use lscpu instead of /proc/cpuinfo --- scripts/generate_multiple_tables.py | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/scripts/generate_multiple_tables.py b/scripts/generate_multiple_tables.py index 2b8d08d..546e8b4 100755 --- a/scripts/generate_multiple_tables.py +++ b/scripts/generate_multiple_tables.py @@ -29,18 +29,24 @@ def get_cpu_model(): - if platform.system() == "Windows": + system = platform.system() + if system == "Windows": return platform.processor() - elif platform.system() == "Darwin": + elif system == "Darwin": os.environ['PATH'] = os.environ['PATH'] + os.pathsep + '/usr/sbin' - command = "sysctl -n machdep.cpu.brand_string" - return subprocess.check_output(command).strip() - elif platform.system() == "Linux": - command = "cat /proc/cpuinfo" - output = subprocess.check_output(command, shell=True).decode().strip() - for line in output.split("\n"): - if line.startswith("model name"): - return line.split(':', 1)[1].strip() + command = ["sysctl", "-n", "machdep.cpu.brand_string"] + return subprocess.check_output(command, text=True).strip() + elif system == "Linux": + output = subprocess.check_output(["lscpu"], text=True) + model_name = None + architecture = None + for line in output.splitlines(): + if "Model name:" in line: + model_name = line.split(":", 1)[1].strip() + elif "Architecture:" in line: + architecture = line.split(":", 1)[1].strip() + # Prefer model_name if available; fallback to architecture + return model_name or architecture or "unknown_cpu" return "unknown_cpu" From 7f5416263f7288ccd6ca49055a34b807f78a2673 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ja=C3=ABl=20Champagne=20Gareau?= Date: Mon, 23 Jun 2025 23:31:15 -0400 Subject: [PATCH 07/10] make the tests run also with clang++ builds --- scripts/aws_tests.bash | 20 +++++++++++++------- scripts/generate_multiple_tables.py | 9 ++++++++- 2 files changed, 21 insertions(+), 8 deletions(-) diff --git a/scripts/aws_tests.bash b/scripts/aws_tests.bash index 5ad1660..add9b66 100755 --- a/scripts/aws_tests.bash +++ b/scripts/aws_tests.bash @@ -49,19 +49,25 @@ process_instance() { ./ ubuntu@${PUBLIC_IP}:~/${PROJECT_DIR} ${SSH_COMMAND} ubuntu@${PUBLIC_IP} << EOF set -e # Exit on error + cd ~/${PROJECT_DIR} echo "Updating and installing dependencies on ${INSTANCE_NAME}..." sudo apt update sudo DEBIAN_FRONTEND=noninteractive \ - apt install -y linux-tools-common linux-tools-generic g++ cmake python3 + apt install -y linux-tools-common linux-tools-generic \ + g++ clang cmake python3 + echo -1 | sudo tee /proc/sys/kernel/perf_event_paranoid - echo "Building project..." - cd ~/${PROJECT_DIR} - cmake -B build . && cmake --build build - echo "Running the python script to generate tests..." - echo -1 | sudo tee /proc/sys/kernel/perf_event_paranoid - ./scripts/generate_multiple_tables.py + echo "Building project with g++ and running the benchmarks..." + CXX=g++ cmake -B build . && cmake --build build + ./scripts/generate_multiple_tables.py g++ + + rm -rf build + + echo "Building project with clang++ and running the benchmarks..." + CXX=clang++ cmake -B build . && cmake --build build + ./scripts/generate_multiple_tables.py clang++ EOF echo "Script executed successfully on ${INSTANCE_NAME}" diff --git a/scripts/generate_multiple_tables.py b/scripts/generate_multiple_tables.py index 546e8b4..a118550 100755 --- a/scripts/generate_multiple_tables.py +++ b/scripts/generate_multiple_tables.py @@ -2,6 +2,7 @@ import subprocess import os import platform +import sys from latex_table import generate_latex_table # Configuration @@ -27,6 +28,12 @@ ['-F6', '-s'], ] +# Get compiler label from command line +if len(sys.argv) < 2: + print("Usage: ./scripts/generate_multiple_tables.py ") + sys.exit(1) +CompilerLabel = sys.argv[1] + def get_cpu_model(): system = platform.system() @@ -71,7 +78,7 @@ def process_job(label, cmd_args, flags): # Build output file name flag_label = ''.join([f.strip('-') for f in flags]) or 'none' safe_label = label.replace('.', '_') - filename = f"{CPUModel}_{safe_label}_{flag_label}.tex" + filename = f"{CPUModel}_{CompilerLabel}_{safe_label}_{flag_label}.tex" out_path = os.path.join(output_dir, filename) # Write to file From 7847a349ec4faf671d68ee5ec97cb952143cd0de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ja=C3=ABl=20Champagne=20Gareau?= Date: Mon, 23 Jun 2025 23:32:08 -0400 Subject: [PATCH 08/10] save CPU, g++ and clang++ info on files --- scripts/aws_tests.bash | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/scripts/aws_tests.bash b/scripts/aws_tests.bash index add9b66..15cd34d 100755 --- a/scripts/aws_tests.bash +++ b/scripts/aws_tests.bash @@ -58,6 +58,11 @@ process_instance() { g++ clang cmake python3 echo -1 | sudo tee /proc/sys/kernel/perf_event_paranoid + echo "Saving some info about the environment..." + mkdir -p outputs + lscpu > outputs/lscpu.txt + g++ --version > outputs/g++.txt + clang++ --version > outputs/clang++.txt echo "Building project with g++ and running the benchmarks..." CXX=g++ cmake -B build . && cmake --build build From 0861e2726413aa8aedbe843a8ff3b76c68b3643d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ja=C3=ABl=20Champagne=20Gareau?= Date: Tue, 24 Jun 2025 01:08:31 -0400 Subject: [PATCH 09/10] fix typo in rsync folder path --- scripts/aws_tests.bash | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/aws_tests.bash b/scripts/aws_tests.bash index 15cd34d..156deef 100755 --- a/scripts/aws_tests.bash +++ b/scripts/aws_tests.bash @@ -76,7 +76,7 @@ process_instance() { EOF echo "Script executed successfully on ${INSTANCE_NAME}" - mkdir -p "${PROJECT_DIR}/outputs/${INSTANCE_NAME}" + mkdir -p "./outputs/${INSTANCE_NAME}" rsync -avz --partial --progress -e "${SSH_COMMAND}" \ ubuntu@${PUBLIC_IP}:~/${PROJECT_DIR}/outputs/ ./outputs/${INSTANCE_NAME}/ From 804c3617d71bfc57936032754fa8ae0cf7aa20a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ja=C3=ABl=20Champagne=20Gareau?= Date: Thu, 26 Jun 2025 17:49:51 -0400 Subject: [PATCH 10/10] resolve all review comments Principally, this PR: - makes security_group and ssh_key configurable via env variables - generates them automatically if unspecified - adds some checks and comments - adds x86 instances. --- .gitignore | 1 + scripts/aws_tests.bash | 198 ++++++++++++++++++++++++++++++++++------- 2 files changed, 167 insertions(+), 32 deletions(-) diff --git a/.gitignore b/.gitignore index f08cad8..7752a08 100644 --- a/.gitignore +++ b/.gitignore @@ -6,4 +6,5 @@ outputs tags compile_commands.json .cache +*.log **/__pycache__ diff --git a/scripts/aws_tests.bash b/scripts/aws_tests.bash index 156deef..a20d174 100755 --- a/scripts/aws_tests.bash +++ b/scripts/aws_tests.bash @@ -1,25 +1,150 @@ #!/bin/bash -# Preliminary setup -# - Install AWS CLI v2 (perhaps it works on v1, untested): -# - on Arch Linux: paru -S aws-cli-v2 -# - Configure AWS Credentials: aws configure +# This script launches EC2 instances to benchmark your project. +# +# Requirements: +# - The programs `git`, `ssh`, `rsync`, and `aws` must be installed. +# - AWS CLI v2 installed and configured (`aws configure`) +# - An EC2-compatible SSH key must exist in AWS, or the script will generate one (and save locally). +# +# Required AWS IAM permissions: +# - ec2:RunInstances +# - ec2:TerminateInstances +# - ec2:DescribeInstances +# - ec2:DescribeVpcs +# - ec2:CreateSecurityGroup +# - ec2:DeleteSecurityGroup +# - ec2:AuthorizeSecurityGroupIngress +# +# Optional environment variables: +# AWS_KEY_NAME use an existing key pair instead of creating one +# AWS_SECURITY_GROUP use an existing SG instead of creating one + +set -euo pipefail + +# -------------------- +# User-configurable variables +# -------------------- + +# Ubuntu 24.04 AMI IDs for x86_64 and aarch64 architectures +declare -A AMI_MAP=( + ["x86_64"]="ami-020cba7c55df1f615" + ["aarch64"]="ami-07041441b708acbd6" +) + +# We need biggest (metal) instances to access perf events on x86 +INSTANCES_x86_64=( + "c5n.metal" # Skylake + "c6i.metal" # Ice Lake + "c7i.metal-24xl" # Sapphire Rapids + "c5a.24xlarge" # EPYC Zen 2 + "c6a.metal" # EPYC Zen 3 + "c7a.metal-48xl" # EPYC Zen 4 +) +INSTANCES_aarch64=( + "c6g.medium" # Graviton 2 - Neoverse N1 + "c7g.medium" # Graviton 3 - Neoverse V1 + "c8g.medium" # Graviton 4 - Neoverse V2 +) -set -e # Exit script on first error +VOLUME_SIZE=10 # in GB -AMI_ID_x86_64="ami-020cba7c55df1f615" -AMI_ID_aarch64="ami-07041441b708acbd6" +KEY_NAME="${AWS_KEY_NAME:-aws_auto}" # Key path is assumed to be ~/.ssh/${KEY_NAME}.pem +SECURITY_GROUP="${AWS_SECURITY_GROUP:-}" -INSTANCES_x86_64=() -INSTANCES_aarch64=("c6g.medium" "c7g.medium" "c8g.medium") +# -------------------- +# Internal variables (do not modify) +# -------------------- -KEY_NAME="openssh" -KEY_PATH="~/.ssh/openssh.pem" +KEY_PATH="$HOME/.ssh/${KEY_NAME}.pem" SSH_COMMAND="ssh -i ${KEY_PATH} -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" -SECURITY_GROUP="sg-0315466a0d7fc99e1" -VOLUME_SIZE=10 # in GB -PROJECT_DIR=$(basename $(pwd)) +PROJECT_DIR=$(basename "$(git rev-parse --show-toplevel)") +CREATED_SECURITY_GROUP="" + +# Cleanup function to delete created security group on exit +cleanup() { + if [ -n "${CREATED_SECURITY_GROUP}" ]; then + echo "Cleaning up security group: ${CREATED_SECURITY_GROUP}" + aws ec2 delete-security-group --group-id "${CREATED_SECURITY_GROUP}" || true + fi +} + +check_prerequisites() { + if ((BASH_VERSINFO[0] < 4)); then + echo "Error: This script requires Bash version 4 or higher." >&2 + exit 1 + fi + + for cmd in git ssh rsync aws; do + if ! command -v "$cmd" >/dev/null 2>&1; then + echo "Error: Required command '$cmd' is not installed." >&2 + exit 1 + fi + done + + if ! git rev-parse --show-toplevel >/dev/null 2>&1; then + echo "Error: This script must be run from within a Git repository." >&2 + exit 1 + fi + + if ! aws sts get-caller-identity >/dev/null 2>&1; then + echo "Error: AWS credentials not configured. Run 'aws configure'." >&2 + exit 1 + fi +} + +create_key_pair() { + if aws ec2 describe-key-pairs --key-names "${KEY_NAME}" >/dev/null 2>&1; then + echo "Using existing AWS key pair: ${KEY_NAME}" + return + fi + + echo "Creating new AWS key pair named ${KEY_NAME}" + mkdir -p ~/.ssh + aws ec2 create-key-pair --key-name "$KEY_NAME" \ + --query 'KeyMaterial' --output text > "$KEY_PATH" + chmod 400 "$KEY_PATH" + echo "Created and saved key pair private key to $KEY_PATH" +} + +create_security_group() { + if [ -n "${SECURITY_GROUP}" ]; then + echo "Using existing security group: ${SECURITY_GROUP}" + return + fi + + echo "Creating a new security group for SSH access..." + VPC_ID=$(aws ec2 describe-vpcs \ + --filters Name=isDefault,Values=true \ + --query "Vpcs[0].VpcId" \ + --output text) + + CREATED_SECURITY_GROUP=$(aws ec2 create-security-group \ + --group-name ssh-public-access \ + --description "Allow SSH access from anywhere (0.0.0.0/0)" \ + --vpc-id "${VPC_ID}" \ + --query "GroupId" \ + --output text) + + aws ec2 authorize-security-group-ingress \ + --group-id "${CREATED_SECURITY_GROUP}" \ + --protocol tcp \ + --port 22 \ + --cidr 0.0.0.0/0 + + SECURITY_GROUP="${CREATED_SECURITY_GROUP}" + echo "Created security group: ${SECURITY_GROUP}" +} + +get_arch() { + local instance_name="$1" + if printf '%s\n' "${INSTANCES_aarch64[@]}" | grep -qx "$instance_name"; then + echo "aarch64" + else + echo "x86_64" + fi +} process_instance() { INSTANCE_NAME=$1 @@ -41,8 +166,7 @@ process_instance() { PUBLIC_IP=$(aws ec2 describe-instances \ --instance-ids ${INSTANCE_ID} \ - --query "Reservations[0].Instances[0].PublicIpAddress" \ - --output text) + --query "Reservations[0].Instances[0].PublicIpAddress" --output text) echo "Instance ${INSTANCE_ID} public IP: ${PUBLIC_IP}" git ls-files -z | rsync -avz --partial --progress --from0 --files-from=- -e "${SSH_COMMAND}" \ @@ -53,10 +177,12 @@ process_instance() { echo "Updating and installing dependencies on ${INSTANCE_NAME}..." sudo apt update - sudo DEBIAN_FRONTEND=noninteractive \ - apt install -y linux-tools-common linux-tools-generic \ - g++ clang cmake python3 - echo -1 | sudo tee /proc/sys/kernel/perf_event_paranoid + sudo DEBIAN_FRONTEND=noninteractive apt install -y \ + linux-tools-common linux-tools-generic g++ clang cmake python3 + + # Enable access to perf events for benchmarking + # Must use `sudo tee` since shell redirection (`>`) is not affected by sudo + echo -1 | sudo tee /proc/sys/kernel/perf_event_paranoid > /dev/null echo "Saving some info about the environment..." mkdir -p outputs @@ -84,17 +210,25 @@ EOF echo "Terminated instance: ${INSTANCE_ID}" } -echo "Launching ${#INSTANCES_aarch64[@]} aarch64 instances and ${#INSTANCES_x86_64[@]} x86_64 instances in parallel..." -for INSTANCE_NAME in "${INSTANCES_x86_64[@]}" "${INSTANCES_aarch64[@]}"; do - if printf '%s\n' "${INSTANCES_aarch64[@]}" | grep -qx "${INSTANCE_NAME}"; then - AMI_ID=${AMI_ID_aarch64} - else - AMI_ID=${AMI_ID_x86_64} - fi +main () { + trap cleanup EXIT + check_prerequisites + create_key_pair + create_security_group - process_instance "${INSTANCE_NAME}" "${AMI_ID}" 2>&1 | tee "${INSTANCE_NAME}.log" & -done + echo "Launching ${#INSTANCES_aarch64[@]} aarch64 instances and ${#INSTANCES_x86_64[@]} x86_64 instances in parallel..." + for INSTANCE_NAME in "${INSTANCES_x86_64[@]}" "${INSTANCES_aarch64[@]}"; do + ARCH=$(get_arch "$INSTANCE_NAME") + AMI_ID="${AMI_MAP[$ARCH]}" + + process_instance "${INSTANCE_NAME}" "${AMI_ID}" 2>&1 | tee "${INSTANCE_NAME}.log" & + done + + # Wait for all background jobs to finish + wait + echo "All instances completed." +} -# Wait for all background jobs to finish -wait -echo "All instances completed." +if [ "$0" = "$BASH_SOURCE" ] ; then + main +fi