diff --git a/.gitignore b/.gitignore index f08cad8..7752a08 100644 --- a/.gitignore +++ b/.gitignore @@ -6,4 +6,5 @@ outputs tags compile_commands.json .cache +*.log **/__pycache__ diff --git a/benchmarks/benchmark.cpp b/benchmarks/benchmark.cpp index 9319852..32ea36d 100644 --- a/benchmarks/benchmark.cpp +++ b/benchmarks/benchmark.cpp @@ -375,7 +375,7 @@ int main(int argc, char **argv) { fmt::println(" ./benchmark --file=data/canada.txt # Run benchmark using numbers from a file"); fmt::println(" ./benchmark --fixed=10 # Test fixed-point representation instead of shortest length"); fmt::println(" ./benchmark --test # Test correctness instead of performance"); - fmt::println(" ./benchmark --volume=1000 --model=uniform # Generate 1000 uniform random numbers"); + fmt::println(" ./benchmark --volume=1000 --model=uniform_01 # Generate 1000 uniform random numbers in [0, 1]"); fmt::println(" ./benchmark --algo-filter=ryu,grisu # Only test algorithms containing 'ryu' or 'grisu'"); fmt::println("\nFor full options list, run: ./benchmark --help"); return EXIT_FAILURE; diff --git a/benchmarks/random_generators.h b/benchmarks/random_generators.h index d6f37ac..7458156 100644 --- a/benchmarks/random_generators.h +++ b/benchmarks/random_generators.h @@ -30,6 +30,24 @@ struct uniform_generator : float_number_generator { T new_float() override { return dis(gen); } }; +template +struct logspace_generator : float_number_generator { + std::random_device rd; + std::mt19937_64 gen; + std::uniform_int_distribution exp; + std::uniform_real_distribution significand; + explicit logspace_generator() + : rd(), gen(rd()), + exp(std::numeric_limits::min_exponent + 1, // +1 skips subnormals + std::numeric_limits::max_exponent), + significand(-1, 1) {} + std::string describe() override { + return "Generate random numbers uniformly in log2 space, i.e. " + "magnitudes uniformly distributed in the interval [-2^max_exponent, 2^max_exponent]"; + } + T new_float() override { return significand(gen) * std::pow(2.0, exp(gen)); } +}; + enum centering { centered, non_centered }; template struct centered_generator : float_number_generator { @@ -112,29 +130,26 @@ struct one_over_rand : float_number_generator { }; constexpr std::array model_names = { - "uniform_01" , "uniform_all" , "integer_uniform" , - "centered" , "non_centered" , - "simple_uniform" , "simple_int" , - "one_over_rand" + "uniform_01" , "logspace_all" , + "centered" , "non_centered" , + "simple_uniform" , "simple_int" , + "one_over_rand" , "integer_uniform" , }; template inline std::unique_ptr> -get_generator_by_name(std::string name) { +get_generator_by_name(const std::string name) { std::cout << "available models (-m): "; - for (std::string name : model_names) { - std::cout << name << " "; + for (const auto& model : model_names) { + std::cout << model << " "; } std::cout << std::endl; // This is naive, but also not very important. if (name == "uniform_01") return std::unique_ptr>(new uniform_generator()); - if (name == "uniform_all") { - return std::unique_ptr>( - new uniform_generator(std::numeric_limits::lowest(), - std::numeric_limits::max()) - ); + if (name == "logspace_all") { + return std::unique_ptr>(new logspace_generator()); } if (name == "centered") return std::unique_ptr>(new centered_generator()); diff --git a/scripts/aws_tests.bash b/scripts/aws_tests.bash new file mode 100755 index 0000000..a20d174 --- /dev/null +++ b/scripts/aws_tests.bash @@ -0,0 +1,234 @@ +#!/bin/bash + +# This script launches EC2 instances to benchmark your project. +# +# Requirements: +# - The programs `git`, `ssh`, `rsync`, and `aws` must be installed. +# - AWS CLI v2 installed and configured (`aws configure`) +# - An EC2-compatible SSH key must exist in AWS, or the script will generate one (and save locally). +# +# Required AWS IAM permissions: +# - ec2:RunInstances +# - ec2:TerminateInstances +# - ec2:DescribeInstances +# - ec2:DescribeVpcs +# - ec2:CreateSecurityGroup +# - ec2:DeleteSecurityGroup +# - ec2:AuthorizeSecurityGroupIngress +# +# Optional environment variables: +# AWS_KEY_NAME use an existing key pair instead of creating one +# AWS_SECURITY_GROUP use an existing SG instead of creating one + +set -euo pipefail + +# -------------------- +# User-configurable variables +# -------------------- + +# Ubuntu 24.04 AMI IDs for x86_64 and aarch64 architectures +declare -A AMI_MAP=( + ["x86_64"]="ami-020cba7c55df1f615" + ["aarch64"]="ami-07041441b708acbd6" +) + +# We need biggest (metal) instances to access perf events on x86 +INSTANCES_x86_64=( + "c5n.metal" # Skylake + "c6i.metal" # Ice Lake + "c7i.metal-24xl" # Sapphire Rapids + "c5a.24xlarge" # EPYC Zen 2 + "c6a.metal" # EPYC Zen 3 + "c7a.metal-48xl" # EPYC Zen 4 +) +INSTANCES_aarch64=( + "c6g.medium" # Graviton 2 - Neoverse N1 + "c7g.medium" # Graviton 3 - Neoverse V1 + "c8g.medium" # Graviton 4 - Neoverse V2 +) + +VOLUME_SIZE=10 # in GB + +KEY_NAME="${AWS_KEY_NAME:-aws_auto}" # Key path is assumed to be ~/.ssh/${KEY_NAME}.pem +SECURITY_GROUP="${AWS_SECURITY_GROUP:-}" + +# -------------------- +# Internal variables (do not modify) +# -------------------- + +KEY_PATH="$HOME/.ssh/${KEY_NAME}.pem" +SSH_COMMAND="ssh -i ${KEY_PATH} -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" + +PROJECT_DIR=$(basename "$(git rev-parse --show-toplevel)") +CREATED_SECURITY_GROUP="" + +# Cleanup function to delete created security group on exit +cleanup() { + if [ -n "${CREATED_SECURITY_GROUP}" ]; then + echo "Cleaning up security group: ${CREATED_SECURITY_GROUP}" + aws ec2 delete-security-group --group-id "${CREATED_SECURITY_GROUP}" || true + fi +} + +check_prerequisites() { + if ((BASH_VERSINFO[0] < 4)); then + echo "Error: This script requires Bash version 4 or higher." >&2 + exit 1 + fi + + for cmd in git ssh rsync aws; do + if ! command -v "$cmd" >/dev/null 2>&1; then + echo "Error: Required command '$cmd' is not installed." >&2 + exit 1 + fi + done + + if ! git rev-parse --show-toplevel >/dev/null 2>&1; then + echo "Error: This script must be run from within a Git repository." >&2 + exit 1 + fi + + if ! aws sts get-caller-identity >/dev/null 2>&1; then + echo "Error: AWS credentials not configured. Run 'aws configure'." >&2 + exit 1 + fi +} + +create_key_pair() { + if aws ec2 describe-key-pairs --key-names "${KEY_NAME}" >/dev/null 2>&1; then + echo "Using existing AWS key pair: ${KEY_NAME}" + return + fi + + echo "Creating new AWS key pair named ${KEY_NAME}" + mkdir -p ~/.ssh + aws ec2 create-key-pair --key-name "$KEY_NAME" \ + --query 'KeyMaterial' --output text > "$KEY_PATH" + chmod 400 "$KEY_PATH" + echo "Created and saved key pair private key to $KEY_PATH" +} + +create_security_group() { + if [ -n "${SECURITY_GROUP}" ]; then + echo "Using existing security group: ${SECURITY_GROUP}" + return + fi + + echo "Creating a new security group for SSH access..." + VPC_ID=$(aws ec2 describe-vpcs \ + --filters Name=isDefault,Values=true \ + --query "Vpcs[0].VpcId" \ + --output text) + + CREATED_SECURITY_GROUP=$(aws ec2 create-security-group \ + --group-name ssh-public-access \ + --description "Allow SSH access from anywhere (0.0.0.0/0)" \ + --vpc-id "${VPC_ID}" \ + --query "GroupId" \ + --output text) + + aws ec2 authorize-security-group-ingress \ + --group-id "${CREATED_SECURITY_GROUP}" \ + --protocol tcp \ + --port 22 \ + --cidr 0.0.0.0/0 + + SECURITY_GROUP="${CREATED_SECURITY_GROUP}" + echo "Created security group: ${SECURITY_GROUP}" +} + +get_arch() { + local instance_name="$1" + if printf '%s\n' "${INSTANCES_aarch64[@]}" | grep -qx "$instance_name"; then + echo "aarch64" + else + echo "x86_64" + fi +} + +process_instance() { + INSTANCE_NAME=$1 + AMI_ID=$2 + echo "Running instance for ${INSTANCE_NAME} with AMI ${AMI_ID}" + + INSTANCE_ID=$(aws ec2 run-instances \ + --image-id ${AMI_ID} \ + --instance-type ${INSTANCE_NAME} \ + --key-name ${KEY_NAME} \ + --block-device-mappings "DeviceName=/dev/sda1,Ebs={VolumeSize=${VOLUME_SIZE}}" \ + --associate-public-ip-address \ + --security-group-ids ${SECURITY_GROUP} \ + --count "1" --query 'Instances[0].InstanceId' --output text) + + echo "Waiting for instance ${INSTANCE_ID} to be ready..." + aws ec2 wait instance-status-ok --instance-ids ${INSTANCE_ID} + echo "Started instance: ${INSTANCE_ID}" + + PUBLIC_IP=$(aws ec2 describe-instances \ + --instance-ids ${INSTANCE_ID} \ + --query "Reservations[0].Instances[0].PublicIpAddress" --output text) + echo "Instance ${INSTANCE_ID} public IP: ${PUBLIC_IP}" + + git ls-files -z | rsync -avz --partial --progress --from0 --files-from=- -e "${SSH_COMMAND}" \ + ./ ubuntu@${PUBLIC_IP}:~/${PROJECT_DIR} + ${SSH_COMMAND} ubuntu@${PUBLIC_IP} << EOF + set -e # Exit on error + cd ~/${PROJECT_DIR} + + echo "Updating and installing dependencies on ${INSTANCE_NAME}..." + sudo apt update + sudo DEBIAN_FRONTEND=noninteractive apt install -y \ + linux-tools-common linux-tools-generic g++ clang cmake python3 + + # Enable access to perf events for benchmarking + # Must use `sudo tee` since shell redirection (`>`) is not affected by sudo + echo -1 | sudo tee /proc/sys/kernel/perf_event_paranoid > /dev/null + + echo "Saving some info about the environment..." + mkdir -p outputs + lscpu > outputs/lscpu.txt + g++ --version > outputs/g++.txt + clang++ --version > outputs/clang++.txt + + echo "Building project with g++ and running the benchmarks..." + CXX=g++ cmake -B build . && cmake --build build + ./scripts/generate_multiple_tables.py g++ + + rm -rf build + + echo "Building project with clang++ and running the benchmarks..." + CXX=clang++ cmake -B build . && cmake --build build + ./scripts/generate_multiple_tables.py clang++ +EOF + + echo "Script executed successfully on ${INSTANCE_NAME}" + mkdir -p "./outputs/${INSTANCE_NAME}" + rsync -avz --partial --progress -e "${SSH_COMMAND}" \ + ubuntu@${PUBLIC_IP}:~/${PROJECT_DIR}/outputs/ ./outputs/${INSTANCE_NAME}/ + + aws ec2 terminate-instances --instance-ids ${INSTANCE_ID} + echo "Terminated instance: ${INSTANCE_ID}" +} + +main () { + trap cleanup EXIT + check_prerequisites + create_key_pair + create_security_group + + echo "Launching ${#INSTANCES_aarch64[@]} aarch64 instances and ${#INSTANCES_x86_64[@]} x86_64 instances in parallel..." + for INSTANCE_NAME in "${INSTANCES_x86_64[@]}" "${INSTANCES_aarch64[@]}"; do + ARCH=$(get_arch "$INSTANCE_NAME") + AMI_ID="${AMI_MAP[$ARCH]}" + + process_instance "${INSTANCE_NAME}" "${AMI_ID}" 2>&1 | tee "${INSTANCE_NAME}.log" & + done + + # Wait for all background jobs to finish + wait + echo "All instances completed." +} + +if [ "$0" = "$BASH_SOURCE" ] ; then + main +fi diff --git a/scripts/generate_multiple_tables.py b/scripts/generate_multiple_tables.py index efdd318..a118550 100755 --- a/scripts/generate_multiple_tables.py +++ b/scripts/generate_multiple_tables.py @@ -2,11 +2,11 @@ import subprocess import os import platform +import sys from latex_table import generate_latex_table # Configuration benchmark_executable = './build/benchmarks/benchmark' -latex_script = './scripts/latex_table.py' output_dir = './outputs' input_files = [ 'data/canada.txt', @@ -14,13 +14,13 @@ ] models = [ 'uniform_01', - 'uniform_all', - 'integer_uniform', - 'centered', - 'non_centered', + # 'logspace_all', + # 'integer_uniform', + # 'centered', + # 'non_centered', ] -runs_r = 1_000 -volume_v = 1_000_000 +runs_r = 100 +volume_v = 100_000 flag_combinations = [ [], ['-F6'], @@ -28,20 +28,32 @@ ['-F6', '-s'], ] +# Get compiler label from command line +if len(sys.argv) < 2: + print("Usage: ./scripts/generate_multiple_tables.py ") + sys.exit(1) +CompilerLabel = sys.argv[1] + def get_cpu_model(): - if platform.system() == "Windows": + system = platform.system() + if system == "Windows": return platform.processor() - elif platform.system() == "Darwin": + elif system == "Darwin": os.environ['PATH'] = os.environ['PATH'] + os.pathsep + '/usr/sbin' - command = "sysctl -n machdep.cpu.brand_string" - return subprocess.check_output(command).strip() - elif platform.system() == "Linux": - command = "cat /proc/cpuinfo" - output = subprocess.check_output(command, shell=True).decode().strip() - for line in output.split("\n"): - if line.startswith("model name"): - return line.split(':', 1)[1].strip() + command = ["sysctl", "-n", "machdep.cpu.brand_string"] + return subprocess.check_output(command, text=True).strip() + elif system == "Linux": + output = subprocess.check_output(["lscpu"], text=True) + model_name = None + architecture = None + for line in output.splitlines(): + if "Model name:" in line: + model_name = line.split(":", 1)[1].strip() + elif "Architecture:" in line: + architecture = line.split(":", 1)[1].strip() + # Prefer model_name if available; fallback to architecture + return model_name or architecture or "unknown_cpu" return "unknown_cpu" @@ -60,20 +72,20 @@ def run_cmd(cmd): def process_job(label, cmd_args, flags): # Run the benchmark cmd = [benchmark_executable] + cmd_args + flags - print(f"Running: {' '.join(cmd)}") + print(f"Running: {' '.join(cmd)}", flush=True) output = run_cmd(cmd) # Build output file name flag_label = ''.join([f.strip('-') for f in flags]) or 'none' safe_label = label.replace('.', '_') - filename = f"{CPUModel}_{safe_label}_{flag_label}.tex" + filename = f"{CPUModel}_{CompilerLabel}_{safe_label}_{flag_label}.tex" out_path = os.path.join(output_dir, filename) # Write to file tex_content = generate_latex_table(output) with open(out_path, 'w') as f: f.write(tex_content) - print(f"Written: {out_path}\n") + print(f"Written: {out_path}\n", flush=True) if __name__ == '__main__':