diff --git a/terraform/uv-threads-benchmarks/client-cloud-init-memtier.yaml b/terraform/uv-threads-benchmarks/client-cloud-init-memtier.yaml new file mode 100644 index 0000000..305f1fa --- /dev/null +++ b/terraform/uv-threads-benchmarks/client-cloud-init-memtier.yaml @@ -0,0 +1,46 @@ +#cloud-config + +package_update: true +package_upgrade: true +packages: + - git + - build-essential + - autoconf + - automake + - libpcre3-dev + - libevent-dev + - pkg-config + - zlib1g-dev + - libssl-dev + - libtool + - ca-certificates + - wget + - zip + +# Create the memtier installation script +write_files: + - path: /tmp/install_memtier.sh + permissions: "0755" + content: | + #!/bin/bash + + set -e # exit immediately on error + + # Clone memtier benchmark + cd /tmp + rm -rf memtier_benchmark + git clone https://github.com/RedisLabs/memtier_benchmark.git + cd memtier_benchmark + + # Build and install + autoreconf -ivf + ./configure + make -j + sudo make install + + echo "Memtier benchmark installed successfully" + +# Run the installation script +runcmd: + - [bash, /tmp/install_memtier.sh] + - [echo, "Cloud-init installation completed"] diff --git a/terraform/uv-threads-benchmarks/client-resources.tf b/terraform/uv-threads-benchmarks/client-resources.tf new file mode 100644 index 0000000..e071290 --- /dev/null +++ b/terraform/uv-threads-benchmarks/client-resources.tf @@ -0,0 +1,32 @@ +resource "aws_instance" "client" { + ami = var.client_instance_ami + instance_type = var.client_instance_type + subnet_id = data.terraform_remote_state.shared_resources.outputs.subnet_public_us_east_2b_id + vpc_security_group_ids = ["${data.terraform_remote_state.shared_resources.outputs.performance_cto_sg_id}"] + key_name = var.key_name + placement_group = data.terraform_remote_state.shared_resources.outputs.placement_group_name_us_east_2b + availability_zone = "us-east-2b" + + user_data = file("./client-cloud-init-memtier.yaml") # Use cloud-init to install memtier benchmark + + root_block_device { + volume_size = var.instance_volume_size + volume_type = var.instance_volume_type + encrypted = var.instance_volume_encrypted + delete_on_termination = true + tags = merge( + local.base_tags, + { + Name = "ebs_block_device-${var.setup_name}-CLIENT" + } + ) + } + + tags = merge( + local.base_tags, + { + Name = "${var.setup_name}-CLIENT" + InstanceType = var.client_instance_type + } + ) +} diff --git a/terraform/uv-threads-benchmarks/common.tf b/terraform/uv-threads-benchmarks/common.tf new file mode 100644 index 0000000..82e5498 --- /dev/null +++ b/terraform/uv-threads-benchmarks/common.tf @@ -0,0 +1,11 @@ + +################################################################################ +# This is the bucket holding this specific setup tfstate +################################################################################ +terraform { + backend "s3" { + bucket = "performance-cto-group" + key = "benchmarks/infrastructure/uv-threads-benchmarks.tfstate" + region = "us-east-1" + } +} diff --git a/terraform/uv-threads-benchmarks/db-cloud-init-dbs.yaml b/terraform/uv-threads-benchmarks/db-cloud-init-dbs.yaml new file mode 100644 index 0000000..4040311 --- /dev/null +++ b/terraform/uv-threads-benchmarks/db-cloud-init-dbs.yaml @@ -0,0 +1,92 @@ +#cloud-config + +package_update: true +package_upgrade: true +packages: + - git + - dpkg-dev + - gcc + - g++ + - libc6-dev + - libssl-dev + - make + - cmake + - clang + - automake + - autoconf + - libtool + - ca-certificates + - wget + - zip +# Parca agent configuration +snap: + commands: + - [install, parca-agent, --classic] + - [ + set, + parca-agent, + remote-store-bearer-token=XXX-YOUR-TOKEN-XXX, + ] + - [start, --enable, parca-agent] +# Create the Redis installation script +write_files: + - path: /tmp/install_redis.sh + permissions: "0755" + content: | + #!/bin/bash + + set -e # exit immediately on error + + GH_ORG=${1:-"redis"} + GH_REPO=${2:-"redis"} + COMMIT=${3:-"HEAD"} + FOLDER=${4:-${GH_REPO}} + + echo "Installing ${GH_ORG}/${GH_REPO} at commit ${COMMIT} in folder ${FOLDER}" + rm -rf ${FOLDER} + git clone https://github.com/${GH_ORG}/${GH_REPO} ${FOLDER} + + cd ${FOLDER} + git checkout ${COMMIT} + + make distclean + make BUILD_TLS=yes -j + sudo make install + + echo "Installed successfully" + + - path: /tmp/compile_redisearch.sh + permissions: "0755" + content: | + #!/bin/bash + + set -e # exit immediately on error + + GH_ORG=${1:-"RediSearch"} + GH_REPO=${2:-"RediSearch"} + COMMIT=${3:-"HEAD"} + FOLDER=${4:-${GH_REPO}} + + echo "Installing ${GH_ORG}/${GH_REPO} at commit ${COMMIT} in folder ${FOLDER}" + rm -rf ${FOLDER} + git clone https://github.com/${GH_ORG}/${GH_REPO} ${FOLDER} + + cd ${FOLDER} + git checkout ${COMMIT} + git submodule update --init --recursive + + cd .install/ + sudo ./install_script.sh + ./install_rust.sh + source $HOME/.cargo/env + cd .. + ./build.sh + + echo "Compiled successfully" + + +# Run the installation script +runcmd: + - [su, ubuntu, -c, "/tmp/install_redis.sh redis redis unstable ~/redis-unstable"] + - [su, ubuntu, -c, "/tmp/compile_redisearch.sh RediSearch RediSearch master ~/RediSearch"] + - [echo, "Cloud-init installation completed"] diff --git a/terraform/uv-threads-benchmarks/db-resources.tf b/terraform/uv-threads-benchmarks/db-resources.tf new file mode 100644 index 0000000..0d701dd --- /dev/null +++ b/terraform/uv-threads-benchmarks/db-resources.tf @@ -0,0 +1,34 @@ +resource "aws_instance" "server" { + for_each = var.server_instances_configs # Create instances for each architecture + ami = each.value.ami + instance_type = each.value.instance_type + subnet_id = data.terraform_remote_state.shared_resources.outputs.subnet_public_us_east_2b_id + vpc_security_group_ids = ["${data.terraform_remote_state.shared_resources.outputs.performance_cto_sg_id}"] + key_name = var.key_name + placement_group = data.terraform_remote_state.shared_resources.outputs.placement_group_name_us_east_2b + availability_zone = "us-east-2b" + + user_data = file("./db-cloud-init-dbs.yaml") # Use cloud-init to install Redis versions + + root_block_device { + volume_size = var.instance_volume_size + volume_type = var.instance_volume_type + encrypted = var.instance_volume_encrypted + delete_on_termination = true + tags = merge( + local.base_tags, + { + Name = "ebs_block_device-${var.setup_name}-${each.value.arch_label}" + } + ) + } + + tags = merge( + local.base_tags, + { + Name = "${var.setup_name}-${each.value.arch_label}" + Architecture = each.value.arch_label + InstanceType = each.value.instance_type + } + ) +} diff --git a/terraform/uv-threads-benchmarks/output.tf b/terraform/uv-threads-benchmarks/output.tf new file mode 100644 index 0000000..82f5cf4 --- /dev/null +++ b/terraform/uv-threads-benchmarks/output.tf @@ -0,0 +1,19 @@ +output "all_server_public_ips" { + value = { + for k, v in var.server_instances_configs : v.name => aws_instance.server[k].public_ip + } +} + +output "all_server_private_ips" { + value = { + for k, v in var.server_instances_configs : v.name => aws_instance.server[k].private_ip + } +} + +output "client_public_ip" { + value = ["${aws_instance.client[*].public_ip}"] +} + +output "client_private_ip" { + value = ["${aws_instance.client[*].private_ip}"] +} diff --git a/terraform/uv-threads-benchmarks/shared_resources.tf b/terraform/uv-threads-benchmarks/shared_resources.tf new file mode 100644 index 0000000..824e6b9 --- /dev/null +++ b/terraform/uv-threads-benchmarks/shared_resources.tf @@ -0,0 +1,17 @@ +# provider +provider "aws" { + region = var.region +} + +################################################################################ +# This is the shared resources bucket key -- you will need it across environments like security rules,etc... +# !! do not change this !! +################################################################################ +data "terraform_remote_state" "shared_resources" { + backend = "s3" + config = { + bucket = "performance-cto-group" + key = "benchmarks/infrastructure/shared_resources.tfstate" + region = "us-east-1" + } +} diff --git a/terraform/uv-threads-benchmarks/variables.tf b/terraform/uv-threads-benchmarks/variables.tf new file mode 100644 index 0000000..64560ca --- /dev/null +++ b/terraform/uv-threads-benchmarks/variables.tf @@ -0,0 +1,133 @@ +################################################################################ +# Variables used for deployment tag +################################################################################ + +variable "setup_name" { + description = "setup name" + default = "uv-threads-benchmarks" +} + +variable "github_actor" { + description = "The name of the person or app that initiated the deployment." + default = "N/A" +} + +variable "github_org" { + description = "The owner name. For example, RedisModules." + default = "N/A" +} + +variable "github_repo" { + description = "The owner and repository name. For example, octocat/Hello-World." + default = "N/A" +} + +variable "github_sha" { + description = "The commit SHA that triggered the deployment." + default = "N/A" +} + +variable "timeout_secs" { + description = "The maximum time to wait prior destroying the VM via the watchdog." + default = "3600" +} + +variable "triggering_env" { + description = "The triggering environment. For example circleci." + default = "N/A" +} + +variable "environment" { + description = "The cost tag." + default = "N/A" +} + +variable "redis_module" { + description = "redis_module" + default = "N/A" +} + +################################################################################ +# Instance(s) options +################################################################################ + +variable "region" { + default = "us-east-2" +} + +variable "server_instances_configs" { + description = "Configuration for each instance type" + type = map(object({ + name = string + instance_type = string + ami = string + arch_label = string + })) + default = { + "intel" = { + name = "Intel m7i.24xlarge" + instance_type = "m7i.24xlarge" + ami = "ami-04f167a56786e4b09" # Ubuntu 24.04 LTS x86_64 + arch_label = "INTEL" + } + } +} + +variable "client_instance_ami" { + description = "AMI for aws EC2 instance - Ubuntu 24.04 LTS x86_64" + default = "ami-04f167a56786e4b09" +} + +variable "client_instance_type" { + description = "type for aws EC2 instance" + default = "c6in.8xlarge" +} + +variable "instance_volume_size" { + description = "EC2 instance volume_size" + default = "128" +} + +variable "instance_volume_type" { + description = "EC2 instance volume_type" + default = "gp3" +} + + +variable "instance_volume_encrypted" { + description = "EC2 instance instance_volume_encrypted" + default = "false" +} + +################################################################################ +# SSH Access +################################################################################ +variable "private_key" { + description = "private key" + default = "/tmp/benchmarks.redislabs.pem" +} + +variable "key_name" { + description = "key name" + default = "perf-cto-us-east-2" +} + +variable "ssh_user" { + description = "ssh_user" + default = "ubuntu" +} + +# Define base tags that are common to multiple resources +locals { + base_tags = { + Environment = var.environment + setup = var.setup_name + redis_module = var.redis_module + triggering_env = var.triggering_env + github_actor = var.github_actor + github_org = var.github_org + github_repo = var.github_repo + github_sha = var.github_sha + timeout_secs = var.timeout_secs + } +}