From c5e4033e1879c2a4fff044cb204aa0dd6cf20612 Mon Sep 17 00:00:00 2001 From: Kevin Kozlowski Date: Thu, 28 Jan 2021 11:10:04 -0700 Subject: [PATCH 1/3] First working draft of benchmarking terraform ec2 setup --- .gitignore | 1 + terraform/ngff-benchmark.tf | 134 +++++++++++++++++++++++++++++ terraform/terraform.tfvars.example | 2 + terraform/tfbackend.config.example | 4 + 4 files changed, 141 insertions(+) create mode 100644 terraform/ngff-benchmark.tf create mode 100644 terraform/terraform.tfvars.example create mode 100644 terraform/tfbackend.config.example diff --git a/.gitignore b/.gitignore index 0344bf0..72d0879 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ __pycache__ data/*.ims data/*.tiff data/*.zarr +*.swp diff --git a/terraform/ngff-benchmark.tf b/terraform/ngff-benchmark.tf new file mode 100644 index 0000000..e7c3e12 --- /dev/null +++ b/terraform/ngff-benchmark.tf @@ -0,0 +1,134 @@ + +provider "aws" { + region = "us-east-1" +} + +terraform { + backend "s3" {} +} + +variable "ssh_client_ip" { + type = string +} + +variable "ssh_public_key" { + type = string +} + +resource "aws_vpc" "vpc" { + cidr_block = "10.0.0.0/16" + enable_dns_support = true + enable_dns_hostnames = true + tags = { + Name = "ngff-benchmarking-vpc" + } +} + +resource "aws_internet_gateway" "gw" { + vpc_id = aws_vpc.vpc.id + tags = { + Name = "ngff-benchmarking-ig" + } +} + +resource "aws_route_table" "route_table" { + vpc_id = aws_vpc.vpc.id + + route { + cidr_block = "0.0.0.0/0" + gateway_id = aws_internet_gateway.gw.id + } + + tags = { + Name = "ngff-benchmarking-rt" + } +} + +resource "aws_subnet" "subnet" { + vpc_id = aws_vpc.vpc.id + cidr_block = "10.0.1.0/24" + map_public_ip_on_launch = true + + tags = { + Name = "ngff-benchmarking-subnet" + } +} + +resource "aws_route_table_association" "rt_association" { + subnet_id = aws_subnet.subnet.id + route_table_id = aws_route_table.route_table.id +} + +resource "aws_security_group" "security_group" { + name = "benchmarking_security_group" + vpc_id = aws_vpc.vpc.id + + ingress { + description = "TLS from VPC" + from_port = 443 + to_port = 443 + protocol = "tcp" + cidr_blocks = [aws_vpc.vpc.cidr_block] + } + + ingress { + description = "SSH" + from_port = 22 + to_port = 22 + protocol = "tcp" + cidr_blocks = ["${var.ssh_client_ip}"] + } + + tags = { + Name = "benchmarking_security_group" + } +} + +data "aws_ami" "latest-ubuntu" { + most_recent = true + owners = ["099720109477"] # Canonical + + filter { + name = "name" + values = ["ubuntu/images/hvm-ssd/ubuntu-bionic-18.04-amd64-server-*"] + } + + filter { + name = "virtualization-type" + values = ["hvm"] + } +} + +resource "aws_key_pair" "ngffkey" { + key_name = "ngff-key" + public_key = "${var.ssh_public_key}" +} + +resource "aws_instance" "nginx_instance" { + ami = "${data.aws_ami.latest-ubuntu.id}" + instance_type = "t2.micro" + subnet_id = aws_subnet.subnet.id + vpc_security_group_ids = [aws_security_group.security_group.id] + root_block_device { + volume_size = 128 + } + key_name = aws_key_pair.ngffkey.key_name + tags = { + Name = "ngff-benchmarking-server" + } +} + +resource "aws_instance" "client_instance" { + ami = "${data.aws_ami.latest-ubuntu.id}" + instance_type = "t2.micro" + subnet_id = aws_subnet.subnet.id + vpc_security_group_ids = [aws_security_group.security_group.id] + root_block_device { + volume_size = 128 + } + key_name = aws_key_pair.ngffkey.key_name + tags = { + Name = "ngff-benchmarking-client" + } +} + diff --git a/terraform/terraform.tfvars.example b/terraform/terraform.tfvars.example new file mode 100644 index 0000000..c3fd7d0 --- /dev/null +++ b/terraform/terraform.tfvars.example @@ -0,0 +1,2 @@ +ssh_client_ip = "1.2.3.4/32" +ssh_public_key = "ssh-rsa abcdefg..." diff --git a/terraform/tfbackend.config.example b/terraform/tfbackend.config.example new file mode 100644 index 0000000..0b71dce --- /dev/null +++ b/terraform/tfbackend.config.example @@ -0,0 +1,4 @@ +bucket = "dev-ngff-bench-tfstate" +key = "terraform.tfstate" +region = "us-east-1" +encrypt = true From 545c7aebf6fc0c965f2eb3a86723526b51ceced9 Mon Sep 17 00:00:00 2001 From: Kevin Kozlowski Date: Fri, 29 Jan 2021 10:23:15 -0700 Subject: [PATCH 2/3] Output public DNS names and add README.md --- terraform/README.md | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 terraform/README.md diff --git a/terraform/README.md b/terraform/README.md new file mode 100644 index 0000000..e2ffd22 --- /dev/null +++ b/terraform/README.md @@ -0,0 +1,19 @@ +# NGFF Benchmarking setup with Terraform + +## Setup +* First, you will need to create an aws account. In this account, you will need an IAM user created with CLI access and admin privileges. These will be the credentials terraform will use to create resources on your behalf. You will want these credentials in the `~/.aws/credentials` file. You can set them as the default or create a terraform aws profile for them (see https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-profiles.html). +* The next thing you'll need to do is create a bucket for terraform to store `.tfstate` files in. These are files terraform uses to keep track of the state of your infrastructure, so that it can add, modify, and destroy compenents correclty. You can keep these files on your local machine, but then there will be issues if other users want to use terraform to modify your setup. You can name this bucket anything you like. +* In order to SSH into your instances, you'll need an RSA key pair. Generate one with a command like `ssh-keygen -l -f .ssh/aws.pem` (see Option 2 in https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-key-pairs.html for details). +* Install terraform https://learn.hashicorp.com/tutorials/terraform/install-cli + +## Run +* Clone this repo and `cd` into the `terraform` directory. +* Rename the files `tfbackend.config.example` and `terraform.tfvars.example` to remove the `.example` and edit them to have the correct values for your needs. The `bucket` in `tfbackend.config` should be the name of the bucket where you plan to store `.tfstate` files. In `terraform.tfvars`, `ssh_client_ip` should be the IP address you intend to SSH into your ec2 instances from, or `0.0.0.0/0` if you want to be able to SSH in from anywhere. `ssh_public_key` should be the public key of the `.pem` file you generated earlier. +* Initialize terraform by running `terraform init`. +* Run `terraform plan` to make show you everything that will be created and confirm that this is correct. +* Run `terraform apply` + +After this, the resources should be available (you can double-check in the aws console) and you should be able to SSH into your new hosts by looking up the DNS (either in the console or by running `terraform output`) and running `ssh -i .pem ubuntu@` + +## Tear Down +To remove the resources you created here, just run `terraform destroy` From 8e59d1d0af67014449efeb8e164312d56f8d513e Mon Sep 17 00:00:00 2001 From: jmoore Date: Thu, 25 Feb 2021 09:34:47 +0100 Subject: [PATCH 3/3] Minor changes after a first run --- terraform/README.md | 2 +- terraform/ngff-benchmark.tf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/terraform/README.md b/terraform/README.md index e2ffd22..7e9ef6f 100644 --- a/terraform/README.md +++ b/terraform/README.md @@ -9,7 +9,7 @@ ## Run * Clone this repo and `cd` into the `terraform` directory. * Rename the files `tfbackend.config.example` and `terraform.tfvars.example` to remove the `.example` and edit them to have the correct values for your needs. The `bucket` in `tfbackend.config` should be the name of the bucket where you plan to store `.tfstate` files. In `terraform.tfvars`, `ssh_client_ip` should be the IP address you intend to SSH into your ec2 instances from, or `0.0.0.0/0` if you want to be able to SSH in from anywhere. `ssh_public_key` should be the public key of the `.pem` file you generated earlier. -* Initialize terraform by running `terraform init`. +* Initialize terraform by running `terraform init --backend-config=tfbackend.config`. * Run `terraform plan` to make show you everything that will be created and confirm that this is correct. * Run `terraform apply` diff --git a/terraform/ngff-benchmark.tf b/terraform/ngff-benchmark.tf index e7c3e12..ac28d74 100644 --- a/terraform/ngff-benchmark.tf +++ b/terraform/ngff-benchmark.tf @@ -1,6 +1,6 @@ provider "aws" { - region = "us-east-1" + region = "eu-west-1" } terraform {