Merge pull request #4 from kkoz/add-terraform

First working draft of benchmarking terraform ec2 setup

.gitignore

@@ -4,3 +4,4 @@ data/*.tiff
 data/*.zarr
 data/*.bfmemo
 data/tmp
+.swp

terraform/README.md

@@ -0,0 +1,19 @@
+# NGFF Benchmarking setup with Terraform
+
+## Setup
+* First, you will need to create an aws account. In this account, you will need an IAM user created with CLI access and admin privileges. These will be the credentials terraform will use to create resources on your behalf. You will want these credentials in the `~/.aws/credentials` file. You can set them as the default or create a terraform aws profile for them (see https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-profiles.html). 
+* The next thing you'll need to do is create a bucket for terraform to store `.tfstate` files in. These are files terraform uses to keep track of the state of your infrastructure, so that it can add, modify, and destroy compenents correclty. You can keep these files on your local machine, but then there will be issues if other users want to use terraform to modify your setup. You can name this bucket anything you like. 
+* In order to SSH into your instances, you'll need an RSA key pair. Generate one with a command like `ssh-keygen -l -f .ssh/aws.pem` (see Option 2 in https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-key-pairs.html for details). 
+* Install terraform https://learn.hashicorp.com/tutorials/terraform/install-cli
+
+## Run
+* Clone this repo and `cd` into the `terraform` directory.
+* Rename the files `tfbackend.config.example` and `terraform.tfvars.example` to remove the `.example` and edit them to have the correct values for your needs. The `bucket` in `tfbackend.config` should be the name of the bucket where you plan to store `.tfstate` files. In `terraform.tfvars`, `ssh_client_ip` should be the IP address you intend to SSH into your ec2 instances from, or `0.0.0.0/0` if you want to be able to SSH in from anywhere. `ssh_public_key` should be the public key of the `.pem` file you generated earlier. 
+* Initialize terraform by running `terraform init --backend-config=tfbackend.config`.
+* Run `terraform plan` to make show you everything that will be created and confirm that this is correct.
+* Run `terraform apply`
+
+After this, the resources should be available (you can double-check in the aws console) and you should be able to SSH into your new hosts by looking up the DNS (either in the console or by running `terraform output`) and running `ssh -i <your-private-key>.pem ubuntu@<your-public-dns>`
+
+## Tear Down
+To remove the resources you created here, just run `terraform destroy`

terraform/ngff-benchmark.tf

@@ -0,0 +1,134 @@
+
+provider "aws" {
+    region = "eu-west-1"
+}
+
+terraform {
+    backend "s3" {}
+}
+
+variable "ssh_client_ip" {
+    type    = string
+}
+
+variable "ssh_public_key" {
+    type    = string
+}
+
+resource "aws_vpc" "vpc" {
+  cidr_block = "10.0.0.0/16"
+  enable_dns_support = true
+  enable_dns_hostnames = true
+  tags = {
+    Name = "ngff-benchmarking-vpc"
+  }
+}
+
+resource "aws_internet_gateway" "gw" {
+  vpc_id = aws_vpc.vpc.id
+  tags = {
+    Name = "ngff-benchmarking-ig"
+  }
+}
+
+resource "aws_route_table" "route_table" {
+  vpc_id = aws_vpc.vpc.id
+
+  route {
+    cidr_block = "0.0.0.0/0"
+    gateway_id = aws_internet_gateway.gw.id
+  }
+
+  tags = {
+    Name = "ngff-benchmarking-rt"
+  }
+}
+
+resource "aws_subnet" "subnet" {
+  vpc_id     = aws_vpc.vpc.id
+  cidr_block = "10.0.1.0/24"
+  map_public_ip_on_launch = true
+
+  tags = {
+    Name = "ngff-benchmarking-subnet"
+  }
+}
+
+resource "aws_route_table_association" "rt_association" {
+  subnet_id      = aws_subnet.subnet.id
+  route_table_id = aws_route_table.route_table.id
+}
+
+resource "aws_security_group" "security_group" {
+  name        = "benchmarking_security_group"
+  vpc_id      = aws_vpc.vpc.id
+
+  ingress {
+    description = "TLS from VPC"
+    from_port   = 443
+    to_port     = 443
+    protocol    = "tcp"
+    cidr_blocks = [aws_vpc.vpc.cidr_block]
+  }
+
+  ingress {
+    description = "SSH"
+    from_port   = 22
+    to_port     = 22
+    protocol    = "tcp"
+    cidr_blocks = ["${var.ssh_client_ip}"]
+  }
+
+  tags = {
+    Name = "benchmarking_security_group"
+  }
+}
+
+data "aws_ami" "latest-ubuntu" {
+  most_recent = true
+  owners = ["099720109477"] # Canonical
+
+  filter {
+      name   = "name"
+      values = ["ubuntu/images/hvm-ssd/ubuntu-bionic-18.04-amd64-server-*"]
+  }
+
+  filter {
+      name   = "virtualization-type"
+      values = ["hvm"]
+  }
+}
+
+resource "aws_key_pair" "ngffkey" {
+  key_name   = "ngff-key"
+  public_key = "${var.ssh_public_key}"
+}
+
+resource "aws_instance" "nginx_instance" {
+    ami = "${data.aws_ami.latest-ubuntu.id}"
+    instance_type = "t2.micro"
+    subnet_id = aws_subnet.subnet.id
+    vpc_security_group_ids = [aws_security_group.security_group.id]
+    root_block_device {
+        volume_size = 128
+    }
+    key_name = aws_key_pair.ngffkey.key_name
+  tags = {
+    Name = "ngff-benchmarking-server"
+  }
+}
+
+resource "aws_instance" "client_instance" {
+    ami = "${data.aws_ami.latest-ubuntu.id}"
+    instance_type = "t2.micro"
+    subnet_id = aws_subnet.subnet.id
+    vpc_security_group_ids = [aws_security_group.security_group.id]
+    root_block_device {
+        volume_size = 128
+    }
+    key_name = aws_key_pair.ngffkey.key_name
+  tags = {
+    Name = "ngff-benchmarking-client"
+  }
+}
+

terraform/terraform.tfvars.example

@@ -0,0 +1,2 @@
+ssh_client_ip = "1.2.3.4/32"
+ssh_public_key = "ssh-rsa abcdefg..."

terraform/tfbackend.config.example

@@ -0,0 +1,4 @@
+bucket          = "dev-ngff-bench-tfstate"
+key             = "terraform.tfstate"
+region          = "us-east-1"
+encrypt         = true