diff --git a/README.md b/README.md index c6c263e..261fabb 100644 --- a/README.md +++ b/README.md @@ -29,6 +29,8 @@ The following modules are available: * [run-pex-as-resource](/modules/run-pex-as-resource): This module prepares a portable environment for running PEX files and runs them as an local-exec provisioner on a null_resource. PEX files are python executables that contain all the requirements necessary to run the script. **(This module requires Python)** +* [enabled-aws-regions](/modules/enabled-aws-regions): This is a module that can be used to query AWS for all enabled + AWS regions on the authenticated AWS account. **(This module requires Python)** The following modules were deprecated and removed: diff --git a/examples/enabled-aws-regions/README.md b/examples/enabled-aws-regions/README.md new file mode 100644 index 0000000..76a40cb --- /dev/null +++ b/examples/enabled-aws-regions/README.md @@ -0,0 +1,13 @@ +# Enabled AWS Regions example + +This folder shows examples of how to use the [enabled-aws-regions module](/modules/enabled-aws-regions) to query your +AWS account for a list of all enabled AWS regions. + + + + +## How do you run these examples? + +1. Install [Terraform](https://www.terraform.io/). +1. `terraform init`. +1. `terraform apply`. diff --git a/examples/enabled-aws-regions/main.tf b/examples/enabled-aws-regions/main.tf new file mode 100644 index 0000000..a8ce38c --- /dev/null +++ b/examples/enabled-aws-regions/main.tf @@ -0,0 +1,12 @@ +terraform { + required_version = ">= 0.12" +} + +module "all_regions" { + # When using these modules in your own templates, you will need to use a Git URL with a ref attribute that pins you + # to a specific version of the modules, such as the following example: + # source = "git::git@github.com:gruntwork-io/package-terraform-utilities.git//modules/enabled-aws-regions?ref=v1.0.8" + source = "../../modules/enabled-aws-regions" + + default_region = var.default_region +} diff --git a/examples/enabled-aws-regions/outputs.tf b/examples/enabled-aws-regions/outputs.tf new file mode 100644 index 0000000..387d69a --- /dev/null +++ b/examples/enabled-aws-regions/outputs.tf @@ -0,0 +1,4 @@ +output "enabled_regions" { + description = "List of region names (e.g us-east-1) that are enabled on the authenticated AWS account." + value = module.all_regions.enabled_regions +} diff --git a/examples/enabled-aws-regions/variables.tf b/examples/enabled-aws-regions/variables.tf new file mode 100644 index 0000000..94753b6 --- /dev/null +++ b/examples/enabled-aws-regions/variables.tf @@ -0,0 +1,5 @@ +variable "default_region" { + description = "Default region to use as a seed for authenticating the AWS SDK. This should be a region that you know to be enabled on your account." + type = string + default = "us-east-1" +} diff --git a/modules/enabled-aws-regions/README.md b/modules/enabled-aws-regions/README.md new file mode 100644 index 0000000..86f3c67 --- /dev/null +++ b/modules/enabled-aws-regions/README.md @@ -0,0 +1,45 @@ +# Enabled AWS Regions Module + +This is a module that can be used to query your AWS connection for all enabled AWS regions on the authenticated AWS +account. + +This module uses Python under the hood so, the Python binary must be installed on the OS. + + +## Example code + +See the [enabled-aws-regions example](/examples/enabled-aws-regions) for working sample code. + + +## Building the binary + +The binary is a python executable that includes the necessary third party requirements. This special version of python +embeds cross platform versions of the requirements that are unpacked at runtime into a virtualenv. This executable is +then used to call out to the entrypoint script, which will import the library function. + +As such, the binary only needs to be built when the requirements change. You do not need to rebuild the binary for any +changes to the source files in the `get-enabled-regions` folder. + +This approach is taken so that consumers of the module do not need to install additional third party libraries on top of +python to utilize the script. To make this work, the `pex` binaries need to be checked into the repository so that they +are distributed with the module. + +The binary is generated using the [`pex`](https://pex.readthedocs.io/en/stable/whatispex.html) utility. Pex will package +the python script with all its requirements into a single binary, that can be made to be compatible with multiple +versions of python and multiple OS platforms. + +To build the binary, you will need the following: + +- A working python environment with **all compatible versions of python** setup (so that you can build binaries for all + versions) +- pex installed (use `pip install -r ./get-enabled-regions/dev_requirements.txt`) + +You can then use the helper script `build.sh` which will build the binary and copy it to the `bin` +directory for distribution. After that, you just need to check in the updated binaries. + +It is recommended to use [`pyenv`](https://github.com/pyenv/pyenv) to help setup an environment with multiple python +interpreters. The latest binaries are built with the following python environment: + +```bash +pyenv shell 2.7.15 3.5.2 3.6.6 3.7.0 +``` diff --git a/modules/enabled-aws-regions/get-enabled-regions/bin/get_enabled_regions_py27_env.pex b/modules/enabled-aws-regions/get-enabled-regions/bin/get_enabled_regions_py27_env.pex new file mode 100755 index 0000000..e3d34a6 Binary files /dev/null and b/modules/enabled-aws-regions/get-enabled-regions/bin/get_enabled_regions_py27_env.pex differ diff --git a/modules/enabled-aws-regions/get-enabled-regions/bin/get_enabled_regions_py3_env.pex b/modules/enabled-aws-regions/get-enabled-regions/bin/get_enabled_regions_py3_env.pex new file mode 100755 index 0000000..7ff9b60 Binary files /dev/null and b/modules/enabled-aws-regions/get-enabled-regions/bin/get_enabled_regions_py3_env.pex differ diff --git a/modules/enabled-aws-regions/get-enabled-regions/build_scripts/.python-version b/modules/enabled-aws-regions/get-enabled-regions/build_scripts/.python-version new file mode 100644 index 0000000..9de4ed6 --- /dev/null +++ b/modules/enabled-aws-regions/get-enabled-regions/build_scripts/.python-version @@ -0,0 +1,4 @@ +2.7.12 +3.5.2 +3.6.6 +3.7.0 diff --git a/modules/enabled-aws-regions/get-enabled-regions/build_scripts/build.sh b/modules/enabled-aws-regions/get-enabled-regions/build_scripts/build.sh new file mode 100755 index 0000000..22ea18a --- /dev/null +++ b/modules/enabled-aws-regions/get-enabled-regions/build_scripts/build.sh @@ -0,0 +1,44 @@ +#!/bin/bash +# +# Script to generate a single script with all requirements packed in that is compatible with multiple python +# versions and multiple platforms. +# + +set -e + +readonly FILEDIR="$(dirname "$0")" + +build() { + echo "Building execution environment for get_enabled_regions" + + # Build python2 + pex --python-shebang='/usr/bin/env python' \ + -r ../requirements.txt \ + --python=python2.7 \ + --platform macosx_10.12-x86_64 \ + --platform macosx_10.13-x86_64 \ + --platform macosx_10.14-x86_64 \ + --platform linux-x86_64 \ + --platform linux-x86_64-cp-27-mu \ + --platform win32 \ + --platform win_amd64 \ + --disable-cache \ + -o ../bin/get_enabled_regions_py27_env.pex + + # Build python3 + pex --python-shebang='/usr/bin/env python' \ + -r ../requirements.txt \ + --python=python3.5 \ + --python=python3.6 \ + --python=python3.7 \ + --platform macosx_10.12-x86_64 \ + --platform macosx_10.13-x86_64 \ + --platform macosx_10.14-x86_64 \ + --platform linux-x86_64 \ + --platform win32 \ + --platform win_amd64 \ + --disable-cache \ + -o ../bin/get_enabled_regions_py3_env.pex +} + +(cd "${FILEDIR}" && build) diff --git a/modules/enabled-aws-regions/get-enabled-regions/dev_requirements.txt b/modules/enabled-aws-regions/get-enabled-regions/dev_requirements.txt new file mode 100644 index 0000000..ff7dfc3 --- /dev/null +++ b/modules/enabled-aws-regions/get-enabled-regions/dev_requirements.txt @@ -0,0 +1,6 @@ +# Install this specific version of pex for better windows support +# https://github.com/pantsbuild/pex/pull/596 +# https://github.com/pantsbuild/pex/pull/638 +git+https://github.com/yorinasub17/pex.git@yori-working-windows-support + +flake8 diff --git a/modules/enabled-aws-regions/get-enabled-regions/get_enabled_regions/__init__.py b/modules/enabled-aws-regions/get-enabled-regions/get_enabled_regions/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/modules/enabled-aws-regions/get-enabled-regions/get_enabled_regions/main.py b/modules/enabled-aws-regions/get-enabled-regions/get_enabled_regions/main.py new file mode 100644 index 0000000..953fcb5 --- /dev/null +++ b/modules/enabled-aws-regions/get-enabled-regions/get_enabled_regions/main.py @@ -0,0 +1,26 @@ +from __future__ import print_function +import click +import boto3 +import json + + +@click.command() +@click.option( + '--region', + required=True, + help='Default region to use as a seed for authenticating the AWS SDK.', +) +def main(region): + """ + Script to get all enabled AWS regions for the authenticated AWS account. + """ + ec2 = boto3.client('ec2', region) + enabled_regions = ec2.describe_regions() + # We use a comma separated value for the list of regions here, primarily because terraform external data source does + # not support conversion of collection types (lists and maps). + enabled_regions_formatted = ','.join([region['RegionName'] for region in enabled_regions.get('Regions', [])]) + print(json.dumps({'enabled_regions': enabled_regions_formatted})) + + +if __name__ == '__main__': + main() diff --git a/modules/enabled-aws-regions/get-enabled-regions/requirements.txt b/modules/enabled-aws-regions/get-enabled-regions/requirements.txt new file mode 100644 index 0000000..d6a1cf6 --- /dev/null +++ b/modules/enabled-aws-regions/get-enabled-regions/requirements.txt @@ -0,0 +1,3 @@ +click==7.0 +six==1.12.0 +boto3==1.10.32 diff --git a/modules/enabled-aws-regions/main.tf b/modules/enabled-aws-regions/main.tf new file mode 100644 index 0000000..e3334ba --- /dev/null +++ b/modules/enabled-aws-regions/main.tf @@ -0,0 +1,50 @@ +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +# GET ALL ENABLED AWS REGIONS +# This terraform module uses the embedded pex binary to use the AWS API to query for all enabled AWS regions for the +# authenticated account. +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +# --------------------------------------------------------------------------------------------------------------------- +# SET TERRAFORM REQUIREMENTS FOR RUNNING THIS MODULE +# --------------------------------------------------------------------------------------------------------------------- + +terraform { + required_version = ">= 0.12" +} + +# --------------------------------------------------------------------------------------------------------------------- +# CALL PYTHON SCRIPT TO GET ENABLED REGIONS +# --------------------------------------------------------------------------------------------------------------------- + +module "all_regions" { + source = "../run-pex-as-data-source" + + # Path components to each of the PEX binary + python2_pex_path_parts = [ + path.module, + "get-enabled-regions", + "bin", + "get_enabled_regions_py27_env.pex", + ] + + python3_pex_path_parts = [ + path.module, + "get-enabled-regions", + "bin", + "get_enabled_regions_py3_env.pex", + ] + + # Path components to the folder that holds the python modules for get-enabled-regions + pex_module_path_parts = [ + path.module, + "get-enabled-regions", + ] + + # The entrypoint of the get-enabled-regions script, encoded as MODULE:FUNCTION + script_main_function = "get_enabled_regions.main:main" + + # Argument to be passed to the entrypoint of get-enabled-regions script + command_args = "--region ${var.default_region}" + + enabled = var.enabled +} diff --git a/modules/enabled-aws-regions/outputs.tf b/modules/enabled-aws-regions/outputs.tf new file mode 100644 index 0000000..cfcc6e3 --- /dev/null +++ b/modules/enabled-aws-regions/outputs.tf @@ -0,0 +1,4 @@ +output "enabled_regions" { + description = "List of region names (e.g us-east-1) that are enabled on the authenticated AWS account." + value = split(",", module.all_regions.result.enabled_regions) +} diff --git a/modules/enabled-aws-regions/variables.tf b/modules/enabled-aws-regions/variables.tf new file mode 100644 index 0000000..d9d1fc9 --- /dev/null +++ b/modules/enabled-aws-regions/variables.tf @@ -0,0 +1,11 @@ +variable "default_region" { + description = "Default region to use as a seed for authenticating the AWS SDK. This should be a region that you know to be enabled on your account." + type = string + default = "us-east-1" +} + +variable "enabled" { + description = "Whether or not to run the script. This is useful when you want to conditionally execute this module." + type = bool + default = true +} diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..981bc2b --- /dev/null +++ b/setup.cfg @@ -0,0 +1,12 @@ +[flake8] +max-line-length = 120 + +[yapf] +based_on_style = google +align_closing_bracket_with_visual_indent = true +column_limit = 120 +blank_line_before_nested_class_or_def = true +coalesce_brackets = false +dedent_closing_brackets = true +split_before_dot = true +split_complex_comprehension = true diff --git a/test/enabled_aws_regions_test.go b/test/enabled_aws_regions_test.go new file mode 100644 index 0000000..4930a8d --- /dev/null +++ b/test/enabled_aws_regions_test.go @@ -0,0 +1,29 @@ +package test + +import ( + "testing" + + "github.com/gruntwork-io/terratest/modules/collections" + "github.com/gruntwork-io/terratest/modules/terraform" + "github.com/stretchr/testify/assert" +) + +func TestGetEnabledAWSRegions(t *testing.T) { + t.Parallel() + + terratestOptions := createBaseTerratestOptions(t, "../examples/enabled-aws-regions") + defer terraform.Destroy(t, terratestOptions) + + terraform.InitAndApply(t, terratestOptions) + + regions := terraform.OutputList(t, terratestOptions, "enabled_regions") + + // Test a few regions that are known to be enabled on our test accounts + assert.True(t, collections.ListContains(regions, "us-east-1")) + assert.True(t, collections.ListContains(regions, "us-west-1")) + assert.True(t, collections.ListContains(regions, "eu-west-1")) + + // ... and verify an opted out region is not included + assert.False(t, collections.ListContains(regions, "ap-east-1")) // Hong Kong + assert.False(t, collections.ListContains(regions, "ap-northeast-3")) // Osaka +}