Merge pull request #21 from gruntwork-io/yori-get-all-regions

Introduce enabled-aws-regions module that can be used to get the list of available regions

README.md

@@ -29,6 +29,8 @@ The following modules are available:
 * [run-pex-as-resource](/modules/run-pex-as-resource): This module prepares a portable environment for running PEX files
   and runs them as an local-exec provisioner on a null_resource. PEX files are python executables that contain all the
   requirements necessary to run the script. **(This module requires Python)**
+* [enabled-aws-regions](/modules/enabled-aws-regions): This is a module that can be used to query AWS for all enabled
+  AWS regions on the authenticated AWS account. **(This module requires Python)**
 
 The following modules were deprecated and removed:
 

examples/enabled-aws-regions/README.md

@@ -0,0 +1,13 @@
+# Enabled AWS Regions example
+
+This folder shows examples of how to use the [enabled-aws-regions module](/modules/enabled-aws-regions) to query your
+AWS account for a list of all enabled AWS regions.
+
+
+
+
+## How do you run these examples?
+
+1. Install [Terraform](https://www.terraform.io/).
+1. `terraform init`.
+1. `terraform apply`.

examples/enabled-aws-regions/main.tf

@@ -0,0 +1,12 @@
+terraform {
+  required_version = ">= 0.12"
+}
+
+module "all_regions" {
+  # When using these modules in your own templates, you will need to use a Git URL with a ref attribute that pins you
+  # to a specific version of the modules, such as the following example:
+  # source = "git::[email protected]:gruntwork-io/package-terraform-utilities.git//modules/enabled-aws-regions?ref=v1.0.8"
+  source = "../../modules/enabled-aws-regions"
+
+  default_region = var.default_region
+}

examples/enabled-aws-regions/outputs.tf

@@ -0,0 +1,4 @@
+output "enabled_regions" {
+  description = "List of region names (e.g us-east-1) that are enabled on the authenticated AWS account."
+  value       = module.all_regions.enabled_regions
+}

examples/enabled-aws-regions/variables.tf

@@ -0,0 +1,5 @@
+variable "default_region" {
+  description = "Default region to use as a seed for authenticating the AWS SDK. This should be a region that you know to be enabled on your account."
+  type        = string
+  default     = "us-east-1"
+}

modules/enabled-aws-regions/README.md

@@ -0,0 +1,45 @@
+# Enabled AWS Regions Module
+
+This is a module that can be used to query your AWS connection for all enabled AWS regions on the authenticated AWS
+account.
+
+This module uses Python under the hood so, the Python binary must be installed on the OS.
+
+
+## Example code
+
+See the [enabled-aws-regions example](/examples/enabled-aws-regions) for working sample code.
+
+
+## Building the binary
+
+The binary is a python executable that includes the necessary third party requirements. This special version of python
+embeds cross platform versions of the requirements that are unpacked at runtime into a virtualenv. This executable is
+then used to call out to the entrypoint script, which will import the library function.
+
+As such, the binary only needs to be built when the requirements change. You do not need to rebuild the binary for any
+changes to the source files in the `get-enabled-regions` folder.
+
+This approach is taken so that consumers of the module do not need to install additional third party libraries on top of
+python to utilize the script. To make this work, the `pex` binaries need to be checked into the repository so that they
+are distributed with the module.
+
+The binary is generated using the [`pex`](https://pex.readthedocs.io/en/stable/whatispex.html) utility. Pex will package
+the python script with all its requirements into a single binary, that can be made to be compatible with multiple
+versions of python and multiple OS platforms.
+
+To build the binary, you will need the following:
+
+- A working python environment with **all compatible versions of python** setup (so that you can build binaries for all
+  versions)
+- pex installed (use `pip install -r ./get-enabled-regions/dev_requirements.txt`)
+
+You can then use the helper script `build.sh` which will build the binary and copy it to the `bin`
+directory for distribution. After that, you just need to check in the updated binaries.
+
+It is recommended to use [`pyenv`](https://github.com/pyenv/pyenv) to help setup an environment with multiple python
+interpreters. The latest binaries are built with the following python environment:
+
+```bash
+pyenv shell 2.7.15 3.5.2 3.6.6 3.7.0
+```

modules/enabled-aws-regions/get-enabled-regions/build_scripts/.python-version

@@ -0,0 +1,4 @@
+2.7.12
+3.5.2
+3.6.6
+3.7.0

modules/enabled-aws-regions/get-enabled-regions/build_scripts/build.sh

@@ -0,0 +1,44 @@
+#!/bin/bash
+#
+# Script to generate a single script with all requirements packed in that is compatible with multiple python
+# versions and multiple platforms.
+#
+
+set -e
+
+readonly FILEDIR="$(dirname "$0")"
+
+build() {
+    echo "Building execution environment for get_enabled_regions"
+
+    # Build python2
+    pex --python-shebang='/usr/bin/env python' \
+        -r ../requirements.txt \
+        --python=python2.7 \
+        --platform macosx_10.12-x86_64 \
+        --platform macosx_10.13-x86_64 \
+        --platform macosx_10.14-x86_64 \
+        --platform linux-x86_64 \
+        --platform linux-x86_64-cp-27-mu \
+        --platform win32 \
+        --platform win_amd64 \
+        --disable-cache \
+        -o ../bin/get_enabled_regions_py27_env.pex
+
+    # Build python3
+    pex --python-shebang='/usr/bin/env python' \
+        -r ../requirements.txt \
+        --python=python3.5 \
+        --python=python3.6 \
+        --python=python3.7 \
+        --platform macosx_10.12-x86_64 \
+        --platform macosx_10.13-x86_64 \
+        --platform macosx_10.14-x86_64 \
+        --platform linux-x86_64 \
+        --platform win32 \
+        --platform win_amd64 \
+        --disable-cache \
+        -o ../bin/get_enabled_regions_py3_env.pex
+}
+
+(cd "${FILEDIR}" && build)

modules/enabled-aws-regions/get-enabled-regions/dev_requirements.txt

@@ -0,0 +1,6 @@
+# Install this specific version of pex for better windows support
+# https://github.com/pantsbuild/pex/pull/596
+# https://github.com/pantsbuild/pex/pull/638
+git+https://github.com/yorinasub17/pex.git@yori-working-windows-support
+
+flake8

modules/enabled-aws-regions/get-enabled-regions/get_enabled_regions/main.py

@@ -0,0 +1,26 @@
+from __future__ import print_function
+import click
+import boto3
+import json
+
+
+@click.command()
+@click.option(
+    '--region',
+    required=True,
+    help='Default region to use as a seed for authenticating the AWS SDK.',
+)
+def main(region):
+    """
+    Script to get all enabled AWS regions for the authenticated AWS account.
+    """
+    ec2 = boto3.client('ec2', region)
+    enabled_regions = ec2.describe_regions()
+    # We use a comma separated value for the list of regions here, primarily because terraform external data source does
+    # not support conversion of collection types (lists and maps).
+    enabled_regions_formatted = ','.join([region['RegionName'] for region in enabled_regions.get('Regions', [])])
+    print(json.dumps({'enabled_regions': enabled_regions_formatted}))
+
+
+if __name__ == '__main__':
+    main()

modules/enabled-aws-regions/get-enabled-regions/requirements.txt

@@ -0,0 +1,3 @@
+click==7.0
+six==1.12.0
+boto3==1.10.32

modules/enabled-aws-regions/main.tf

@@ -0,0 +1,50 @@
+# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+# GET ALL ENABLED AWS REGIONS
+# This terraform module uses the embedded pex binary to use the AWS API to query for all enabled AWS regions for the
+# authenticated account.
+# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+# ---------------------------------------------------------------------------------------------------------------------
+# SET TERRAFORM REQUIREMENTS FOR RUNNING THIS MODULE
+# ---------------------------------------------------------------------------------------------------------------------
+
+terraform {
+  required_version = ">= 0.12"
+}
+
+# ---------------------------------------------------------------------------------------------------------------------
+# CALL PYTHON SCRIPT TO GET ENABLED REGIONS
+# ---------------------------------------------------------------------------------------------------------------------
+
+module "all_regions" {
+  source = "../run-pex-as-data-source"
+
+  # Path components to each of the PEX binary
+  python2_pex_path_parts = [
+    path.module,
+    "get-enabled-regions",
+    "bin",
+    "get_enabled_regions_py27_env.pex",
+  ]
+
+  python3_pex_path_parts = [
+    path.module,
+    "get-enabled-regions",
+    "bin",
+    "get_enabled_regions_py3_env.pex",
+  ]
+
+  # Path components to the folder that holds the python modules for get-enabled-regions
+  pex_module_path_parts = [
+    path.module,
+    "get-enabled-regions",
+  ]
+
+  # The entrypoint of the get-enabled-regions script, encoded as MODULE:FUNCTION
+  script_main_function = "get_enabled_regions.main:main"
+
+  # Argument to be passed to the entrypoint of get-enabled-regions script
+  command_args = "--region ${var.default_region}"
+
+  enabled = var.enabled
+}

modules/enabled-aws-regions/outputs.tf

@@ -0,0 +1,4 @@
+output "enabled_regions" {
+  description = "List of region names (e.g us-east-1) that are enabled on the authenticated AWS account."
+  value       = split(",", module.all_regions.result.enabled_regions)
+}

modules/enabled-aws-regions/variables.tf

@@ -0,0 +1,11 @@
+variable "default_region" {
+  description = "Default region to use as a seed for authenticating the AWS SDK. This should be a region that you know to be enabled on your account."
+  type        = string
+  default     = "us-east-1"
+}
+
+variable "enabled" {
+  description = "Whether or not to run the script. This is useful when you want to conditionally execute this module."
+  type        = bool
+  default     = true
+}

setup.cfg

@@ -0,0 +1,12 @@
+[flake8]
+max-line-length = 120
+
+[yapf]
+based_on_style = google
+align_closing_bracket_with_visual_indent = true
+column_limit = 120
+blank_line_before_nested_class_or_def = true
+coalesce_brackets = false
+dedent_closing_brackets = true
+split_before_dot = true
+split_complex_comprehension = true

test/enabled_aws_regions_test.go

@@ -0,0 +1,29 @@
+package test
+
+import (
+	"testing"
+
+	"github.com/gruntwork-io/terratest/modules/collections"
+	"github.com/gruntwork-io/terratest/modules/terraform"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestGetEnabledAWSRegions(t *testing.T) {
+	t.Parallel()
+
+	terratestOptions := createBaseTerratestOptions(t, "../examples/enabled-aws-regions")
+	defer terraform.Destroy(t, terratestOptions)
+
+	terraform.InitAndApply(t, terratestOptions)
+
+	regions := terraform.OutputList(t, terratestOptions, "enabled_regions")
+
+	// Test a few regions that are known to be enabled on our test accounts
+	assert.True(t, collections.ListContains(regions, "us-east-1"))
+	assert.True(t, collections.ListContains(regions, "us-west-1"))
+	assert.True(t, collections.ListContains(regions, "eu-west-1"))
+
+	// ... and verify an opted out region is not included
+	assert.False(t, collections.ListContains(regions, "ap-east-1"))      // Hong Kong
+	assert.False(t, collections.ListContains(regions, "ap-northeast-3")) // Osaka
+}