diff --git a/.dockerignore b/.dockerignore index 39fcf3a5..22150c8d 100644 --- a/.dockerignore +++ b/.dockerignore @@ -12,6 +12,4 @@ examples .gitignore .golangci.yml CODEOWNERS -LICENSE -Makefile **/*.md diff --git a/Makefile b/Makefile deleted file mode 100644 index ba63dbd2..00000000 --- a/Makefile +++ /dev/null @@ -1,212 +0,0 @@ -SUBSTATION_DIR ?= $(shell git rev-parse --show-toplevel 2> /dev/null) -SUBSTATION_VERSION ?= "latest" -IMAGE_ARCH ?= $(shell uname -m) -AWS_ACCOUNT_ID ?= $(shell aws sts get-caller-identity --query 'Account' --output text 2> /dev/null) -AWS_REGION ?= $(shell aws configure get region 2> /dev/null) -FONT_RED := $(shell tput setaf 1) -FONT_RESET := $(shell tput sgr0) - -check: - @printf "$(FONT_RED)>> Checking Go version...$(FONT_RESET)\n" - -ifeq ($(shell go version 2> /dev/null),"") - @echo "Go is not installed! Please install the latest version: https://go.dev/doc/install." -else - @echo "$(shell go version 2> /dev/null)" -endif - - @printf "$(FONT_RED)>> Checking Python version...$(FONT_RESET)\n" - -ifeq ($(shell python3 --version 2> /dev/null),"") - @echo "Python is not installed! Please install the latest version: https://www.python.org/downloads." -else - @echo "$(shell python3 --version 2> /dev/null)" -endif - - @printf "$(FONT_RED)>> Checking Terraform version...$(FONT_RESET)\n" - -ifeq ($(shell terraform --version 2> /dev/null),"") - @echo "Terraform is not installed! Please install the latest version: https://www.terraform.io/downloads.html." -else - @echo "$(shell terraform --version 2> /dev/null)" -endif - - @printf "$(FONT_RED)>> Checking Jsonnet version...$(FONT_RESET)\n" - -ifeq ($(shell jsonnet --version 2> /dev/null),"") - @echo "Jsonnet is not installed! Please install the latest version: https://github.com/google/go-jsonnet." -else - @echo "$(shell jsonnet --version 2> /dev/null)" -endif - - @printf "$(FONT_RED)>> Checking Docker version...$(FONT_RESET)\n" - -ifeq ($(shell docker --version 2> /dev/null),"") - @echo "Docker is not installed! Please install the latest version: https://docs.docker.com/get-docker." -else - @echo "$(shell docker --version 2> /dev/null)" -endif - - @printf "$(FONT_RED)>> Checking Substation variables...$(FONT_RESET)\n" - -ifeq ("${SUBSTATION_DIR}","") - @echo "SUBSTATION_DIR variable is missing!" -else - @echo "SUBSTATION_DIR: ${SUBSTATION_DIR}" -endif - -ifeq ("${SUBSTATION_VERSION}","") - @echo "SUBSTATION_VERSION variable is missing!" -else - @echo "SUBSTATION_VERSION: ${SUBSTATION_VERSION}" -endif - -ifeq ("${IMAGE_ARCH}","") - @echo "IMAGE_ARCH variable is missing!" -else - @echo "IMAGE_ARCH: ${IMAGE_ARCH}" -endif - - @printf "$(FONT_RED)>> Checking AWS variables...$(FONT_RESET)\n" - -ifeq ("${AWS_ACCOUNT_ID}","") - @echo "AWS_ACCOUNT_ID variable is missing!" -else - @echo "AWS_ACCOUNT_ID: ${AWS_ACCOUNT_ID}" -endif - -ifeq ("${AWS_REGION}","") - @echo "AWS_REGION variable is missing!" -else - @echo "AWS_REGION: ${AWS_REGION}" -endif - -ifeq ("${AWS_APPCONFIG_ENV}","") - @echo "AWS_APPCONFIG_ENV variable is missing! This must match the AppConfig environment defined in Terraform." -else - @echo "AWS_APPCONFIG_ENV: ${AWS_APPCONFIG_ENV}" -endif - - @printf "$(FONT_RED)>> Checking deployment variables...$(FONT_RESET)\n" - -ifeq ("${DEPLOYMENT_DIR}","") - @echo "DEPLOYMENT_DIR variable is missing! This must point to a directory containing configuration (Jsonnet) and infrastructure (Terraform) files." -else - @echo "DEPLOYMENT_DIR: ${DEPLOYMENT_DIR}" -endif - -build-config: - @printf "$(FONT_RED)>> Building configuration files...$(FONT_RESET)\n" - @cd $(SUBSTATION_DIR) && bash build/scripts/config/compile.sh - -build-go: - @printf "$(FONT_RED)>> Building Go binaries...$(FONT_RESET)\n" - @for file in $(shell find $(SUBSTATION_DIR) -name main.go); do \ - cd $$(dirname $$file) && go build; \ - done - -build-aws: - @$(MAKE) build-aws-appconfig - @$(MAKE) build-aws-substation - @$(MAKE) build-aws-validate - @$(MAKE) build-aws-autoscale - @$(MAKE) build-config - -build-aws-appconfig: - @printf "$(FONT_RED)>> Building AppConfig extension...$(FONT_RESET)\n" - @cd $(SUBSTATION_DIR) && AWS_ARCHITECTURE=$(IMAGE_ARCH) AWS_REGION=$(AWS_REGION) bash build/scripts/aws/lambda/get_appconfig_extension.sh - -build-aws-substation: - @printf "$(FONT_RED)>> Building Substation Lambda function...$(FONT_RESET)\n" - @cd $(SUBSTATION_DIR) && docker build --build-arg ARCH=$(IMAGE_ARCH) -f build/container/aws/lambda/substation/Dockerfile -t substation:latest-$(IMAGE_ARCH) . - -build-aws-validate: - @printf "$(FONT_RED)>> Building Validator Lambda function...$(FONT_RESET)\n" - @cd $(SUBSTATION_DIR) && docker build --build-arg ARCH=$(IMAGE_ARCH) -f build/container/aws/lambda/validate/Dockerfile -t validate:latest-$(IMAGE_ARCH) . - -build-aws-autoscale: - @printf "$(FONT_RED)>> Building Autoscaler Lambda function...$(FONT_RESET)\n" - @cd $(SUBSTATION_DIR) && docker build --build-arg ARCH=$(IMAGE_ARCH) -f build/container/aws/lambda/autoscale/Dockerfile -t autoscale:latest-$(IMAGE_ARCH) . - -deploy-aws: - @$(MAKE) deploy-aws-tf-init - @$(MAKE) deploy-aws-images - @$(MAKE) deploy-aws-tf-all - @$(MAKE) deploy-aws-config - -deploy-aws-tf-init: - @printf "$(FONT_RED)>> Initializing cloud infrastructure to AWS with Terraform...$(FONT_RESET)\n" - - @cd $$(dirname $$(find $(DEPLOYMENT_DIR) -name _provider.tf)) && \ - terraform init && \ - terraform apply \ - -target=module.kms \ - -target=module.ecr \ - -target=module.ecr_autoscale \ - -target=module.ecr_validate - -deploy-aws-tf-all: - @printf "$(FONT_RED)>> Deploying cloud infrastructure to AWS with Terraform...$(FONT_RESET)\n" - @cd $$(dirname $$(find $(DEPLOYMENT_DIR) -name _provider.tf)) && terraform apply - -deploy-aws-images: - @printf "$(FONT_RED)>> Deploying images to AWS ECR with Docker...$(FONT_RESET)\n" - @cd $(SUBSTATION_DIR) && AWS_ACCOUNT_ID=$(AWS_ACCOUNT_ID) AWS_REGION=$(AWS_REGION) bash build/scripts/aws/ecr_login.sh - - @cd $$(dirname $$(find $(DEPLOYMENT_DIR) -name _provider.tf)) -ifneq ("$(shell aws ecr describe-repositories --region $(AWS_REGION) --repository-names substation --output text 2> /dev/null)","") - @docker tag substation:latest-$(IMAGE_ARCH) $(AWS_ACCOUNT_ID).dkr.ecr.$(AWS_REGION).amazonaws.com/substation:$(SUBSTATION_VERSION) - @docker push $(AWS_ACCOUNT_ID).dkr.ecr.$(AWS_REGION).amazonaws.com/substation:$(SUBSTATION_VERSION) -endif - -ifneq ("$(shell aws ecr describe-repositories --repository-names validate --output text 2> /dev/null)","") - @docker tag validate:latest-$(IMAGE_ARCH) $(AWS_ACCOUNT_ID).dkr.ecr.$(AWS_REGION).amazonaws.com/validate:$(SUBSTATION_VERSION) - @docker push $(AWS_ACCOUNT_ID).dkr.ecr.$(AWS_REGION).amazonaws.com/validate:$(SUBSTATION_VERSION) -endif - -ifneq ("$(shell aws ecr describe-repositories --repository-names autoscale --output text 2> /dev/null)","") - @docker tag autoscale:latest-$(IMAGE_ARCH) $(AWS_ACCOUNT_ID).dkr.ecr.$(AWS_REGION).amazonaws.com/autoscale:$(SUBSTATION_VERSION) - @docker push $(AWS_ACCOUNT_ID).dkr.ecr.$(AWS_REGION).amazonaws.com/autoscale:$(SUBSTATION_VERSION) -endif - -deploy-aws-config: - @printf "$(FONT_RED)>> Deploying configurations to AppConfig with Python...$(FONT_RESET)\n" - @cd $(SUBSTATION_DIR) && SUBSTATION_CONFIG_DIRECTORY=$(DEPLOYMENT_DIR) AWS_DEFAULT_REGION=$(AWS_REGION) AWS_APPCONFIG_APPLICATION_NAME=substation AWS_APPCONFIG_ENVIRONMENT=$(AWS_APPCONFIG_ENV) AWS_APPCONFIG_DEPLOYMENT_STRATEGY=Instant python3 build/scripts/aws/appconfig/appconfig_upload.py - -destroy-aws: - @printf "$(FONT_RED)>> Destroying configurations in AppConfig with Python...$(FONT_RESET)\n" - @cd $(SUBSTATION_DIR) - - @for file in $(shell find $(DEPLOYMENT_DIR) -name config.jsonnet); do \ - AWS_DEFAULT_REGION=$(AWS_REGION) AWS_APPCONFIG_APPLICATION_NAME=substation AWS_APPCONFIG_PROFILE_NAME=$$(basename $$(dirname $$file)) python3 build/scripts/aws/appconfig/appconfig_delete.py; \ - done - - @printf "$(FONT_RED)>> Destroying cloud infrastructure in AWS with Terraform...$(FONT_RESET)\n" - @cd $$(dirname $$(find $(DEPLOYMENT_DIR) -name _provider.tf)) && terraform destroy - -test-local: - @$(MAKE) build-go - @$(MAKE) build-config - - @printf "$(FONT_RED)>> Printing data file...$(FONT_RESET)\n" - @cat examples/cmd/client/file/substation/data.json - - @printf "$(FONT_RED)>> Running Substation...$(FONT_RESET)\n" - @cd examples/cmd/client/file/substation && ./substation -config config.json -file data.json - -test-aws: - @$(MAKE) build-aws - @$(MAKE) deploy-aws DEPLOYMENT_DIR=examples/terraform/aws/kinesis/time_travel AWS_APPCONFIG_ENV=example - - @printf "$(FONT_RED)>> Sending data to AWS Kinesis...$(FONT_RESET)\n" - @cd build/scripts/aws/kinesis && \ - echo '{"ip":"8.8.8.8"}' >> data.json && \ - echo '{"ip":"9.9.9.9"}' >> data.json && \ - echo '{"ip":"1.1.1.1"}' >> data.json && \ - echo '{"ip":"8.8.8.8","context":"GOOGLE"}' >> data.json && \ - echo '{"ip":"9.9.9.9","context":"QUAD9"}' >> data.json && \ - echo '{"ip":"1.1.1.1","context":"CLOUDFLARENET"}' >> data.json && \ - AWS_DEFAULT_REGION=$(AWS_REGION) python3 put_records.py substation data.json --print-response && \ - rm data.json - - @printf "$(FONT_RED)>> Log in to the AWS console and review the logs written by the Substation Lambda functions. When finished, run 'make destroy-aws' to clean up the resources. $(FONT_RESET)\n" diff --git a/README.md b/README.md index 7954cf7c..644b9ece 100644 --- a/README.md +++ b/README.md @@ -34,27 +34,43 @@ You can run Substation on these platforms: - [macOS / Linux](https://substation.readme.io/v1.0.0/docs/try-substation-on-macos-linux) - [AWS](https://substation.readme.io/v1.0.0/docs/try-substation-on-aws) -The project includes a Makefile that simplifies local development and test deployments. To test the system in AWS, run this from the project root: +### Development +[VS Code](https://code.visualstudio.com/docs/devcontainers/containers) is the recommended development environment for Substation. The project includes a [development container](.devcontainer/Dockerfile) that can be used to develop and test the system. Refer to the [development guide](CONTRIBUTING.md) for more information. + +### Testing + +The development container can be used to test the system locally and in the cloud. If you're not using VS Code, then you should run the development container from the command line: + +```sh +git clone https://github.com/brexhq/substation.git && cd substation && \ +docker build -t substation-dev .devcontainer/ && \ +docker run -v $(pwd):/workspaces/substation/ -w /workspaces/substation -v /var/run/docker.sock:/var/run/docker.sock -it substation-dev +``` + +To try the system locally, run this from the [examples](examples) directory: ```sh -# Checks that dependencies are installed and environment variables are set. -make check -# Deploys Substation to AWS. This deploys the Kinesis Time Travel example -# and writes data to the Kinesis stream. -make test-aws +git config --global --add safe.directory /workspaces/substation && \ +cd examples && \ +make -s quickstart ``` -The [AWS examples](examples/terraform/aws) folder contains reusable deployment patterns that demonstrate best practices for managing the system using [Terraform](https://www.terraform.io/) and [Jsonnet](https://jsonnet.org/). Deploy them using these commands: +To try the system in the cloud, choose an [AWS example](examples/terraform/aws) to deploy: +```sh +git config --global --add safe.directory /workspaces/substation && \ +cd examples && \ +aws configure && \ +make -s check && \ +make -s build && \ +make -s deploy EXAMPLE=terraform/aws/dynamodb/cdc +``` +After testing is complete, the cloud deployment should be destroyed: ```sh -make check -# Builds dependencies required for AWS deployments. -make build-aws -# Deploys the DynamoDB Change Data Capture example. -make deploy-aws DEPLOYMENT_DIR=examples/terraform/aws/dynamodb/cdc AWS_APPCONFIG_ENV=example +make -s destroy EXAMPLE=terraform/aws/dynamodb/cdc ``` -**We do not recommend managing cloud deployments from a local machine using the Makefile. Production deployments should use a CI/CD pipeline with a remote state backend to manage infrastructure.** +**We do not recommend managing cloud deployments from a local machine using the examples Makefile. Production deployments should use a CI/CD pipeline with a remote state backend to manage infrastructure.** ## Transforming Event Logs diff --git a/build/scripts/aws/lambda/get_appconfig_extension.sh b/build/scripts/aws/lambda/get_appconfig_extension.sh index 02908769..9372f1ac 100644 --- a/build/scripts/aws/lambda/get_appconfig_extension.sh +++ b/build/scripts/aws/lambda/get_appconfig_extension.sh @@ -133,7 +133,6 @@ else exit 1 fi -aws lambda get-layer-version-by-arn \ - --region $AWS_REGION --arn $AWS_ARN \ +aws lambda get-layer-version-by-arn --region $AWS_REGION --arn $AWS_ARN --output json \ | jq -r '.Content.Location' \ - | xargs curl -s -o $DIR/extension.zip + | xargs curl -s -o $DIR/extension.zip diff --git a/build/terraform/aws/cloudwatch/destination/_variables.tf b/build/terraform/aws/cloudwatch/destination/_variables.tf index f1f3ae5b..128dc03d 100644 --- a/build/terraform/aws/cloudwatch/destination/_variables.tf +++ b/build/terraform/aws/cloudwatch/destination/_variables.tf @@ -3,7 +3,8 @@ variable "kms" { arn = string id = string }) - description = "KMS key used to encrypt the resources." + default = null + description = "KMS key used to encrypt the resources. This is not required if the config.destination_arn does not use customer-managed encryption." } variable "config" { diff --git a/examples/Makefile b/examples/Makefile new file mode 100644 index 00000000..cce0d598 --- /dev/null +++ b/examples/Makefile @@ -0,0 +1,133 @@ +SUBSTATION_DIR ?= $(shell git rev-parse --show-toplevel 2> /dev/null) +SUBSTATION_VERSION ?= $(shell git describe --tags --always 2> /dev/null) +AWS_ACCOUNT_ID ?= $(shell aws sts get-caller-identity --query 'Account' --output text 2> /dev/null) +AWS_REGION ?= $(shell aws configure get region 2> /dev/null) + +FONT_RED := $(shell tput setaf 1) +FONT_RESET := $(shell tput sgr0) + +check: + @printf "$(FONT_RED)>> Checking Substation variables...$(FONT_RESET)\n" + +ifeq ("${SUBSTATION_DIR}","") + @echo "SUBSTATION_DIR variable is missing!" +else + @echo "SUBSTATION_DIR: ${SUBSTATION_DIR}" +endif + +ifeq ("${SUBSTATION_VERSION}","") + @echo "SUBSTATION_VERSION variable is missing!" +else + @echo "SUBSTATION_VERSION: ${SUBSTATION_VERSION}" +endif + + @printf "$(FONT_RED)>> Checking AWS variables...$(FONT_RESET)\n" + +ifeq ("${AWS_ACCOUNT_ID}","") + @echo "AWS_ACCOUNT_ID variable is missing!" +else + @echo "AWS_ACCOUNT_ID: ${AWS_ACCOUNT_ID}" +endif + +ifeq ("${AWS_REGION}","") + @echo "AWS_REGION variable is missing!" +else + @echo "AWS_REGION: ${AWS_REGION}" +endif + +.PHONY: build +build: + @$(MAKE) build-go + @$(MAKE) build-config + @$(MAKE) build-images + +build-config: + @printf "$(FONT_RED)>> Building configuration files...$(FONT_RESET)\n" + @cd $(SUBSTATION_DIR) && bash build/scripts/config/compile.sh + +build-go: + @printf "$(FONT_RED)>> Building Go binaries...$(FONT_RESET)\n" + @for file in $(shell find $(SUBSTATION_DIR) -name main.go); do \ + cd $$(dirname $$file) && go build; \ + done + +build-images: + @printf "$(FONT_RED)>> Building AppConfig extension...$(FONT_RESET)\n" + @cd $(SUBSTATION_DIR) && AWS_ARCHITECTURE=arm64 AWS_REGION=$(AWS_REGION) bash build/scripts/aws/lambda/get_appconfig_extension.sh + + @printf "$(FONT_RED)>> Building Docker images...$(FONT_RESET)\n" + @cd $(SUBSTATION_DIR) && \ + docker buildx build --platform linux/arm64 --build-arg ARCH=arm64 -f build/container/aws/lambda/substation/Dockerfile -t substation:latest-arm64 . && \ + docker buildx build --platform linux/arm64 --build-arg ARCH=arm64 -f build/container/aws/lambda/autoscale/Dockerfile -t autoscale:latest-arm64 . && \ + docker buildx build --platform linux/arm64 --build-arg ARCH=arm64 -f build/container/aws/lambda/validate/Dockerfile -t validate:latest-arm64 . + +deploy: + @$(MAKE) deploy-aws-init + @$(MAKE) deploy-aws-images + @$(MAKE) deploy-aws-infra + @$(MAKE) deploy-aws-config + @$(MAKE) deploy-aws-post-script + +deploy-aws-init: + @printf "$(FONT_RED)>> Initializing cloud infrastructure in AWS with Terraform...$(FONT_RESET)\n" + + @cd $(EXAMPLE)/terraform && \ + terraform init && \ + terraform apply -auto-approve -compact-warnings \ + -target=module.kms \ + -target=module.ecr \ + -target=module.ecr_autoscale \ + -target=module.ecr_validate + +deploy-aws-images: + @printf "$(FONT_RED)>> Deploying images to AWS ECR with Docker...$(FONT_RESET)\n" + + @aws ecr get-login-password | docker login --username AWS --password-stdin $(AWS_ACCOUNT_ID).dkr.ecr.$(AWS_REGION).amazonaws.com + +ifneq ("$(shell aws ecr describe-repositories --repository-names substation --output text 2> /dev/null)","") + @docker tag substation:latest-arm64 $(AWS_ACCOUNT_ID).dkr.ecr.$(AWS_REGION).amazonaws.com/substation:$(SUBSTATION_VERSION) + @docker push $(AWS_ACCOUNT_ID).dkr.ecr.$(AWS_REGION).amazonaws.com/substation:$(SUBSTATION_VERSION) +endif + +ifneq ("$(shell aws ecr describe-repositories --repository-names validate --output text 2> /dev/null)","") + @docker tag validate:latest-arm64 $(AWS_ACCOUNT_ID).dkr.ecr.$(AWS_REGION).amazonaws.com/validate:$(SUBSTATION_VERSION) + @docker push $(AWS_ACCOUNT_ID).dkr.ecr.$(AWS_REGION).amazonaws.com/validate:$(SUBSTATION_VERSION) +endif + +ifneq ("$(shell aws ecr describe-repositories --repository-names autoscale --output text 2> /dev/null)","") + @docker tag autoscale:latest-arm64 $(AWS_ACCOUNT_ID).dkr.ecr.$(AWS_REGION).amazonaws.com/autoscale:$(SUBSTATION_VERSION) + @docker push $(AWS_ACCOUNT_ID).dkr.ecr.$(AWS_REGION).amazonaws.com/autoscale:$(SUBSTATION_VERSION) +endif + +deploy-aws-infra: + @printf "$(FONT_RED)>> Deploying cloud infrastructure in AWS with Terraform...$(FONT_RESET)\n" + @cd $(EXAMPLE)/terraform && terraform apply -auto-approve -compact-warnings + +deploy-aws-config: + @printf "$(FONT_RED)>> Deploying configurations to AppConfig with Python...$(FONT_RESET)\n" + @cd $(SUBSTATION_DIR) && SUBSTATION_CONFIG_DIRECTORY=examples/$(EXAMPLE) AWS_APPCONFIG_APPLICATION_NAME=substation AWS_APPCONFIG_ENVIRONMENT=example AWS_APPCONFIG_DEPLOYMENT_STRATEGY=Instant python3 build/scripts/aws/appconfig/appconfig_upload.py + +deploy-aws-post-script: +ifneq ("$(wildcard $(EXAMPLE)/post_deploy.sh)","") + @printf "$(FONT_RED)>> Running post-deploy script...$(FONT_RESET)\n" + @bash $(EXAMPLE)/post_deploy.sh +endif + +destroy: + @printf "$(FONT_RED)>> Destroying configurations in AppConfig with Python...$(FONT_RESET)\n" + @for file in $(shell find $(EXAMPLE) -name config.jsonnet); do \ + AWS_DEFAULT_REGION=$(AWS_REGION) AWS_APPCONFIG_APPLICATION_NAME=substation AWS_APPCONFIG_PROFILE_NAME=$$(basename $$(dirname $$file)) python3 $(SUBSTATION_DIR)/build/scripts/aws/appconfig/appconfig_delete.py; \ + done + + @printf "$(FONT_RED)>> Destroying cloud infrastructure in AWS with Terraform...$(FONT_RESET)\n" + @cd $(EXAMPLE)/terraform && terraform destroy -auto-approve -compact-warnings + +quickstart: + @$(MAKE) build-go + @$(MAKE) build-config + + @printf "$(FONT_RED)>> Printing data file...$(FONT_RESET)\n" + @cat cmd/client/file/substation/data.json + + @printf "$(FONT_RED)>> Running Substation...$(FONT_RESET)\n" + @cd cmd/client/file/substation && ./substation -config config.json -file data.json diff --git a/examples/terraform/aws/README.md b/examples/terraform/aws/README.md index 53de72b5..36c7a7f2 100644 --- a/examples/terraform/aws/README.md +++ b/examples/terraform/aws/README.md @@ -82,6 +82,34 @@ flowchart LR end ``` +# Firehose + +## Data Transform + +Deploys a [Firehose](https://aws.amazon.com/firehose/) delivery stream with [data transformation](https://docs.aws.amazon.com/firehose/latest/dev/data-transformation.html) enabled. + +```mermaid + +flowchart LR + %% resources + data[/Data/] + firehose([Kinesis Data Firehose]) + s3([S3 Bucket]) + + nodeHandler[[Handler]] + nodeTransforms[Transforms] + + %% connections + data --> firehose --> nodeHandler + + subgraph Substation Node + nodeHandler --> nodeTransforms --> nodeHandler + end + + nodeHandler --> firehose + firehose --> s3 +``` + # Kinesis ## Autoscale @@ -110,32 +138,6 @@ flowchart LR cw_upscale -. monitors .- kds ``` -## Firehose - -Deploys a [Kinesis Data Firehose](https://aws.amazon.com/kinesis/data-firehose/) delivery stream with [data transformation](https://docs.aws.amazon.com/firehose/latest/dev/data-transformation.html) enabled. - -```mermaid - -flowchart LR - %% resources - data[/Data/] - firehose([Kinesis Data Firehose]) - s3([S3 Bucket]) - - nodeHandler[[Handler]] - nodeTransforms[Transforms] - - %% connections - data --> firehose --> nodeHandler - - subgraph Substation Node - nodeHandler --> nodeTransforms --> nodeHandler - end - - nodeHandler --> firehose - firehose --> s3 -``` - ## Multi-Stream Deploys a data pipeline that implements a multi-phase streaming data pattern using Kinesis Data Streams. diff --git a/examples/terraform/aws/cloudwatch_logs/cross_account_cross_region/terraform/_provider.tf b/examples/terraform/aws/cloudwatch_logs/cross_account_cross_region/terraform/_provider.tf index af2358d2..e2cba33f 100644 --- a/examples/terraform/aws/cloudwatch_logs/cross_account_cross_region/terraform/_provider.tf +++ b/examples/terraform/aws/cloudwatch_logs/cross_account_cross_region/terraform/_provider.tf @@ -1,10 +1,8 @@ provider "aws" { - # profile = "default" region = "us-east-1" } provider "aws" { - # profile = "default" alias = "usw2" region = "us-west-2" } diff --git a/examples/terraform/aws/cloudwatch_logs/cross_account_cross_region/terraform/_resources.tf b/examples/terraform/aws/cloudwatch_logs/cross_account_cross_region/terraform/_resources.tf index 381d9cc8..0b0ceb2d 100644 --- a/examples/terraform/aws/cloudwatch_logs/cross_account_cross_region/terraform/_resources.tf +++ b/examples/terraform/aws/cloudwatch_logs/cross_account_cross_region/terraform/_resources.tf @@ -77,8 +77,8 @@ module "cw_subscription_use1" { name = "substation" destination_arn = module.cw_destination_use1.arn log_groups = [ - # This example causes recursion. Add other log groups for resources in us-east-1. - # "/aws/lambda/consumer", + # This log group does not exist. Add other log groups for resources in the account and region. + # "/aws/lambda/test", ] } } @@ -91,7 +91,6 @@ module "cw_destination_usw2" { aws = aws.usw2 } - kms = module.kms config = { name = "substation" destination_arn = module.kds.arn diff --git a/examples/terraform/aws/cloudwatch_logs/cross_account_cross_region/terraform/autoscaler.tf b/examples/terraform/aws/cloudwatch_logs/cross_account_cross_region/terraform/autoscaler.tf index 1456018d..c0d95021 100644 --- a/examples/terraform/aws/cloudwatch_logs/cross_account_cross_region/terraform/autoscaler.tf +++ b/examples/terraform/aws/cloudwatch_logs/cross_account_cross_region/terraform/autoscaler.tf @@ -7,7 +7,7 @@ module "lambda_autoscaling" { config = { name = "autoscale" description = "Autoscaler for Kinesis Data Streams" - image_uri = "${module.ecr_autoscale.url}:latest" + image_uri = "${module.ecr_autoscale.url}:v1.2.0" image_arm = true } diff --git a/examples/terraform/aws/cloudwatch_logs/cross_account_cross_region/terraform/consumer.tf b/examples/terraform/aws/cloudwatch_logs/cross_account_cross_region/terraform/consumer.tf index a52c8168..ea5eef3d 100644 --- a/examples/terraform/aws/cloudwatch_logs/cross_account_cross_region/terraform/consumer.tf +++ b/examples/terraform/aws/cloudwatch_logs/cross_account_cross_region/terraform/consumer.tf @@ -5,7 +5,7 @@ module "lambda_consumer" { config = { name = "consumer" description = "Substation node that consumes from Kinesis" - image_uri = "${module.ecr.url}:latest" + image_uri = "${module.ecr.url}:v1.2.0" image_arm = true env = { diff --git a/examples/terraform/aws/cloudwatch_logs/to_lambda/terraform/_provider.tf b/examples/terraform/aws/cloudwatch_logs/to_lambda/terraform/_provider.tf deleted file mode 100644 index 872a6309..00000000 --- a/examples/terraform/aws/cloudwatch_logs/to_lambda/terraform/_provider.tf +++ /dev/null @@ -1,4 +0,0 @@ -provider "aws" { - # profile = "default" - region = "us-east-1" -} diff --git a/examples/terraform/aws/cloudwatch_logs/to_lambda/terraform/consumer.tf b/examples/terraform/aws/cloudwatch_logs/to_lambda/terraform/consumer.tf index 7c13eff3..22e297ab 100644 --- a/examples/terraform/aws/cloudwatch_logs/to_lambda/terraform/consumer.tf +++ b/examples/terraform/aws/cloudwatch_logs/to_lambda/terraform/consumer.tf @@ -8,7 +8,7 @@ module "lambda_consumer" { config = { name = "consumer" description = "Substation node that is invoked by CloudWatch" - image_uri = "${module.ecr.url}:latest" + image_uri = "${module.ecr.url}:v1.2.0" image_arm = true env = { @@ -42,8 +42,8 @@ module "cw_subscription" { name = "substation" destination_arn = module.lambda_consumer.arn log_groups = [ - # This group does not exist. Add other log groups for resources in the account and region. - "/aws/lambda/test", + # This log group does not exist. Add other log groups for resources in the account and region. + # "/aws/lambda/test", ] } } diff --git a/examples/terraform/aws/dynamodb/cdc/terraform/_provider.tf b/examples/terraform/aws/dynamodb/cdc/terraform/_provider.tf deleted file mode 100644 index 872a6309..00000000 --- a/examples/terraform/aws/dynamodb/cdc/terraform/_provider.tf +++ /dev/null @@ -1,4 +0,0 @@ -provider "aws" { - # profile = "default" - region = "us-east-1" -} diff --git a/examples/terraform/aws/dynamodb/cdc/terraform/node.tf b/examples/terraform/aws/dynamodb/cdc/terraform/node.tf index 974ce57d..90ab6ca2 100644 --- a/examples/terraform/aws/dynamodb/cdc/terraform/node.tf +++ b/examples/terraform/aws/dynamodb/cdc/terraform/node.tf @@ -5,7 +5,7 @@ module "node" { config = { name = "node" description = "Substation node that receives CDC events" - image_uri = "${module.ecr.url}:latest" + image_uri = "${module.ecr.url}:v1.2.0" image_arm = true env = { "SUBSTATION_CONFIG" : "http://localhost:2772/applications/substation/environments/example/configurations/node" diff --git a/examples/terraform/aws/kinesis/firehose/config/processor/config.jsonnet b/examples/terraform/aws/firehose/data_transform/config/transform_node/config.jsonnet similarity index 100% rename from examples/terraform/aws/kinesis/firehose/config/processor/config.jsonnet rename to examples/terraform/aws/firehose/data_transform/config/transform_node/config.jsonnet diff --git a/examples/terraform/aws/kinesis/firehose/terraform/_resources.tf b/examples/terraform/aws/firehose/data_transform/terraform/_resources.tf similarity index 86% rename from examples/terraform/aws/kinesis/firehose/terraform/_resources.tf rename to examples/terraform/aws/firehose/data_transform/terraform/_resources.tf index c487c3c9..05ff91ce 100644 --- a/examples/terraform/aws/kinesis/firehose/terraform/_resources.tf +++ b/examples/terraform/aws/firehose/data_transform/terraform/_resources.tf @@ -4,6 +4,8 @@ locals { data "aws_caller_identity" "caller" {} +resource "random_uuid" "id" {} + module "kms" { source = "../../../../../../build/terraform/aws/kms" @@ -34,7 +36,7 @@ module "ecr" { } ################################## -# Kinesis Data Firehose resources +# Firehose resources ################################## # IAM @@ -52,11 +54,10 @@ data "aws_iam_policy_document" "firehose" { } resource "aws_iam_role" "firehose" { - name = "sub-${local.name}" + name = "substation-firehose-${local.name}" assume_role_policy = data.aws_iam_policy_document.firehose.json } - data "aws_iam_policy_document" "firehose_s3" { statement { effect = "Allow" @@ -89,7 +90,7 @@ data "aws_iam_policy_document" "firehose_s3" { } resource "aws_iam_policy" "firehose_s3" { - name = "sub-${local.name}" + name = "substation-firehose-${resource.random_uuid.id.id}" description = "Policy for the ${local.name} Kinesis Data Firehose." policy = data.aws_iam_policy_document.firehose_s3.json } @@ -104,8 +105,8 @@ resource "aws_iam_role_policy_attachment" "firehose_s3" { resource "random_uuid" "firehose_s3" {} resource "aws_s3_bucket" "firehose_s3" { - bucket = "${random_uuid.firehose_s3.result}-substation" - + bucket = "${random_uuid.firehose_s3.result}-substation" + force_destroy = true } resource "aws_s3_bucket_ownership_controls" "firehose_s3" { @@ -115,11 +116,6 @@ resource "aws_s3_bucket_ownership_controls" "firehose_s3" { } } -resource "aws_s3_bucket_acl" "firehose_s3" { - bucket = aws_s3_bucket.firehose_s3.id - acl = "private" -} - resource "aws_s3_bucket_server_side_encryption_configuration" "firehose_s3" { bucket = aws_s3_bucket.firehose_s3.bucket @@ -157,28 +153,28 @@ resource "aws_kinesis_firehose_delivery_stream" "firehose" { parameters { parameter_name = "LambdaArn" # LATEST is always used for container images. - parameter_value = "${module.processor.arn}:$LATEST" + parameter_value = "${module.transform.arn}:$LATEST" } } } } } -module "processor" { +module "transform" { source = "../../../../../../build/terraform/aws/lambda" kms = module.kms appconfig = module.appconfig config = { - name = "processor" - description = "Processes Kinesis Data Firehose records." - image_uri = "${module.ecr.url}:latest" + name = "transform_node" + description = "Transforms Kinesis Data Firehose records." + image_uri = "${module.ecr.url}:v1.2.0" image_arm = true memory = 128 timeout = 60 env = { - "SUBSTATION_CONFIG" : "http://localhost:2772/applications/substation/environments/example/configurations/processor" + "SUBSTATION_CONFIG" : "http://localhost:2772/applications/substation/environments/example/configurations/transform_node" "SUBSTATION_LAMBDA_HANDLER" : "AWS_KINESIS_DATA_FIREHOSE" "SUBSTATION_DEBUG" : true } diff --git a/examples/terraform/aws/kinesis/autoscale/terraform/_provider.tf b/examples/terraform/aws/kinesis/autoscale/terraform/_provider.tf deleted file mode 100644 index 872a6309..00000000 --- a/examples/terraform/aws/kinesis/autoscale/terraform/_provider.tf +++ /dev/null @@ -1,4 +0,0 @@ -provider "aws" { - # profile = "default" - region = "us-east-1" -} diff --git a/examples/terraform/aws/kinesis/autoscale/terraform/_resources.tf b/examples/terraform/aws/kinesis/autoscale/terraform/_resources.tf index 2d7e38e6..2a2fd423 100644 --- a/examples/terraform/aws/kinesis/autoscale/terraform/_resources.tf +++ b/examples/terraform/aws/kinesis/autoscale/terraform/_resources.tf @@ -43,7 +43,7 @@ module "lambda_autoscale" { config = { name = "autoscale" description = "Autoscaler for Kinesis Data Streams." - image_uri = "${module.ecr.url}:latest" # This should use the project's release tags. + image_uri = "${module.ecr.url}:v1.2.0" # This should use the project's release tags. image_arm = true # Override the default Autoscale configuration using environment variables. diff --git a/examples/terraform/aws/kinesis/firehose/terraform/_provider.tf b/examples/terraform/aws/kinesis/firehose/terraform/_provider.tf deleted file mode 100644 index 872a6309..00000000 --- a/examples/terraform/aws/kinesis/firehose/terraform/_provider.tf +++ /dev/null @@ -1,4 +0,0 @@ -provider "aws" { - # profile = "default" - region = "us-east-1" -} diff --git a/examples/terraform/aws/kinesis/multistream/terraform/_provider.tf b/examples/terraform/aws/kinesis/multistream/terraform/_provider.tf deleted file mode 100644 index 872a6309..00000000 --- a/examples/terraform/aws/kinesis/multistream/terraform/_provider.tf +++ /dev/null @@ -1,4 +0,0 @@ -provider "aws" { - # profile = "default" - region = "us-east-1" -} diff --git a/examples/terraform/aws/kinesis/multistream/terraform/autoscaler.tf b/examples/terraform/aws/kinesis/multistream/terraform/autoscaler.tf index 79fae97a..b34c8971 100644 --- a/examples/terraform/aws/kinesis/multistream/terraform/autoscaler.tf +++ b/examples/terraform/aws/kinesis/multistream/terraform/autoscaler.tf @@ -8,7 +8,7 @@ module "lambda_autoscaling" { config = { name = "autoscale" description = "Autoscaler for Kinesis Data Streams" - image_uri = "${module.ecr_autoscale.url}:latest" + image_uri = "${module.ecr_autoscale.url}:v1.2.0" image_arm = true } diff --git a/examples/terraform/aws/kinesis/multistream/terraform/publisher.tf b/examples/terraform/aws/kinesis/multistream/terraform/publisher.tf index 55de43ec..0e4e71f1 100644 --- a/examples/terraform/aws/kinesis/multistream/terraform/publisher.tf +++ b/examples/terraform/aws/kinesis/multistream/terraform/publisher.tf @@ -6,7 +6,7 @@ module "lambda_publisher" { config = { name = "publisher" description = "Substation node that publishes to Kinesis" - image_uri = "${module.ecr.url}:latest" + image_uri = "${module.ecr.url}:v1.2.0" image_arm = true env = { diff --git a/examples/terraform/aws/kinesis/multistream/terraform/subscriber.tf b/examples/terraform/aws/kinesis/multistream/terraform/subscriber.tf index bb8d5b2e..615a0bf1 100644 --- a/examples/terraform/aws/kinesis/multistream/terraform/subscriber.tf +++ b/examples/terraform/aws/kinesis/multistream/terraform/subscriber.tf @@ -6,7 +6,7 @@ module "lambda_subscriber" { config = { name = "subscriber" description = "Substation node subscribes to Kinesis" - image_uri = "${module.ecr.url}:latest" + image_uri = "${module.ecr.url}:v1.2.0" image_arm = true env = { diff --git a/examples/terraform/aws/kinesis/time_travel/terraform/_provider.tf b/examples/terraform/aws/kinesis/time_travel/terraform/_provider.tf deleted file mode 100644 index 872a6309..00000000 --- a/examples/terraform/aws/kinesis/time_travel/terraform/_provider.tf +++ /dev/null @@ -1,4 +0,0 @@ -provider "aws" { - # profile = "default" - region = "us-east-1" -} diff --git a/examples/terraform/aws/kinesis/time_travel/terraform/autoscaler.tf b/examples/terraform/aws/kinesis/time_travel/terraform/autoscaler.tf index 1456018d..c0d95021 100644 --- a/examples/terraform/aws/kinesis/time_travel/terraform/autoscaler.tf +++ b/examples/terraform/aws/kinesis/time_travel/terraform/autoscaler.tf @@ -7,7 +7,7 @@ module "lambda_autoscaling" { config = { name = "autoscale" description = "Autoscaler for Kinesis Data Streams" - image_uri = "${module.ecr_autoscale.url}:latest" + image_uri = "${module.ecr_autoscale.url}:v1.2.0" image_arm = true } diff --git a/examples/terraform/aws/kinesis/time_travel/terraform/enrichment.tf b/examples/terraform/aws/kinesis/time_travel/terraform/enrichment.tf index d1fa004a..79655353 100644 --- a/examples/terraform/aws/kinesis/time_travel/terraform/enrichment.tf +++ b/examples/terraform/aws/kinesis/time_travel/terraform/enrichment.tf @@ -5,7 +5,7 @@ module "lambda_enrichment" { config = { name = "enrichment" description = "Substation node that enriches data from Kinesis and writes it to DynamoDB" - image_uri = "${module.ecr.url}:latest" + image_uri = "${module.ecr.url}:v1.2.0" image_arm = true env = { diff --git a/examples/terraform/aws/kinesis/time_travel/terraform/subscriber.tf b/examples/terraform/aws/kinesis/time_travel/terraform/subscriber.tf index 8a30ce66..feff9332 100644 --- a/examples/terraform/aws/kinesis/time_travel/terraform/subscriber.tf +++ b/examples/terraform/aws/kinesis/time_travel/terraform/subscriber.tf @@ -5,7 +5,7 @@ module "lambda_subscriber" { config = { name = "subscriber" description = "Substation node that reads from Kinesis with a delay to support enrichment" - image_uri = "${module.ecr.url}:latest" + image_uri = "${module.ecr.url}:v1.2.0" image_arm = true env = { diff --git a/examples/terraform/aws/lambda/appconfig/terraform/_provider.tf b/examples/terraform/aws/lambda/appconfig/terraform/_provider.tf deleted file mode 100644 index 872a6309..00000000 --- a/examples/terraform/aws/lambda/appconfig/terraform/_provider.tf +++ /dev/null @@ -1,4 +0,0 @@ -provider "aws" { - # profile = "default" - region = "us-east-1" -} diff --git a/examples/terraform/aws/lambda/appconfig/terraform/_resources.tf b/examples/terraform/aws/lambda/appconfig/terraform/_resources.tf index 8fe2c25f..9d9b7c95 100644 --- a/examples/terraform/aws/lambda/appconfig/terraform/_resources.tf +++ b/examples/terraform/aws/lambda/appconfig/terraform/_resources.tf @@ -15,7 +15,7 @@ module "validate" { config = { name = "validate" description = "Substation configuration validator that is executed by AppConfig." - image_uri = "${module.ecr_validate.url}:latest" + image_uri = "${module.ecr_validate.url}:v1.2.0" image_arm = true memory = 128 diff --git a/examples/terraform/aws/lambda/appconfig/terraform/node.tf b/examples/terraform/aws/lambda/appconfig/terraform/node.tf index 3b859902..c76ea03a 100644 --- a/examples/terraform/aws/lambda/appconfig/terraform/node.tf +++ b/examples/terraform/aws/lambda/appconfig/terraform/node.tf @@ -7,7 +7,7 @@ module "node" { config = { name = "node" description = "Substation node that never receives a configuration." - image_uri = "${module.ecr.url}:latest" + image_uri = "${module.ecr.url}:v1.2.0" image_arm = true memory = 128 diff --git a/examples/terraform/aws/lambda/microservice/terraform/_provider.tf b/examples/terraform/aws/lambda/microservice/terraform/_provider.tf deleted file mode 100644 index 872a6309..00000000 --- a/examples/terraform/aws/lambda/microservice/terraform/_provider.tf +++ /dev/null @@ -1,4 +0,0 @@ -provider "aws" { - # profile = "default" - region = "us-east-1" -} diff --git a/examples/terraform/aws/lambda/microservice/terraform/microservice.tf b/examples/terraform/aws/lambda/microservice/terraform/microservice.tf index 16f36e0a..05e3cf4c 100644 --- a/examples/terraform/aws/lambda/microservice/terraform/microservice.tf +++ b/examples/terraform/aws/lambda/microservice/terraform/microservice.tf @@ -5,7 +5,7 @@ module "microservice" { config = { name = "microservice" description = "Substation node that acts as a synchronous microservice" - image_uri = "${module.ecr.url}:latest" + image_uri = "${module.ecr.url}:v1.2.0" image_arm = true memory = 128 diff --git a/examples/terraform/aws/lambda/vpc/terraform/_provider.tf b/examples/terraform/aws/lambda/vpc/terraform/_provider.tf deleted file mode 100644 index 872a6309..00000000 --- a/examples/terraform/aws/lambda/vpc/terraform/_provider.tf +++ /dev/null @@ -1,4 +0,0 @@ -provider "aws" { - # profile = "default" - region = "us-east-1" -} diff --git a/examples/terraform/aws/lambda/vpc/terraform/whatismyip.tf b/examples/terraform/aws/lambda/vpc/terraform/whatismyip.tf index 8d1a8c05..9a0a0d90 100644 --- a/examples/terraform/aws/lambda/vpc/terraform/whatismyip.tf +++ b/examples/terraform/aws/lambda/vpc/terraform/whatismyip.tf @@ -5,7 +5,7 @@ module "whatismyip" { config = { name = "whatismyip" description = "Substation node that acts as a synchronous microservice" - image_uri = "${module.ecr.url}:latest" + image_uri = "${module.ecr.url}:v1.2.0" image_arm = true memory = 128 diff --git a/examples/terraform/aws/s3/data_lake/terraform/_provider.tf b/examples/terraform/aws/s3/data_lake/terraform/_provider.tf deleted file mode 100644 index 872a6309..00000000 --- a/examples/terraform/aws/s3/data_lake/terraform/_provider.tf +++ /dev/null @@ -1,4 +0,0 @@ -provider "aws" { - # profile = "default" - region = "us-east-1" -} diff --git a/examples/terraform/aws/s3/data_lake/terraform/node.tf b/examples/terraform/aws/s3/data_lake/terraform/node.tf index a9ac8734..5c32fa8a 100644 --- a/examples/terraform/aws/s3/data_lake/terraform/node.tf +++ b/examples/terraform/aws/s3/data_lake/terraform/node.tf @@ -18,7 +18,7 @@ module "lambda_node" { config = { name = "node" description = "Substation node that writes data to S3" - image_uri = "${module.ecr.url}:latest" + image_uri = "${module.ecr.url}:v1.2.0" image_arm = true env = { diff --git a/examples/terraform/aws/s3/retry_on_failure/terraform/_provider.tf b/examples/terraform/aws/s3/retry_on_failure/terraform/_provider.tf deleted file mode 100644 index 872a6309..00000000 --- a/examples/terraform/aws/s3/retry_on_failure/terraform/_provider.tf +++ /dev/null @@ -1,4 +0,0 @@ -provider "aws" { - # profile = "default" - region = "us-east-1" -} diff --git a/examples/terraform/aws/s3/retry_on_failure/terraform/_resources.tf b/examples/terraform/aws/s3/retry_on_failure/terraform/_resources.tf index 9c9fe097..d6a5f9b7 100644 --- a/examples/terraform/aws/s3/retry_on_failure/terraform/_resources.tf +++ b/examples/terraform/aws/s3/retry_on_failure/terraform/_resources.tf @@ -2,8 +2,8 @@ module "appconfig" { source = "../../../../../../build/terraform/aws/appconfig" config = { - name = "substation" - environments = [{name = "example"}] + name = "substation" + environments = [{ name = "example" }] } } diff --git a/examples/terraform/aws/s3/retry_on_failure/terraform/node_with_retrier.tf b/examples/terraform/aws/s3/retry_on_failure/terraform/node_with_retrier.tf index 7488d0c0..05ab7cae 100644 --- a/examples/terraform/aws/s3/retry_on_failure/terraform/node_with_retrier.tf +++ b/examples/terraform/aws/s3/retry_on_failure/terraform/node_with_retrier.tf @@ -1,11 +1,11 @@ module "lambda_node" { - source = "../../../../../../build/terraform/aws/lambda" + source = "../../../../../../build/terraform/aws/lambda" appconfig = module.appconfig config = { name = "node" description = "Substation node that reads data from S3. The node will retry forever if it fails." - image_uri = "${module.ecr.url}:latest" + image_uri = "${module.ecr.url}:v1.2.0" image_arm = true env = { @@ -58,13 +58,13 @@ resource "aws_lambda_function_event_invoke_config" "node" { } module "lambda_retrier" { - source = "../../../../../../build/terraform/aws/lambda" + source = "../../../../../../build/terraform/aws/lambda" appconfig = module.appconfig config = { name = "retrier" description = "Substation node that receives events from the retry queue and invokes the original Lambda function." - image_uri = "${module.ecr.url}:latest" + image_uri = "${module.ecr.url}:v1.2.0" image_arm = true # This value should be 1/6th of the visibility timeout of the SQS queue. diff --git a/examples/terraform/aws/s3/sns/terraform/_provider.tf b/examples/terraform/aws/s3/sns/terraform/_provider.tf deleted file mode 100644 index 872a6309..00000000 --- a/examples/terraform/aws/s3/sns/terraform/_provider.tf +++ /dev/null @@ -1,4 +0,0 @@ -provider "aws" { - # profile = "default" - region = "us-east-1" -} diff --git a/examples/terraform/aws/s3/sns/terraform/node.tf b/examples/terraform/aws/s3/sns/terraform/node.tf index fe9394c1..4ca272f0 100644 --- a/examples/terraform/aws/s3/sns/terraform/node.tf +++ b/examples/terraform/aws/s3/sns/terraform/node.tf @@ -6,7 +6,7 @@ module "lambda_node" { config = { name = "node" description = "Substation node that reads data from S3 via SNS." - image_uri = "${module.ecr.url}:latest" + image_uri = "${module.ecr.url}:v1.2.0" image_arm = true env = { diff --git a/examples/terraform/aws/s3/xdr/terraform/_provider.tf b/examples/terraform/aws/s3/xdr/terraform/_provider.tf deleted file mode 100644 index 872a6309..00000000 --- a/examples/terraform/aws/s3/xdr/terraform/_provider.tf +++ /dev/null @@ -1,4 +0,0 @@ -provider "aws" { - # profile = "default" - region = "us-east-1" -} diff --git a/examples/terraform/aws/s3/xdr/terraform/node.tf b/examples/terraform/aws/s3/xdr/terraform/node.tf index cc994026..e61fdf56 100644 --- a/examples/terraform/aws/s3/xdr/terraform/node.tf +++ b/examples/terraform/aws/s3/xdr/terraform/node.tf @@ -5,7 +5,7 @@ module "lambda_node" { config = { name = "node" description = "Substation node that reads and writes data to S3." - image_uri = "${module.ecr.url}:latest" + image_uri = "${module.ecr.url}:v1.2.0" image_arm = true env = { diff --git a/examples/terraform/aws/sns/pub_sub/terraform/_provider.tf b/examples/terraform/aws/sns/pub_sub/terraform/_provider.tf deleted file mode 100644 index 872a6309..00000000 --- a/examples/terraform/aws/sns/pub_sub/terraform/_provider.tf +++ /dev/null @@ -1,4 +0,0 @@ -provider "aws" { - # profile = "default" - region = "us-east-1" -} diff --git a/examples/terraform/aws/sns/pub_sub/terraform/subscribers.tf b/examples/terraform/aws/sns/pub_sub/terraform/subscribers.tf index fd5a2069..2c56a770 100644 --- a/examples/terraform/aws/sns/pub_sub/terraform/subscribers.tf +++ b/examples/terraform/aws/sns/pub_sub/terraform/subscribers.tf @@ -5,7 +5,7 @@ module "subscriber_x" { config = { name = "subscriber_x" description = "Substation node that subscribes to SNS" - image_uri = "${module.ecr.url}:latest" + image_uri = "${module.ecr.url}:v1.2.0" image_arm = true env = { "SUBSTATION_CONFIG" : "http://localhost:2772/applications/substation/environments/example/configurations/subscriber_x" @@ -45,7 +45,7 @@ module "subscriber_y" { config = { name = "subscriber_y" description = "Substation node that subscribes to SNS" - image_uri = "${module.ecr.url}:latest" + image_uri = "${module.ecr.url}:v1.2.0" image_arm = true env = { "SUBSTATION_CONFIG" : "http://localhost:2772/applications/substation/environments/example/configurations/subscriber_y" @@ -86,7 +86,7 @@ module "subscriber_z" { config = { name = "subscriber_z" description = "Substation node that subscribes to SNS" - image_uri = "${module.ecr.url}:latest" + image_uri = "${module.ecr.url}:v1.2.0" image_arm = true env = { "SUBSTATION_CONFIG" : "http://localhost:2772/applications/substation/environments/example/configurations/subscriber_z" diff --git a/examples/terraform/aws/sqs/microservice/terraform/_provider.tf b/examples/terraform/aws/sqs/microservice/terraform/_provider.tf deleted file mode 100644 index 872a6309..00000000 --- a/examples/terraform/aws/sqs/microservice/terraform/_provider.tf +++ /dev/null @@ -1,4 +0,0 @@ -provider "aws" { - # profile = "default" - region = "us-east-1" -} diff --git a/examples/terraform/aws/sqs/microservice/terraform/frontend.tf b/examples/terraform/aws/sqs/microservice/terraform/frontend.tf index 56d08f7a..940bf32c 100644 --- a/examples/terraform/aws/sqs/microservice/terraform/frontend.tf +++ b/examples/terraform/aws/sqs/microservice/terraform/frontend.tf @@ -5,7 +5,7 @@ module "frontend" { config = { name = "frontend" description = "Substation node that acts as a frontend to an asynchronous microservice" - image_uri = "${module.ecr.url}:latest" + image_uri = "${module.ecr.url}:v1.2.0" image_arm = true env = { "SUBSTATION_CONFIG" : "http://localhost:2772/applications/substation/environments/example/configurations/frontend" diff --git a/examples/terraform/aws/sqs/microservice/terraform/microservice.tf b/examples/terraform/aws/sqs/microservice/terraform/microservice.tf index 7baf1e89..73ec1832 100644 --- a/examples/terraform/aws/sqs/microservice/terraform/microservice.tf +++ b/examples/terraform/aws/sqs/microservice/terraform/microservice.tf @@ -5,7 +5,7 @@ module "microservice" { config = { name = "microservice" description = "Substation node that acts as an asynchronous microservice" - image_uri = "${module.ecr.url}:latest" + image_uri = "${module.ecr.url}:v1.2.0" image_arm = true memory = 128