chef_server_ha.yaml

AWSTemplateFormatVersion: '2010-09-09'
Description: AWS Native Chef Server (auto-scaling HA) v5.1.1

Parameters:
  # Required Parameters
  AutomationBucket:
    Default: 'aws-native-chef-server'
    Type: String
    Description: AWS S3 bucket name that contains all of the cloudformation templates and scripts for this stack
  VPC:
    Description: Choose VPC to use
    Type: AWS::EC2::VPC::Id
  ChefServerSubnets:
    Description: Provide a list of Subnet IDs for the Chef Servers (must be within the specified VPC)
    Type: List<AWS::EC2::Subnet::Id>
  LoadBalancerSubnets:
    Description: Provide a comma separated list of Subnet IDs for the load balancers (Leave blank to use the ChefServerSubnets specified above)
    Type: CommaDelimitedList
  DatabaseSubnets:
    Description: Provide a comma separated list of Subnet IDs for the Postgres database (Leave blank to use the ChefServerSubnets specified above)
    Type: CommaDelimitedList
  ElasticSearchSubnets:
    Description: Provide a comma separated list of Subnet IDs for the ElasticSearch cluster (Leave blank to use the ChefServerSubnets specified above)
    Type: CommaDelimitedList
  SSLCertificateARN:
    Description: SSL Certficate ARN for SSL Certficate
    Type: String
  InboundAdminSecurityGroupId:
    Description: Select an existing Security Group in your VPC to define administrative ACLs (SSH, monitoring tools, etc) to the Chef servers
    Type: AWS::EC2::SecurityGroup::Id
  ImageId:
    Description: The AMI ID to be used for launching instances
    Type: AWS::EC2::Image::Id
  KeyName:
    Description: Name of an existing EC2 KeyPair to enable SSH access to the instance
    Type: AWS::EC2::KeyPair::KeyName
  DBPassword:
    Description: Enter DB Password
    NoEcho: true
    Type: String
  ContactEmail:
    Description: Contact email for Cloudwatch notifications and instance tagging
    Type: String
  ContactDept:
    Description: Contact department for billing purposes
    Type: String
  AlertNotificationTopic:
    Description: AWS SNS Alert notification topic
    Type: String
  ###############################################################################
  # Performance Settings
  InstanceType:
    Description: EC2 Instance type for Chef Server Frontends (high-CPU recommended)
    Default: c5.large
    Type: String
    AllowedValues: [t3.small, t3.medium, t3.large, t3.xlarge, t3.2xlarge,
      m5.large, m5.xlarge, m5.2xlarge, m5.4xlarge, m5.12xlarge, m5.24xlarge,
      c5.large, c5.xlarge, c5.2xlarge, c5.4xlarge, c5.9xlarge, c5.18xlarge]
  MaxFrontendInstances:
    Description: The maximum number of additional frontend instances to launch
    Type: Number
    Default: 3
  MinFrontendInstances:
    Description: The minimum number of frontend instances to launch in addition to the Bootstrap Frontend
    Type: Number
    Default: 1
  LicenseCount:
    Default: '25'
    Description: Enter how many licenses you have purchased
    Type: String
  DBInstanceClass:
    Description: EC2 Instance type for RDS DBs (M5 class recommended for performance)
    Default: 'db.m5.large'
    Type: String
    AllowedValues: [db.t3.small, db.t3.medium, db.t3.large, db.t3.xlarge, db.t3.2xlarge,
      db.m4.large, db.m4.xlarge, db.m4.2xlarge, db.m4.4xlarge, db.m4.10xlarge, db.m4.16xlarge,
      db.r4.large, db.r4.xlarge, db.r4.2xlarge, db.r4.4xlarge, db.r4.8xlarge, db.r4.16xlarge,
      db.m5.large, db.m5.xlarge, db.m5.2xlarge, db.m5.4xlarge, db.m5.12xlarge, db.m5.24xlarge]
  DBStorageType:
    Description: Select gp2 or io1 DB storage. io1 is much higher performing, must select minimum 100GB storage and an IOPS value
    Default: 'gp2'
    Type: String
    AllowedValues: [gp2, io1]
  DBAllocatedStorage:
    Description: Storage size allocated for the database
    Default: '100'
    Type: String
  DBIops:
    Description: IOPS allocated to the storage (storage size * 10)
    Default: '1000'
    Type: String
  DBBackupDaystoKeep:
    Description: Number of days of automatic backups to keep
    Default: 7
    Type: Number
    MinValue: 0
    MaxValue: 35
  ElasticSearchInstanceType:
    Description: The Instance type to use for ElasticSearch instances (Note, must have ephemeral storage, the instance type affects the total amount of elasticsearch storage. i3 strongly recommended)
    Type: String
    Default: 'i3.large.elasticsearch'
    AllowedValues: [
      'i3.large.elasticsearch', 'i3.xlarge.elasticsearch', 'i3.2xlarge.elasticsearch', 'i3.4xlarge.elasticsearch', 'i3.8xlarge.elasticsearch', 'i3.16xlarge.elasticsearch',
      'i2.xlarge.elasticsearch', 'i2.2xlarge.elasticsearch',
      'm3.medium.elasticsearch', 'm3.large.elasticsearch', 'm3.xlarge.elasticsearch', 'm3.medium.elasticsearch',
      'r3.large.elasticsearch', 'r3.xlarge.elasticsearch', 'r3.2xlarge.elasticsearch', 'r3.4xlarge.elasticsearch', 'r3.8xlarge.elasticsearch' ]
  ElasticSearchVersion:
    Description: Version of ElasticSearch to use
    Type: String
    Default: '5.6'
    AllowedValues:
    - '2.3'
    - '5.3'
    - '5.5'
    - '5.6'
  ElasticSearchShardCount:
    Description: Number of ElasticSearch hosts to provision at launch (3 recommended)
    Default: 3
    Type: Number
  ElasticSearchReplicaCount:
    Description: Replication factor for ElasticSearch shards (how many extra copies to keep)
    Default: 2
    Type: Number
  ###############################################################################
  # Security Settings
  LoadBalancerScheme:
    Description: Network Scheme for the ELB
    Type: String
    Default: internet-facing
    AllowedValues:
    - 'internet-facing'
    - 'internal'
  LoadBalancerSecurityGroupId:
    Description: Supply a security group for your load balancer (leave blank to have it created for you). Using the default security group is recommended.
    Type: String
  FrontendSecurityGroupId:
    Description: Supply a security group for your chef frontends (leave blank to have it created for you). Using the default security group is recommended.
    Type: String
  ChefServerAssociatePublicIpAddress:
    Description: Assign public IP addresses to the Chef Servers or not
    Type: String
    Default: true
    AllowedValues:
    - true
    - false
  DisableSignup:
    AllowedValues:
    - 'true'
    - 'false'
    Default: 'false'
    Description: Allow users to sign themselves up to the chef server? (enter true or false)
    Type: String
  ChefServerIamRole:
    Description: Supply an IAM Role for the Chef Servers (leave blank to have it created for you). NOTE If you supply your own role, you must also provide an S3 Bucket which that role can access
    Type: String
  ChefSecretsBucket:
    Description: Supply an S3 Bucket name for the Chef Servers to read/write config files to (leave blank to have it created for you)
    Type: String
  ###############################################################################
  # Other Settings
  ChefAutomateServerUrl:
    Description: The URL of the Chef Automate Server, for forwarding Visibility data.  Leave blank to disable.
    Type: String
    Default: ''
  ChefAutomateToken:
    Description: The secret token used to communicate with the Chef Automate Server, for forwarding Visibility data.  Leave blank to disable.
    Type: String
    Default: ''
  ChefDefaultOrgName:
    Default: ''
    Description: Default Chef organization name (typically only used when migrating from Chef Server 11, otherwise leave blank)
    Type: String
  ChefServerCustomConfig:
    Default: ''
    Type: String
    Description: The S3 location of a custom chef-server configuration snippet that will be injected into the chef-server.rb (optional)
  SupermarketUrl:
    Description: The URL of the Private Supermarket Server, for oc-id authentication.  Leave blank to disable.
    Type: String
    Default: ''
  Route53HostedZone:
    Type: String
    Default: ''
    Description: Supply a Route 53 Hosted Zone name (eg. mydomain.com) for auto-creating a DNS record. Must NOT end in a dot. (Leave blank to disable)
  Route53RecordName:
    Type: String
    Default: 'chef'
    Description: Supply a DNS record name that will be prepended to the Route 53 Hosted Zone
  SystemLogsGroup:
    Description: The logical name of the Cloudwatch Logs log group for systemn logs
    Type: String
  AppLogsGroup:
    Description: The logical name of the Cloudwatch Logs log group for systemn logs
    Type: String
  TemplateVersion:
    Type: String
    Default: "5.1.1"

Metadata:
  cfn-lint:
    config:
      ignore_checks:
        - W2507

Conditions:
  CreateRoute53Record:
    !Not [!Equals [ !Ref Route53HostedZone, '' ] ]
  UseServerSubnetsForLoadBalancers:
    !Equals [ !Select [ 0, !Ref LoadBalancerSubnets], '' ]
  UseServerSubnetsForDatabase:
    !Equals [ !Select [ 0, !Ref DatabaseSubnets], '' ]
  UseServerSubnetsForElasticSearch:
    !Equals [ !Select [ 0, !Ref ElasticSearchSubnets], '' ]

Resources:
# Frontend Autoscale Groups
#########################################################################################
# The first chef server we launch is the 'bootstrap' which needs to come up first and set schema before the rest
  BootstrapAutoScaleGroup:
    Type: AWS::AutoScaling::AutoScalingGroup
    DependsOn:
      - PostgreSQLStack
      - ElasticsearchStack
    Properties:
      LaunchConfigurationName: !Ref ServerLaunchConfig
      TargetGroupARNs:
      - !Ref ChefTargetGroup
      - !Ref ChefBootstrapTargetGroup
      LoadBalancerNames:
      - !Ref ChefPJELB
      MaxSize: '1'
      MinSize: '1'
      NotificationConfigurations:
      - TopicARN: !Ref AlertNotificationTopic
        NotificationTypes:
        - autoscaling:EC2_INSTANCE_LAUNCH
        - autoscaling:EC2_INSTANCE_LAUNCH_ERROR
        - autoscaling:EC2_INSTANCE_TERMINATE
        - autoscaling:EC2_INSTANCE_TERMINATE_ERROR
      Tags:
      - Key: Name
        Value: !Sub ${AWS::StackName}-bootstrap-frontend
        PropagateAtLaunch: true
      - Key: X-Dept
        Value: !Ref ContactDept
        PropagateAtLaunch: true
      - Key: X-Contact
        Value: !Ref ContactEmail
        PropagateAtLaunch: true
      VPCZoneIdentifier: !Ref ChefServerSubnets

# The rest of the servers work in a scalable ASG based on CPU load
  FrontendAutoScaleGroup:
    Type: AWS::AutoScaling::AutoScalingGroup
    DependsOn:
      - BootstrapAutoScaleGroup
      - WaitCondition
    Properties:
      LaunchConfigurationName: !Ref ServerLaunchConfig
      TargetGroupARNs:
      - !Ref ChefTargetGroup
      MaxSize: !Sub '${MaxFrontendInstances}'
      MinSize: !Sub '${MinFrontendInstances}'
      HealthCheckGracePeriod: 900
      HealthCheckType: ELB
      NotificationConfigurations:
      - TopicARN: !Ref AlertNotificationTopic
        NotificationTypes:
        - autoscaling:EC2_INSTANCE_LAUNCH
        - autoscaling:EC2_INSTANCE_LAUNCH_ERROR
        - autoscaling:EC2_INSTANCE_TERMINATE
        - autoscaling:EC2_INSTANCE_TERMINATE_ERROR
      Tags:
      - Key: Name
        Value: !Sub ${AWS::StackName}-frontend
        PropagateAtLaunch: true
      - Key: X-Dept
        Value: !Ref ContactDept
        PropagateAtLaunch: true
      - Key: X-Contact
        Value: !Ref ContactEmail
        PropagateAtLaunch: true
      VPCZoneIdentifier: !Ref ChefServerSubnets

  FrontendAutoScaleUpPolicy:
    Type: AWS::AutoScaling::ScalingPolicy
    Properties:
      AdjustmentType: ChangeInCapacity
      AutoScalingGroupName: !Ref FrontendAutoScaleGroup
      Cooldown: '60'
      ScalingAdjustment: 1

  FrontendAutoScaleDownPolicy:
    Type: AWS::AutoScaling::ScalingPolicy
    Properties:
      AdjustmentType: ChangeInCapacity
      AutoScalingGroupName: !Ref FrontendAutoScaleGroup
      Cooldown: '60'
      ScalingAdjustment: -1

  ChefInstanceProfile:
    Type: AWS::IAM::InstanceProfile
    Properties:
      Path: /
      Roles:
      - !Ref ChefServerIamRole

  # A special ELB just for Push Jobs traffic (TCP ports 10000-10003)
  # Note: Push Jobs server isn't truly HA. Although job state is stored in the database,
  #       all node state is stored ONLY in memory. Therefore we can only have 1
  #       active push jobs server at a time. An elegant solution for this in AWS
  #       is to have all Push Jobs traffic only routed to the BootStrap ASG
  ChefPJELB:
    Type: "AWS::ElasticLoadBalancing::LoadBalancer"
    Properties:
      SecurityGroups:
        - !Ref LoadBalancerSecurityGroupId
      Subnets: !If [UseServerSubnetsForLoadBalancers, !Ref ChefServerSubnets, !Ref LoadBalancerSubnets ]
      Scheme: !Ref LoadBalancerScheme
      Listeners:
        - LoadBalancerPort: '10000'
          InstancePort: '10000'
          Protocol: TCP
        - LoadBalancerPort: '10002'
          InstancePort: '10002'
          Protocol: TCP
        - LoadBalancerPort: '10003'
          InstancePort: '10003'
          Protocol: TCP
      Tags:
        - Key: Name
          Value: chef-server-pushjobs
        - Key: X-Dept
          Value: !Ref ContactDept
        - Key: X-Contact
          Value: !Ref ContactEmail

  ChefALB:
    Type: "AWS::ElasticLoadBalancingV2::LoadBalancer"
    Properties:
      SecurityGroups:
        - !Ref LoadBalancerSecurityGroupId
      Subnets: !If [UseServerSubnetsForLoadBalancers, !Ref ChefServerSubnets, !Ref LoadBalancerSubnets ]
      # IpAddressType: dualstack
      Scheme: !Ref LoadBalancerScheme
      Tags:
        - Key: Name
          Value: !Sub chef-${AWS::StackName}
        - Key: StackName
          Value: !Sub ${AWS::StackName}
        - Key: X-Dept
          Value: !Ref ContactDept
        - Key: X-Contact
          Value: !Ref ContactEmail

  ChefALBListener:
    Type: "AWS::ElasticLoadBalancingV2::Listener"
    Properties:
        Certificates:
          - CertificateArn: !Ref SSLCertificateARN
        LoadBalancerArn: !Ref ChefALB
        Port: 443
        Protocol: HTTPS
        SslPolicy: ELBSecurityPolicy-TLS-1-2-2017-01
        DefaultActions:
        - Type: forward
          TargetGroupArn: !Ref ChefTargetGroup

  # Ensure that requests to the Push Jobs ("pushy") API are only routed to the bootstrap machine
  ChefALBPJListenerRule:
    Type: "AWS::ElasticLoadBalancingV2::ListenerRule"
    Properties:
      Actions:
        - Type: forward
          TargetGroupArn: !Ref ChefBootstrapTargetGroup
      Conditions:
        - Field: path-pattern
          Values:
            - "/organizations/*/pushy/*"
      ListenerArn: !Ref ChefALBListener
      Priority: 1

  ChefTargetGroup:
      Type: "AWS::ElasticLoadBalancingV2::TargetGroup"
      Properties:
        HealthCheckIntervalSeconds: 30
        UnhealthyThresholdCount: 2
        HealthyThresholdCount: 5
        HealthCheckPath: /_status
        VpcId: !Ref VPC
        Port: 443
        Protocol: HTTPS
        TargetGroupAttributes:
          - Key: stickiness.enabled
            Value: 'true'
          - Key: stickiness.type
            Value: lb_cookie
        Tags:
          - Key: X-Dept
            Value: !Ref ContactDept
          - Key: X-Contact
            Value: !Ref ContactEmail

  ChefBootstrapTargetGroup:
      Type: "AWS::ElasticLoadBalancingV2::TargetGroup"
      Properties:
        HealthCheckIntervalSeconds: 30
        UnhealthyThresholdCount: 2
        HealthyThresholdCount: 5
        HealthCheckPath: /_status
        VpcId: !Ref VPC
        Port: 443
        Protocol: HTTPS
        TargetGroupAttributes:
          - Key: stickiness.enabled
            Value: 'true'
          - Key: stickiness.type
            Value: lb_cookie
        Tags:
          - Key: X-Dept
            Value: !Ref ContactDept
          - Key: X-Contact
            Value: !Ref ContactEmail

  ServerLaunchConfig:
    Type: AWS::AutoScaling::LaunchConfiguration
    Properties:
      AssociatePublicIpAddress: !Ref ChefServerAssociatePublicIpAddress
      EbsOptimized: true
      BlockDeviceMappings:
      - DeviceName: /dev/sda1
        Ebs:
          VolumeSize: 20
          VolumeType: gp2
          DeleteOnTermination: true
      IamInstanceProfile: !Ref ChefInstanceProfile
      ImageId: !Ref ImageId
      InstanceType: !Ref InstanceType
      KeyName: !Ref KeyName
      SecurityGroups:
      - !Ref FrontendSecurityGroupId
      - !Ref InboundAdminSecurityGroupId
      UserData:
        "Fn::Base64": !Sub |
          #!/bin/bash -ex
          # Uncomment this next line before going to production, note: It can cause instances to cycle rapidly, watch Cloudwatch Logs for errors
          # trap "aws autoscaling set-instance-health --health-status Unhealthy --region ${AWS::Region} --instance-id $(curl -s http://169.254.169.254/latest/meta-data/instance-id)" ERR
          # Execute AWS::CloudFormation::Init
          /opt/aws/bin/cfn-init -v --stack ${AWS::StackName} --resource ServerLaunchConfig --region ${AWS::Region}
          # All is well so signal success and let CF know wait function is complete, ignore error after stack creation, CF will handle the missing signal anyway
          /opt/aws/bin/cfn-signal -e 0 -r "Server setup complete" '${WaitHandle}' || echo "Ignore error"
    Metadata:
      AWS::CloudFormation::Init:
        configSets:
          default: [awslogs, es_proxy_setup, chef_server_setup, filebeat, final_stage]
        es_proxy_setup:
          files:
            /etc/systemd/system/aws-signing-proxy.service:
              content: |
                [Unit]
                Description=aws-signing-proxy
                After=network.target
                #
                [Service]
                ExecStart=/usr/local/bin/aws-signing-proxy
                Restart=always
                #
                [Install]
                WantedBy=multi-user.target
            /etc/aws-signing-proxy.yml:
              content: !Sub |
                listen-address: 127.0.0.1
                port: 9200
                target: https://${ElasticsearchStack.Outputs.DomainEndpoint}
                region: ${AWS::Region}
                no-file-log: false
                stdout-log: true
          commands:
            01_start_es_proxy:
              command: |
                systemctl daemon-reload && systemctl start aws-signing-proxy
        chef_server_setup:
          files:
            /etc/opscode-push-jobs-server/opscode-push-jobs-server.rb:
              content: !Sub |
                opscode_pushy_server['vip'] = '127.0.0.1'
                opscode_pushy_server['server_name_advertised'] = '${ChefPJELB.DNSName}'.downcase
            /etc/chef-manage/manage.rb:
              content: !Sub |
                disable_sign_up ${DisableSignup}
            /etc/opscode/chef-server.rb:
              content: !Sub
              - |
                api_fqdn '${DNSNAME}'.downcase
                nginx['enable_non_ssl'] = true
                license['nodes'] = ${LicenseCount}
                postgresql['external'] = true
                postgresql['vip'] = '${PostgreSQLStack.Outputs.EndpointAddress}'
                postgresql['db_superuser'] = 'chefadmin'
                postgresql['db_superuser_password'] = '${DBPassword}'
                oc_chef_authz['http_init_count'] = 100
                oc_chef_authz['http_queue_max'] = 200
                opscode_erchef['authz_pooler_timeout'] = 2000
                oc_bifrost['db_pool_init'] = 10
                oc_bifrost['db_pool_max'] = 20
                oc_bifrost['db_pool_queue_max'] = 40
                opscode_erchef['depsolver_worker_count'] = 4
                opscode_erchef['depsolver_timeout'] = 20000
                opscode_erchef['db_pool_init'] = 10
                opscode_erchef['db_pool_max'] = 20
                opscode_erchef['db_pool_queue_max'] = 40
                opscode_erchef['keygen_cache_workers'] = 2
                opscode_erchef['keygen_cache_size'] = 100
                opscode_erchef['nginx_bookshelf_caching'] = :on
                opscode_erchef['s3_url_expiry_window_size'] = '100%'
                opscode_erchef['s3_parallel_ops_fanout'] = 10
                opscode_erchef['search_provider'] = 'elasticsearch'
                opscode_erchef['search_queue_mode'] = 'batch'
                opscode_solr4['external'] = true
                opscode_solr4['external_url'] = 'http://localhost:9200'
                opscode_solr4['elasticsearch_shard_count'] = ${ElasticSearchShardCount}
                opscode_solr4['elasticsearch_replica_count'] = ${ElasticSearchReplicaCount}
                bookshelf['storage_type'] = :sql
                bookshelf['db_pool_init'] = 10
                bookshelf['db_pool_max'] = 20
                bookshelf['vip'] = '${DNSNAME}'.downcase
                rabbitmq['enable'] = false
                rabbitmq['management_enabled'] = false
                rabbitmq['queue_length_monitor_enabled'] = false
                opscode_expander['enable'] = false
                dark_launch['actions'] = false
                unless '${SupermarketUrl}'.empty?
                  oc_id['applications'] ||= {}
                  oc_id['applications']['supermarket'] = {
                    'redirect_uri' => '${SupermarketUrl}/auth/chef_oauth2/callback'
                  }
                end
                default_orgname '${ChefDefaultOrgName}'.empty? ? null : '${ChefDefaultOrgName}'
                # special handling for Chef Automate URL and Token which might be empty strings
                unless '${ChefAutomateServerUrl}'.empty? || '${ChefAutomateToken}'.empty?
                  data_collector['root_url'] = '${ChefAutomateServerUrl}/data-collector/v0/'
                  data_collector['token'] = '${ChefAutomateToken}'
                  profiles['root_url'] = '${ChefAutomateServerUrl}'
                end
              - { DNSNAME: !If [CreateRoute53Record, !Sub "${Route53RecordName}.${Route53HostedZone}", !GetAtt ChefALB.DNSName] }
          commands:
            02_before_script:
              command: !Sub |
                # If we're using a marketplace, install locally cached packages
                pushd /var/cache/marketplace
                rpm -ivh chef-server-core.rpm chef-manage.rpm push-jobs-server.rpm
                popd
                if [ -n '${ChefServerCustomConfig}' ]; then
                    aws s3 cp ${ChefServerCustomConfig} /etc/opscode/chef-server-custom.rb
                    echo "# Appended from chef-server-custom.rb" >> /etc/opscode/chef-server.rb
                    cat /etc/opscode/chef-server-custom.rb >> /etc/opscode/chef-server.rb
                fi
            03_configure_chef_server:
              command: !Sub |
                export STACKNAME=${AWS::StackName}
                export BUCKET=${ChefSecretsBucket}
                export AWS_REGION=${AWS::Region}
                export WAITHANDLE="${WaitHandle}"

                bash -x /var/cache/marketplace/main.sh
            05_configure_cloudwatch_monitoring:
              command: |
                crontab -l | { cat; echo "*/5 * * * * /opt/cloudwatch_monitoring/aws-scripts-mon/mon-put-instance-data.pl --auto-scaling --mem-util --disk-space-util --disk-path=/ --from-cron"; } | crontab -
        awslogs:
          files:
            /etc/awslogs/awscli.conf:
              content: !Sub |
                [default]
                region = ${AWS::Region}
                [plugins]
                cwlogs = cwlogs
              mode: '000644'
              owner: root
              group: root
            /etc/awslogs/awslogs.conf:
              content: !Sub |
                [general]
                state_file = /var/lib/awslogs/agent-state
                [/var/log/messages]
                datetime_format = %b %d %H:%M:%S
                file = /var/log/messages
                log_stream_name = {instance_id}
                log_group_name = ${SystemLogsGroup}
                [/var/log/secure]
                datetime_format = %b %d %H:%M:%S
                file = /var/log/secure
                log_stream_name = {instance_id}
                log_group_name = ${SystemLogsGroup}
                [/var/log/cron]
                datetime_format = %b %d %H:%M:%S
                file = /var/log/cron
                log_stream_name = {instance_id}
                log_group_name = ${SystemLogsGroup}
                [/var/log/cloud-init.log]
                datetime_format = %b %d %H:%M:%S
                file = /var/log/cloud-init.log
                log_stream_name = {instance_id}
                log_group_name = ${SystemLogsGroup}
                [/var/log/cfn-init.log]
                datetime_format = %Y-%m-%d %H:%M:%S
                file = /var/log/cfn-init.log
                log_stream_name = {instance_id}
                log_group_name = ${SystemLogsGroup}
                [/var/log/cfn-init-cmd.log]
                datetime_format = %Y-%m-%d %H:%M:%S
                file = /var/log/cfn-init-cmd.log
                log_stream_name = {instance_id}
                log_group_name = ${SystemLogsGroup}
                [/var/log/dmesg]
                file = /var/log/dmesg
                log_stream_name = {instance_id}
                log_group_name = ${SystemLogsGroup}
                [/var/log/aws-signing-proxy/proxy.log]
                log_group_name = ${AppLogsGroup}
                file = /var/log/aws-signing-proxy/proxy.log
                log_stream_name = {instance_id}
                [/var/log/opscode/bookshelf/current]
                log_group_name = ${AppLogsGroup}
                file = /var/log/opscode/bookshelf/current
                log_stream_name = {instance_id}
                [/var/log/opscode/oc_bifrost/current]
                log_group_name = ${AppLogsGroup}
                file = /var/log/opscode/oc_bifrost/current
                log_stream_name = {instance_id}
                [/var/log/opscode/opscode-erchef/current]
                log_group_name = ${AppLogsGroup}
                file = /var/log/opscode/opscode-erchef/current
                log_stream_name = {instance_id}
                [/var/log/opscode/opscode-erchef/requests.log]
                log_group_name = ${AppLogsGroup}
                file = /var/log/opscode/opscode-erchef/requests.log.*
                log_stream_name = {instance_id}
                [/var/log/opscode/opscode-pushy-server/current]
                log_group_name = ${AppLogsGroup}
                file = /var/log/opscode/opscode-pushy-server/current
                log_stream_name = {instance_id}
                [/var/log/opscode/redis_lb/current]
                log_group_name = ${AppLogsGroup}
                file = /var/log/opscode/redis_lb/current
                log_stream_name = {instance_id}
                [/var/log/opscode/oc_id/current]
                log_group_name = ${AppLogsGroup}
                file = /var/log/opscode/oc_id/current
                log_stream_name = {instance_id}
                [/var/log/opscode/nginx/access.log]
                log_group_name = ${AppLogsGroup}
                file = /var/log/opscode/nginx/access.log
                log_stream_name = {instance_id}
                [/var/log/opscode/nginx/error.log]
                log_group_name = ${AppLogsGroup}
                file = /var/log/opscode/nginx/error.log
                log_stream_name = {instance_id}
                [/var/log/chef-manage/redis/current]
                log_group_name = ${AppLogsGroup}
                file = /var/log/chef-manage/redis/current
                log_stream_name = {instance_id}
                [/var/log/chef-manage/web/current]
                log_group_name = ${AppLogsGroup}
                file = /var/log/chef-manage/web/current
                log_stream_name = {instance_id}
                [/var/log/chef-manage/worker/current]
                log_group_name = ${AppLogsGroup}
                file = /var/log/chef-manage/worker/current
                log_stream_name = {instance_id}
              mode: '000644'
              owner: root
              group: root
          commands:
            01_setup:
              command: !Sub |
                mkdir -p /var/lib/awslogs
                python /var/cache/marketplace/awslogs-agent-setup.py --region ${AWS::Region} --non-interactive -c /etc/awslogs/awslogs.conf
        filebeat:
          files:
            /etc/filebeat/filebeat.yml:
              content: |
                filebeat.prospectors:
                - input_type: log
                  paths:
                    - /var/log/*
                    - /var/log/opscode/*/current
                    - /var/log/opscode/nginx/*.log
                    - /var/log/opscode/opscode-erchef/requests.log.*
                    - /var/log/chef-manage/*/current
                output.elasticsearch:
                  hosts: ["localhost:9200"]
                setup.kibana:
                  host: "http://localhost:9200/_plugin/kibana/"
                dashboards.enabled: true
              mode: '000644'
              owner: root
              group: root
          commands:
            01_systemd:
              command: |
                rpm -ivh /var/cache/marketplace/filebeat-5.6.6-x86_64.rpm
                systemctl daemon-reload && systemctl start filebeat.service
        final_stage:
          commands:
            09_cleanup:
              command: |
                # clean up the chef package cache, to maximize available disk space after install
                if [ -d /var/cache/marketplace ]; then
                    rm -f /var/cache/marketplace/*.rpm
                    rm -f /var/cache/marketplace/*.aib
                fi

  WaitCondition:
    Type: AWS::CloudFormation::WaitCondition
    DependsOn: ServerLaunchConfig
    Properties:
      Handle: !Ref WaitHandle
      Timeout: '1500'

  WaitHandle:
    Type: AWS::CloudFormation::WaitConditionHandle

# PostgreSQL DB
#########################################################################################
  PostgreSQLStack:
    Type: AWS::CloudFormation::Stack
    Properties:
      TemplateURL: !Sub https://${AutomationBucket}.s3.amazonaws.com/${TemplateVersion}/chef_rds.yaml
      Parameters:
        VPC: !Sub ${VPC}
        ChefServerSubnets: !If [UseServerSubnetsForDatabase, !Join [ ",", !Ref ChefServerSubnets ], !Join [ ",", !Ref DatabaseSubnets ] ]
        DBPassword: !Ref DBPassword
        ContactEmail: !Ref ContactEmail
        ContactDept: !Ref ContactDept
        DBInstanceClass: !Ref DBInstanceClass
        DBStorageType: !Ref DBStorageType
        DBAllocatedStorage: !Ref DBAllocatedStorage
        DBIops: !Ref DBIops
        BackupDaystoKeep: !Ref DBBackupDaystoKeep
        FrontendSecurityGroupId: !Ref FrontendSecurityGroupId
        AlertNotificationTopic: !Ref AlertNotificationTopic

# ElasticSearch
#########################################################################################
  ElasticsearchStack:
    Type: AWS::CloudFormation::Stack
    Properties:
      TemplateURL: !Sub https://${AutomationBucket}.s3.amazonaws.com/${TemplateVersion}/chef_elasticsearch.yaml
      Parameters:
        VPC: !Sub ${VPC}
        ChefServerSubnets: !If [UseServerSubnetsForElasticSearch, !Join [ ",", !Ref ChefServerSubnets ], !Join [ ",", !Ref ElasticSearchSubnets ] ]
        ContactEmail: !Ref ContactEmail
        ContactDept: !Ref ContactDept
        ElasticSearchInstanceType: !Ref ElasticSearchInstanceType
        ElasticSearchVersion: !Ref ElasticSearchVersion
        ElasticSearchShardCount: !Ref ElasticSearchShardCount
        FrontendSecurityGroupId: !Ref FrontendSecurityGroupId
        ChefRole: !Sub "arn:aws:iam::${AWS::AccountId}:role/${ChefServerIamRole}"
        AlertNotificationTopic: !Ref AlertNotificationTopic

# Route 53 Record
#########################################################################################
  ChefLBDNSRecord:
    Type: AWS::Route53::RecordSet
    Condition: CreateRoute53Record
    Properties:
      HostedZoneName: !Sub ${Route53HostedZone}.
      Name: !Sub ${Route53RecordName}.${Route53HostedZone}.
      Comment: !Sub Created by Cloudformation ${AWS::StackName}
      Type: CNAME
      TTL: 900
      ResourceRecords:
      - !GetAtt ChefALB.DNSName


# Operations Dashboard (Cloudwatch)
#########################################################################################
  OpsDashboardStack:
    Type: AWS::CloudFormation::Stack
    Properties:
      TemplateURL: !Sub https://${AutomationBucket}.s3.amazonaws.com/${TemplateVersion}/chef_server_ops_dashboard.yaml
      Parameters:
        BootstrapAutoScaleGroup: !Ref BootstrapAutoScaleGroup
        FrontendAutoScaleGroup: !Ref FrontendAutoScaleGroup
        ElasticsearchDomain: !GetAtt ElasticsearchStack.Outputs.ElasticsearchDomain
        DBPostgres: !GetAtt PostgreSQLStack.Outputs.DBPostgres
        TargetGroupFullName: !GetAtt ChefTargetGroup.TargetGroupFullName
        LoadBalancerFullName: !GetAtt ChefALB.LoadBalancerFullName

# Monitoring
#########################################################################################
  DiskSpaceUtilizationNvme:
    Type: AWS::CloudWatch::Alarm
    Properties:
      AlarmName: !Sub DiskSpaceUtilizationNvme-${AWS::StackName}
      AlarmDescription: Alarms when an disk utilization reaches a specified threshold.
      AlarmActions: [!Ref AlertNotificationTopic]
      MetricName: DiskSpaceUtilization
      Namespace: System/Linux
      ComparisonOperator: GreaterThanOrEqualToThreshold
      EvaluationPeriods: 2
      Period: 300
      Statistic: Average
      Threshold: 80
      Unit: Percent
      Dimensions:
        - Name: AutoScalingGroupName
          Value: !Ref FrontendAutoScaleGroup
        - Name: MountPath
          Value: '/'
        - Name: Filesystem
          Value: '/dev/nvme0n1p1'

  BootstrapDiskSpaceUtilizationNvme:
    Type: AWS::CloudWatch::Alarm
    Properties:
      AlarmName: !Sub DiskSpaceUtilizationNvme-bootstrap-${AWS::StackName}
      AlarmDescription: Alarms when an disk utilization reaches a specified threshold.
      AlarmActions: [!Ref AlertNotificationTopic]
      MetricName: DiskSpaceUtilization
      Namespace: System/Linux
      ComparisonOperator: GreaterThanOrEqualToThreshold
      EvaluationPeriods: 2
      Period: 300
      Statistic: Average
      Threshold: 80
      Unit: Percent
      Dimensions:
        - Name: AutoScalingGroupName
          Value: !Ref BootstrapAutoScaleGroup
        - Name: MountPath
          Value: '/'
        - Name: Filesystem
          Value: '/dev/nvme0n1p1'

  MemoryUtilization:
    Type: AWS::CloudWatch::Alarm
    Properties:
      AlarmName: !Sub MemoryUtilization-${AWS::StackName}
      AlarmDescription: Alarms when an memory utilization reaches a specified threshold.
      AlarmActions: [!Ref AlertNotificationTopic]
      MetricName: MemoryUtilization
      Namespace: System/Linux
      ComparisonOperator: GreaterThanOrEqualToThreshold
      EvaluationPeriods: 2
      Period: 300
      Statistic: Average
      Threshold: 80
      Unit: Percent
      Dimensions:
        - Name: AutoScalingGroupName
          Value: !Ref FrontendAutoScaleGroup

  ELB5XXExceeded:
    Type: AWS::CloudWatch::Alarm
    Properties:
      AlarmName: !Sub AppELB_5XX_Exceeded-${AWS::StackName}
      AlarmDescription: Alarms when an 5xx requests exceed a specified threshold.
      AlarmActions: [!Ref AlertNotificationTopic]
      MetricName: HTTPCode_ELB_5XX_Count
      Namespace: AWS/ApplicationELB
      ComparisonOperator: GreaterThanOrEqualToThreshold
      EvaluationPeriods: 2
      Period: 60
      Statistic: Sum
      Threshold: 10
      Dimensions:
        - Name: LoadBalancer
          Value: !GetAtt ChefALB.LoadBalancerFullName

  BootstrapTGUnhealthyHosts:
    Type: AWS::CloudWatch::Alarm
    Properties:
      AlarmName: !Sub Chef-Bootstrap-TG-Unhealthy-Hosts-${AWS::StackName}
      AlarmDescription: Alarms when UnHealthyHostCount is greater than threshold.
      AlarmActions: [!Ref AlertNotificationTopic]
      MetricName: UnHealthyHostCount
      Namespace: AWS/ApplicationELB
      ComparisonOperator: GreaterThanOrEqualToThreshold
      EvaluationPeriods: 2
      Period: 60
      Statistic: Sum
      Threshold: 1
      Dimensions:
        - Name: LoadBalancer
          Value: !GetAtt ChefALB.LoadBalancerFullName
        - Name: TargetGroup
          Value: !GetAtt ChefBootstrapTargetGroup.TargetGroupFullName

  FETGUnhealthyHosts:
    Type: AWS::CloudWatch::Alarm
    Properties:
      AlarmName: !Sub Chef-FE-TG-Unhealthy-Hosts-${AWS::StackName}
      AlarmDescription: Alarms when UnHealthyHostCount is greater than threshold.
      AlarmActions: [!Ref AlertNotificationTopic]
      MetricName: UnHealthyHostCount
      Namespace: AWS/ApplicationELB
      ComparisonOperator: GreaterThanOrEqualToThreshold
      EvaluationPeriods: 2
      Period: 60
      Statistic: Sum
      Threshold: 1
      Dimensions:
        - Name: LoadBalancer
          Value: !GetAtt ChefALB.LoadBalancerFullName
        - Name: TargetGroup
          Value: !GetAtt ChefTargetGroup.TargetGroupFullName

  CPUAlarmHigh:
    Type: AWS::CloudWatch::Alarm
    Properties:
      AlarmName: !Sub CPUAlarmHigh-${AWS::StackName}
      AlarmDescription: Scale up when CPU > 60% for 5 minutes
      AlarmActions: [!Ref FrontendAutoScaleUpPolicy]
      MetricName: CPUUtilization
      Namespace: AWS/EC2
      ComparisonOperator: GreaterThanOrEqualToThreshold
      EvaluationPeriods: 5
      Period: 60
      Statistic: Average
      Threshold: 60
      Dimensions:
        - Name: AutoScalingGroupName
          Value: !Ref FrontendAutoScaleGroup

  CPUAlarmLow:
    Type: AWS::CloudWatch::Alarm
    Properties:
      AlarmName: !Sub CPUAlarmLow-${AWS::StackName}
      AlarmDescription: Scale down when CPU < 40% for 10 minutes
      AlarmActions: [!Ref FrontendAutoScaleDownPolicy]
      MetricName: CPUUtilization
      Namespace: AWS/EC2
      ComparisonOperator: LessThanOrEqualToThreshold
      EvaluationPeriods: 10
      Period: 60
      Statistic: Average
      Threshold: 40
      Dimensions:
        - Name: AutoScalingGroupName
          Value: !Ref FrontendAutoScaleGroup

Outputs:
  DNSName:
    Description: The DNS name of the load balancer
    Value: !If [CreateRoute53Record, !Sub "${Route53RecordName}.${Route53HostedZone}", !GetAtt ChefALB.DNSName]
    Export:
      Name: !Sub "${AWS::StackName}-DNSName"
  StackName:
    Description: The name of this stack, as an export
    Value: !Ref AWS::StackName