.gitlab-ci.yml

stages:
  - source_test
  - binary_build
  - integration_test
  - package_build
  - check_deploy
  - testkitchen_deploy
  - testkitchen_testing
  - testkitchen_cleanup
  - image_build
  - image_deploy
  - deploy
  - deploy_invalidate
  - e2e

variables:
  SRC_PATH: /src/github.com/DataDog/datadog-agent
  OMNIBUS_BASE_DIR: $CI_PROJECT_DIR/.omnibus/
  OMNIBUS_PACKAGE_DIR: $CI_PROJECT_DIR/.omnibus/pkg/
  # make sure the types of RPM packages are kept separate
  OMNIBUS_BASE_DIR_SUSE: $CI_PROJECT_DIR/.omnibus/suse/
  OMNIBUS_PACKAGE_DIR_SUSE: $CI_PROJECT_DIR/.omnibus/suse/pkg/
  DD_AGENT_TESTING_DIR: $CI_PROJECT_DIR/test/kitchen
  STATIC_BINARIES_DIR: bin/static
  DOGSTATSD_BINARIES_DIR: bin/dogstatsd
  AGENT_BINARIES_DIR: bin/agent
  CLUSTER_AGENT_BINARIES_DIR: bin/datadog-cluster-agent
  DEB_S3_BUCKET: apt.datad0g.com
  RPM_S3_BUCKET: yum.datad0g.com
  WIN_S3_BUCKET: dd-agent-mstesting
  DEB_RPM_BUCKET_BRANCH: nightly  # branch of the DEB_S3_BUCKET and RPM_S3_BUCKET repos to release to, 'nightly' or 'beta'
  DEB_TESTING_S3_BUCKET: apttesting.datad0g.com
  RPM_TESTING_S3_BUCKET: yumtesting.datad0g.com
  WINDOWS_TESTING_S3_BUCKET: $WIN_S3_BUCKET/pipelines/$CI_PIPELINE_ID
  WINDOWS_BUILDS_S3_BUCKET: $WIN_S3_BUCKET/builds
  DEB_RPM_TESTING_BUCKET_BRANCH: testing  # branch of the DEB_TESTING_S3_BUCKET and RPM_TESTING_S3_BUCKET repos to release to, 'testing'
  DD_REPO_BRANCH_NAME: $CI_COMMIT_REF_NAME
  S3_CP_OPTIONS: --only-show-errors --region us-east-1 --sse AES256
  S3_CP_CMD: aws s3 cp $S3_CP_OPTIONS
  S3_ARTEFACTS_URI: s3://dd-ci-artefacts-build-stable/$CI_PROJECT_NAME/$CI_PIPELINE_ID
  S3_OMNIBUS_CACHE_BUCKET: dd-ci-datadog-agent-omnibus-cache-build-stable
  S3_DSD6_URI: s3://dsd6-staging/linux
  RELEASE_VERSION: nightly


# Default before_script for all the jobs. If you create a new job and don't want this to execute
# you NEED to overwrite it.
before_script:
  # We need to install go deps from within the GOPATH, which we set to / on builder images; that's because pointing
  # GOPATH to the project folder would be too complex (we'd need to replicate the `src/github/project` scheme).
  # So we copy the agent sources to / and bootstrap from there the vendor dependencies before running any job.
  - echo running default before_script
  - rsync -azr --delete ./ $SRC_PATH
  - cd $SRC_PATH
  - pip install -U pip
  - inv -e deps

#
# Trigger conditions
#

# run job only when triggered by an external tool (ex: Jenkins). This is used
# for jobs that run both on nightlies and tags
.run_when_triggered: &run_when_triggered
  only:
    - triggers

# run job only when triggered by an external tool (ex: Jenkins) and when
# RELEASE_VERSION is NOT "nightly". In this setting we are building either a
# new tagged version of the agent (an RC for example). In both cases the
# artifacts should be uploaded to our staging repository.

.run_when_triggered_on_tag: &run_when_triggered_on_tag
  only:
    refs:
      - triggers
  except: # we have to use except since gitlab doens't handle '!=' operator
    variables:
      - $RELEASE_VERSION == "nightly"
      - $RELEASE_VERSION == "" # no  RELEASE_VERSION means a nightly build for omnibus

# run job only when triggered by an external tool (ex: Jenkins) and when
# RELEASE_VERSION is "nightly". In this setting we build from master and update
# the nightly build for windows, linux and docker.

.run_when_triggered_on_nightly: &run_when_triggered_on_nightly
  only:
    refs:
      - triggers
    variables:
      - $RELEASE_VERSION == "nightly"

#
# source_test
#

# run tests for deb-x64
run_tests_deb-x64:
  stage: source_test
  image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-builders/deb_x64:latest
  tags: [ "runner:main", "size:large" ]
  script:
    - inv -e test --race --profile

# run tests for rpm-x64
run_test_rpm-x64:
  stage: source_test
  image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-builders/rpm_x64:latest
  tags: [ "runner:main", "size:large" ]
  script:
    - inv -e test --race --profile

# scan the dependencies for security vulnerabilities with snyk
run_security_scan_test:
  stage: source_test
  image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/snyk:latest
  tags: ["runner:main", "size:large"]
  before_script:
    # this image isn't built in the datadog-agent-builders repo
    # it doesn't have invoke so we install the dependencies without invoke
    - mkdir -p $GOPATH/src/github.com/DataDog/datadog-agent
    - rsync -azr --delete ./ $GOPATH/src/github.com/DataDog/datadog-agent
    - cd $GOPATH/src/github.com/DataDog/datadog-agent
    - pip install invoke
    - inv deps
  script:
    - set +x     # don't print the api key to the logs
    - SNYK_TOKEN=$(aws ssm get-parameter --region us-east-1 --name ci.datadog-agent.snyk_token --with-decryption --query "Parameter.Value" --out text)
      snyk monitor # send the list of the dependencies to snyk
  only:
    - master

#
# binary_build
#


# build dogstatsd static for deb-x64
build_dogstatsd_static-deb_x64:
  stage: binary_build
  image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-builders/deb_x64:latest
  tags: [ "runner:main", "size:large" ]
  script:
    - inv -e dogstatsd.build --static
    - $S3_CP_CMD $SRC_PATH/$STATIC_BINARIES_DIR/dogstatsd $S3_ARTEFACTS_URI/static/dogstatsd

# build puppy agent for deb-x64, to make sure the build is not broken because of build flags
build_puppy_agent-deb_x64:
  stage: binary_build
  image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-builders/deb_x64:latest
  tags: [ "runner:main", "size:large" ]
  script:
    - inv -e agent.build --puppy
    - $S3_CP_CMD $SRC_PATH/$AGENT_BINARIES_DIR/agent $S3_ARTEFACTS_URI/puppy/agent

# build puppy agent for ARM, to make sure the build is not broken because of build targets
build_puppy_agent-deb_x64_arm:
  stage: binary_build
  image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-builders/deb_x64:latest
  tags: [ "runner:main", "size:large" ]
  script:
    - GOOS=linux GOARCH=arm inv -e agent.build --puppy

# build dogstatsd for deb-x64
build_dogstatsd-deb_x64:
  stage: binary_build
  image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-builders/deb_x64:latest
  tags: [ "runner:main", "size:large" ]
  script:
    - inv -e dogstatsd.build
    - $S3_CP_CMD $SRC_PATH/$DOGSTATSD_BINARIES_DIR/dogstatsd $S3_ARTEFACTS_URI/dogstatsd/dogstatsd

# build cluster-agent bin
cluster_agent-build:
  stage: binary_build
  image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-builders/deb_x64:latest
  tags: [ "runner:main", "size:large" ]
  script:
    - inv -e cluster-agent.build
    - $S3_CP_CMD $SRC_PATH/$CLUSTER_AGENT_BINARIES_DIR/datadog-cluster-agent $S3_ARTEFACTS_URI/datadog-cluster-agent

#
# integration_test


# run benchmarks on deb
# run_benchmarks-deb_x64:
#   stage: integration_test
#   image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-builders/deb_x64:latest
#   allow_failure: true  # FIXME: this was set to true to temporarily unblock the pipeline
#   tags: [ "runner:main", "size:large" ]
#   script:
#     - inv -e bench.aggregator
#     # FIXME: in our docker image, non ascii characters printed by the benchmark
#     # make invoke traceback. For now, the workaround is to call the benchmarks
#     # manually
#     - inv -e bench.build-dogstatsd

#     - set +x # make sure we don't output the creds to the build log
#     - DD_AGENT_API_KEY=$(aws ssm get-parameter --region us-east-1 --name ci.datadog-agent.dd_agent_api_key --with-decryption --query "Parameter.Value" --out text)

#     # dogstatsd validation - not really benchmarking: gitlab isn't the right place to do this.
#     - ./bin/benchmarks/dogstatsd -pps=20000 -dur 30 -ser 5 -branch $DD_REPO_BRANCH_NAME -api-key $DD_AGENT_API_KEY
#   artifacts:
#     expire_in: 2 weeks
#     paths:
#       - benchmarks

# check the size of the static dogstatsd binary
run_dogstatsd_size_test:
  stage: integration_test
  image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-builders/deb_x64:latest
  tags: [ "runner:main", "size:large" ]
  before_script:
    # Disable global before_script
    - mkdir -p $STATIC_BINARIES_DIR
    - $S3_CP_CMD $S3_ARTEFACTS_URI/static/dogstatsd $STATIC_BINARIES_DIR/dogstatsd
  script:
    - inv -e dogstatsd.size-test --skip-build

#
# package_build



# build Agent package for deb-x64
agent_deb-x64:
  stage: package_build
  image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-builders/deb_x64:latest
  tags: [ "runner:main", "size:large" ]
  variables:
    AWS_CONTAINER_CREDENTIALS_RELATIVE_URI: /credentials
  script:
    # remove artifacts from previous pipelines that may come from the cache
    - rm -rf $OMNIBUS_PACKAGE_DIR/*
    # Artifacts and cache must live within project directory but we run omnibus
    # from the GOPATH (see above). We then call `invoke` passing --base-dir,
    # pointing to a gitlab-friendly location.
    - inv -e agent.omnibus-build --release-version "$RELEASE_VERSION" --base-dir $OMNIBUS_BASE_DIR --omnibus-s3-cache
    - dpkg -c $OMNIBUS_PACKAGE_DIR/datadog-agent*_amd64.deb
    - $S3_CP_CMD $OMNIBUS_PACKAGE_DIR/datadog-agent*_amd64.deb $S3_ARTEFACTS_URI/datadog-agent_amd64.deb
  # TODO: enabling the cache cause builds to be slower and slower on `master`. Re-enable once this is investigated/fixed
  # cache:
  #   # cache per branch
  #   key: $CI_COMMIT_REF_NAME
  #   paths:
  #     - $OMNIBUS_BASE_DIR
  artifacts:
    expire_in: 2 weeks
    paths:
      - $OMNIBUS_PACKAGE_DIR

# build Agent package for rpm-x64
agent_rpm-x64:
  stage: package_build
  image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-builders/rpm_x64:latest
  tags: [ "runner:main", "size:large" ]
  variables:
    AWS_CONTAINER_CREDENTIALS_RELATIVE_URI: /credentials
  script:
    # remove artifacts from previous pipelines that may come from the cache
    - rm -rf $OMNIBUS_PACKAGE_DIR/*
    # Artifacts and cache must live within project directory but we run omnibus
    # from the GOPATH (see above). We then call `invoke` passing --base-dir,
    # pointing to a gitlab-friendly location.
    - set +x
    - RPM_GPG_KEY=$(aws ssm get-parameter --region us-east-1 --name ci.datadog-agent.rpm_signing_private_key --with-decryption --query "Parameter.Value" --out text)
    - printf -- "$RPM_GPG_KEY" | gpg --import --batch
    - export RPM_SIGNING_PASSPHRASE=$(aws ssm get-parameter --region us-east-1 --name ci.datadog-agent.rpm_signing_key_passphrase --with-decryption --query "Parameter.Value" --out text)
    - set -x
    - inv -e agent.omnibus-build --release-version "$RELEASE_VERSION" --base-dir $OMNIBUS_BASE_DIR --omnibus-s3-cache
    - rpm -i $OMNIBUS_PACKAGE_DIR/*.rpm
  # TODO: enabling the cache cause builds to be slower and slower on `master`. Re-enable once this is investigated/fixed
  # cache:
  #   # cache per branch
  #   key: $CI_COMMIT_REF_NAME
  #   paths:
  #     - $OMNIBUS_BASE_DIR
  artifacts:
    expire_in: 2 weeks
    paths:
      - $OMNIBUS_PACKAGE_DIR

# build Agent package for rpm-x64
agent_suse-x64:
  allow_failure: true
  stage: package_build
  image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-builders/suse_x64:latest
  tags: [ "runner:main", "size:large" ]
  variables:
    AWS_CONTAINER_CREDENTIALS_RELATIVE_URI: /credentials
  script:
    # remove artifacts from previous pipelines that may come from the cache
    - rm -rf $OMNIBUS_PACKAGE_DIR/*
    # Artifacts and cache must live within project directory but we run omnibus
    # from the GOPATH (see above). We then call `invoke` passing --base-dir,
    # pointing to a gitlab-friendly location.
    - set +x
    - RPM_GPG_KEY=$(aws ssm get-parameter --region us-east-1 --name ci.datadog-agent.rpm_signing_private_key --with-decryption --query "Parameter.Value" --out text)
    - printf -- "$RPM_GPG_KEY" | gpg --import --batch
    - export RPM_SIGNING_PASSPHRASE=$(aws ssm get-parameter --region us-east-1 --name ci.datadog-agent.rpm_signing_key_passphrase --with-decryption --query "Parameter.Value" --out text)
    - set -x
    - inv -e agent.omnibus-build --release-version "$RELEASE_VERSION" --base-dir $OMNIBUS_BASE_DIR_SUSE --omnibus-s3-cache
    # FIXME: skip the installation step until we fix the preinst/postinst scripts in the rpm package
    # to also work with SUSE11
    # - rpm -i $OMNIBUS_PACKAGE_DIR_SUSE/*.rpm
  # TODO: enabling the cache cause builds to be slower and slower on `master`. Re-enable once this is investigated/fixed
  # cache:
  #   # cache per branch
  #   key: $CI_COMMIT_REF_NAME
  #   paths:
  #     - $OMNIBUS_BASE_DIR_SUSE
  artifacts:
    expire_in: 2 weeks
    paths:
      - $OMNIBUS_PACKAGE_DIR_SUSE

# build Agent package for Windows
build_windows_msi_x64:
  before_script:
    - if exist .omnibus rd /s/q .omnibus
    - if exist \omnibus-ruby rd /s/q \omnibus-ruby
    - if exist \opt\datadog-agent rd /s/q \opt\datadog-agent
    - if exist %GOPATH%\src\github.com\DataDog\datadog-agent rd /s/q %GOPATH%\src\github.com\DataDog\datadog-agent
    - mkdir %GOPATH%\src\github.com\DataDog\datadog-agent
    - xcopy /q/h/e/s * %GOPATH%\src\github.com\DataDog\datadog-agent
    - cd %GOPATH%\src\github.com\DataDog\datadog-agent
    - inv -e deps
  stage: package_build
  variables:
    WINDOWS_BUILDER: 'true'
    CREDENTIALS_FILE_PATH: 'c:\users\gitlab\.aws\config'
  tags: ["runner:windows-agent6"]
  script:
    # remove artifacts from previous pipelines that may come from the cache
    - rm -rf .omnibus/pkg/
    - cd %GOPATH%\src\github.com\DataDog\datadog-agent
    - inv agent.omnibus-build --release-version %RELEASE_VERSION% --omnibus-s3-cache
  artifacts:
    expire_in: 2 weeks
    paths:
      - .omnibus/pkg


# build Agent package for android
agent_android_apk:
  allow_failure: true
  stage: package_build
  image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-builders/android_builder:v538460-2b07be3
  tags: [ "runner:main", "size:large" ]
  variables:
    AWS_CONTAINER_CREDENTIALS_RELATIVE_URI: /credentials
  before_script:
    # We need to install go deps from within the GOPATH, which we set to / on builder images; that's because pointing
    # GOPATH to the project folder would be too complex (we'd need to replicate the `src/github/project` scheme).
    # So we copy the agent sources to / and bootstrap from there the vendor dependencies before running any job.
    - echo running default before_script
    - rsync -azr --delete ./ $SRC_PATH
    - cd $SRC_PATH
    - pip install -U pip
    - inv -e deps --android
  script:
    # remove artifacts from previous pipelines that may come from the cache
    - rm -rf $OMNIBUS_PACKAGE_DIR/*
    # for now do the steps manually.  Should eventually move this to an invoke
    # task
    - inv -e android.build
    - mkdir -p $CI_PROJECT_DIR/apk
    - cp ./bin/agent/ddagent-release-unsigned.apk $CI_PROJECT_DIR/apk
  artifacts:
    expire_in: 2 weeks
    paths:
      - $CI_PROJECT_DIR/apk

# build Dogstastd package for deb-x64
dogstatsd_deb-x64:
  stage: package_build
  image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-builders/deb_x64:latest
  tags: [ "runner:main", "size:large" ]
  variables:
    AWS_CONTAINER_CREDENTIALS_RELATIVE_URI: /credentials
  script:
    # remove artifacts from previous pipelines that may come from the cache
    - rm -rf $OMNIBUS_PACKAGE_DIR/*
    # Artifacts and cache must live within project directory but we run omnibus
    # from the GOPATH (see above). We then call `invoke` passing --base-dir,
    # pointing to a gitlab-friendly location.
    - inv -e dogstatsd.omnibus-build --base-dir $OMNIBUS_BASE_DIR --omnibus-s3-cache
    - dpkg -c $OMNIBUS_PACKAGE_DIR/datadog-dogstatsd*_amd64.deb
    - $S3_CP_CMD $OMNIBUS_PACKAGE_DIR/datadog-dogstatsd*_amd64.deb $S3_ARTEFACTS_URI/datadog-dogstatsd_amd64.deb
  # TODO: enabling the cache cause builds to be slower and slower on `master`. Re-enable once this is investigated/fixed
  # cache:
  #   # cache per branch
  #   key: $CI_COMMIT_REF_NAME
  #   paths:
  #     - $OMNIBUS_BASE_DIR
  artifacts:
    expire_in: 2 weeks
    paths:
      - $OMNIBUS_PACKAGE_DIR

# build Dogstastd package for rpm-x64
dogstatsd_rpm-x64:
  stage: package_build
  image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-builders/rpm_x64:latest
  tags: [ "runner:main", "size:large" ]
  variables:
    AWS_CONTAINER_CREDENTIALS_RELATIVE_URI: /credentials
  script:
    # remove artifacts from previous pipelines that may come from the cache
    - rm -rf $OMNIBUS_PACKAGE_DIR/*
    # Artifacts and cache must live within project directory but we run omnibus
    # from the GOPATH (see above). We then call `invoke` passing --base-dir,
    # pointing to a gitlab-friendly location.
    - set +x
    - RPM_GPG_KEY=$(aws ssm get-parameter --region us-east-1 --name ci.datadog-agent.rpm_signing_private_key --with-decryption --query "Parameter.Value" --out text)
    - printf -- "$RPM_GPG_KEY" | gpg --import --batch
    - export RPM_SIGNING_PASSPHRASE=$(aws ssm get-parameter --region us-east-1 --name ci.datadog-agent.rpm_signing_key_passphrase --with-decryption --query "Parameter.Value" --out text)
    - set -x
    - inv -e dogstatsd.omnibus-build --base-dir $OMNIBUS_BASE_DIR --omnibus-s3-cache
    - rpm -i $OMNIBUS_PACKAGE_DIR/*.rpm
  # TODO: enabling the cache cause builds to be slower and slower on `master`. Re-enable once this is investigated/fixed
  # cache:
  #   # cache per branch
  #   key: $CI_COMMIT_REF_NAME
  #   paths:
  #     - $OMNIBUS_BASE_DIR
  artifacts:
    expire_in: 2 weeks
    paths:
      - $OMNIBUS_PACKAGE_DIR


# build Dogstastd package for rpm-x64
dogstatsd_suse-x64:
  allow_failure: true
  stage: package_build
  image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-builders/suse_x64:latest
  tags: [ "runner:main", "size:large" ]
  variables:
    AWS_CONTAINER_CREDENTIALS_RELATIVE_URI: /credentials
  script:
    # remove artifacts from previous pipelines that may come from the cache
    - rm -rf $OMNIBUS_PACKAGE_DIR/*
    # Artifacts and cache must live within project directory but we run omnibus
    # from the GOPATH (see above). We then call `invoke` passing --base-dir,
    # pointing to a gitlab-friendly location.
    - set +x
    - RPM_GPG_KEY=$(aws ssm get-parameter --region us-east-1 --name ci.datadog-agent.rpm_signing_private_key --with-decryption --query "Parameter.Value" --out text)
    - printf -- "$RPM_GPG_KEY" | gpg --import --batch
    - export RPM_SIGNING_PASSPHRASE=$(aws ssm get-parameter --region us-east-1 --name ci.datadog-agent.rpm_signing_key_passphrase --with-decryption --query "Parameter.Value" --out text)
    - set -x
    - inv -e dogstatsd.omnibus-build --base-dir $OMNIBUS_BASE_DIR_SUSE --omnibus-s3-cache
    - rpm -i $OMNIBUS_PACKAGE_DIR_SUSE/*.rpm
  # TODO: enabling the cache cause builds to be slower and slower on `master`. Re-enable once this is investigated/fixed
  # cache:
  #   # cache per branch
  #   key: $CI_COMMIT_REF_NAME
  #   paths:
  #     - $OMNIBUS_BASE_DIR_SUSE
  artifacts:
    expire_in: 2 weeks
    paths:
      - $OMNIBUS_PACKAGE_DIR_SUSE

# deploy debian packages to apt staging repo
deploy_deb_testing:
  stage: testkitchen_deploy
  image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-builders/deploy:latest
  before_script:
    - ls $OMNIBUS_PACKAGE_DIR
  <<: *run_when_triggered
  tags: [ "runner:main", "size:large" ]
  script:
    - source /usr/local/rvm/scripts/rvm
    - rvm use 2.4

    - set +x # make sure we don't output the creds to the build log

    - APT_SIGNING_KEY_ID=$(aws ssm get-parameter --region us-east-1 --name ci.datadog-agent.apt_signing_key_id --with-decryption --query "Parameter.Value" --out text)
    - APT_SIGNING_PRIVATE_KEY_PART1=$(aws ssm get-parameter --region us-east-1 --name ci.datadog-agent.apt_signing_private_key_part1 --with-decryption --query "Parameter.Value" --out text)
    - APT_SIGNING_PRIVATE_KEY_PART2=$(aws ssm get-parameter --region us-east-1 --name ci.datadog-agent.apt_signing_private_key_part2 --with-decryption --query "Parameter.Value" --out text)
    - APT_SIGNING_KEY_PASSPHRASE=$(aws ssm get-parameter --region us-east-1 --name ci.datadog-agent.apt_signing_key_passphrase --with-decryption --query "Parameter.Value" --out text)

    - echo "$APT_SIGNING_KEY_ID"
    - printf -- "$APT_SIGNING_PRIVATE_KEY_PART1\n$APT_SIGNING_PRIVATE_KEY_PART2\n" | gpg --import --batch

    - echo "$APT_SIGNING_KEY_PASSPHRASE" | deb-s3 upload -c "pipeline-$CI_PIPELINE_ID" -b $DEB_TESTING_S3_BUCKET -a amd64 --sign=$APT_SIGNING_KEY_ID --gpg_options="--passphrase-fd 0 --pinentry-mode loopback --batch --digest-algo SHA512" --preserve_versions --visibility public $OMNIBUS_PACKAGE_DIR/*amd64.deb
    - echo "$APT_SIGNING_KEY_PASSPHRASE" | deb-s3 upload -c "pipeline-$CI_PIPELINE_ID" -b $DEB_TESTING_S3_BUCKET -a x86_64 --sign=$APT_SIGNING_KEY_ID --gpg_options="--passphrase-fd 0 --pinentry-mode loopback --batch --digest-algo SHA512" --preserve_versions --visibility public $OMNIBUS_PACKAGE_DIR/*amd64.deb


# deploy rpm packages to yum staging repo
deploy_rpm_testing:
  <<: *run_when_triggered
  stage: testkitchen_deploy
  image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-builders/deploy:latest
  before_script:
    - ls $OMNIBUS_PACKAGE_DIR
  tags: [ "runner:main", "size:large" ]
  script:
    - source /usr/local/rvm/scripts/rvm
    - rvm use 2.4
    - mkdir -p ./rpmrepo/x86_64/
    - aws s3 sync s3://$RPM_TESTING_S3_BUCKET/pipeline-$CI_PIPELINE_ID ./rpmrepo/
    - cp $OMNIBUS_PACKAGE_DIR/*x86_64.rpm ./rpmrepo/x86_64/
    - createrepo --update -v --checksum sha ./rpmrepo/x86_64
    - aws s3 sync ./rpmrepo/ s3://$RPM_TESTING_S3_BUCKET/pipeline-$CI_PIPELINE_ID --grants read=uri=http://acs.amazonaws.com/groups/global/AllUsers full=id=3a6e02b08553fd157ae3fb918945dd1eaae5a1aa818940381ef07a430cf25732

# deploy rpm packages to yum staging repo
deploy_suse_rpm_testing:
  <<: *run_when_triggered
  allow_failure: true
  stage: testkitchen_deploy
  image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-builders/deploy:latest
  before_script:
    - ls $OMNIBUS_PACKAGE_DIR_SUSE
  tags: [ "runner:main", "size:large" ]
  script:
    - source /usr/local/rvm/scripts/rvm
    - rvm use 2.4
    - mkdir -p ./rpmrepo/suse/x86_64/
    - aws s3 sync s3://$RPM_TESTING_S3_BUCKET/suse/pipeline-$CI_PIPELINE_ID ./rpmrepo/
    - cp $OMNIBUS_PACKAGE_DIR_SUSE/*x86_64.rpm ./rpmrepo/suse/x86_64/
    - createrepo --update -v --checksum sha ./rpmrepo/suse/x86_64
    - aws s3 sync ./rpmrepo/suse/ s3://$RPM_TESTING_S3_BUCKET/suse/pipeline-$CI_PIPELINE_ID --grants read=uri=http://acs.amazonaws.com/groups/global/AllUsers full=id=3a6e02b08553fd157ae3fb918945dd1eaae5a1aa818940381ef07a430cf25732

# deploy windows packages to our testing bucket
deploy_windows_testing:
  <<: *run_when_triggered
  allow_failure: true
  stage: testkitchen_deploy
  image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-builders/deploy:latest
  before_script:
    - ls $OMNIBUS_PACKAGE_DIR_SUSE
  tags: [ "runner:main", "size:large" ]
  script:
    - $S3_CP_CMD --recursive --exclude "*" --include "*.msi" $OMNIBUS_PACKAGE_DIR s3://$WINDOWS_TESTING_S3_BUCKET/ --grants read=uri=http://acs.amazonaws.com/groups/global/AllUsers full=id=3a6e02b08553fd157ae3fb918945dd1eaae5a1aa818940381ef07a430cf25732

# run dd-agent-testing on windows
kitchen_windows:
  stage: testkitchen_testing
  allow_failure: true
  image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-builders/dd-agent-testing:latest
  only:
    - master
    - tags
    - triggers
  before_script:
    - rsync -azr --delete ./ $SRC_PATH
  tags: [ "runner:main", "size:large" ]
  script:
    - cd $CI_PROJECT_DIR
    - cd $DD_AGENT_TESTING_DIR
    - mkdir $CI_PROJECT_DIR/kitchen_logs
    - ln -s $CI_PROJECT_DIR/kitchen_logs $DD_AGENT_TESTING_DIR/.kitchen
    - export TEST_PLATFORMS="win2012,MicrosoftWindowsServer:WindowsServer:2012-Datacenter:3.127.20171115"
    - export TEST_PLATFORMS="$TEST_PLATFORMS|win2012r2,MicrosoftWindowsServer:WindowsServer:2012-R2-Datacenter:4.127.20171115"
    - export TEST_PLATFORMS="$TEST_PLATFORMS|win2016,MicrosoftWindowsServer:WindowsServer:2016-Datacenter:2016.127.20171116"
    - bash -l tasks/run-test-kitchen.sh
  artifacts:
    expire_in: 2 weeks
    when: always
    paths:
      - $CI_PROJECT_DIR/kitchen_logs

# run dd-agent-testing on centos
kitchen_centos:
  stage: testkitchen_testing
  allow_failure: true
  image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-builders/dd-agent-testing:latest
  only:
    - master
    - tags
    - triggers
  before_script:
    - rsync -azr --delete ./ $SRC_PATH
  tags: [ "runner:main", "size:large" ]
  script:
    - cd $CI_PROJECT_DIR
    - cd $DD_AGENT_TESTING_DIR
    - mkdir $CI_PROJECT_DIR/kitchen_logs
    - ln -s $CI_PROJECT_DIR/kitchen_logs $DD_AGENT_TESTING_DIR/.kitchen
    - export TEST_PLATFORMS="centos-74,OpenLogic:CentOS:7.4:7.4.20171110"
    - bash -l tasks/run-test-kitchen.sh
  artifacts:
    expire_in: 2 weeks
    when: always
    paths:
      - $CI_PROJECT_DIR/kitchen_logs

# run dd-agent-testing on ubuntu (FIXME: Allowed to fail until we resolve the issue with apt locks/azure agent)
kitchen_ubuntu:
  stage: testkitchen_testing
  allow_failure: true
  image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-builders/dd-agent-testing:latest
  only:
    - master
    - tags
    - triggers
  before_script:
    - rsync -azr --delete ./ $SRC_PATH
  tags: [ "runner:main", "size:large" ]
  script:
    - cd $CI_PROJECT_DIR
    - cd $DD_AGENT_TESTING_DIR
    - mkdir $CI_PROJECT_DIR/kitchen_logs
    - ln -s $CI_PROJECT_DIR/kitchen_logs $DD_AGENT_TESTING_DIR/.kitchen
    - export TEST_PLATFORMS="ubuntu-14-04,Canonical:UbuntuServer:14.04.5-LTS:14.04.201803080"
    - export TEST_PLATFORMS="$TEST_PLATFORMS|ubuntu-16-04,Canonical:UbuntuServer:16.04.0-LTS:16.04.201802220"
    - bash -l tasks/run-test-kitchen.sh
  artifacts:
    expire_in: 2 weeks
    when: always
    paths:
      - $CI_PROJECT_DIR/kitchen_logs

# run dd-agent-testing on suse
kitchen_suse:
  stage: testkitchen_testing
  allow_failure: true
  image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-builders/dd-agent-testing:latest
  only:
    - master
    - tags
    - triggers
  before_script:
    - rsync -azr --delete ./ $SRC_PATH
  tags: [ "runner:main", "size:large" ]
  script:
    - cd $CI_PROJECT_DIR
    - cd $DD_AGENT_TESTING_DIR
    - mkdir $CI_PROJECT_DIR/kitchen_logs
    - ln -s $CI_PROJECT_DIR/kitchen_logs $DD_AGENT_TESTING_DIR/.kitchen
    - export TEST_PLATFORMS="sles-12,SUSE:SLES:12-SP3:2017.12.11"
    - bash -l tasks/run-test-kitchen.sh
  artifacts:
    expire_in: 2 weeks
    when: always
    paths:
      - $CI_PROJECT_DIR/kitchen_logs

# run dd-agent-testing on debian (FIXME: Allowed to fail until we resolve the issue with apt locks/azure agent)
kitchen_debian:
  stage: testkitchen_testing
  allow_failure: true
  image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-builders/dd-agent-testing:latest
  only:
    - master
    - tags
    - triggers
  before_script:
    - rsync -azr --delete ./ $SRC_PATH
  tags: [ "runner:main", "size:large" ]
  script:
    - cd $CI_PROJECT_DIR
    - cd $DD_AGENT_TESTING_DIR
    - mkdir $CI_PROJECT_DIR/kitchen_logs
    - ln -s $CI_PROJECT_DIR/kitchen_logs $DD_AGENT_TESTING_DIR/.kitchen
    - export TEST_PLATFORMS="debian-8,credativ:Debian:8:8.0.201803130"
    - bash -l tasks/run-test-kitchen.sh
  artifacts:
    expire_in: 2 weeks
    when: always
    paths:
      - $CI_PROJECT_DIR/kitchen_logs

# run dd-agent-testing
testkitchen_cleanup_s3:
  stage: testkitchen_cleanup
  image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-builders/deploy:latest
  only:
    - master
    - tags
    - triggers
  tags: [ "runner:main", "size:large" ]
  before_script:
    - ls
  # even if this fails, it shouldn't block the pipeline.
  allow_failure: true
  when: always
  script:
    - aws s3 rm s3://$DEB_TESTING_S3_BUCKET/dists/pipeline-$CI_PIPELINE_ID --recursive
    - aws s3 rm s3://$RPM_TESTING_S3_BUCKET/pipeline-$CI_PIPELINE_ID --recursive
    - aws s3 rm s3://$RPM_TESTING_S3_BUCKET/suse/pipeline-$CI_PIPELINE_ID --recursive
    - aws s3 rm s3://$WINDOWS_TESTING_S3_BUCKET --recursive
    - cd $OMNIBUS_PACKAGE_DIR
    - for deb in $(ls *amd64.deb); do aws s3 rm s3://$DEB_TESTING_S3_BUCKET/pool/d/da/$deb --recursive; done

# run dd-agent-testing
testkitchen_cleanup_azure:
  stage: testkitchen_cleanup
  image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-builders/dd-agent-testing:latest
  only:
    - master
    - tags
    - triggers
  # even if this fails, it shouldn't block the pipeline.
  allow_failure: true
  when: always
  tags: [ "runner:main", "size:large" ]
  before_script:
    - rsync -azr --delete ./ $SRC_PATH
  script:
    - cd $DD_AGENT_TESTING_DIR
    - bash -l tasks/clean.sh

#
# image_build
#

.docker_build_job_definition: &docker_build_job_definition
  stage: image_build
  tags: [ "runner:main", "size:large" ]
  image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/docker:18.03.1
  services:
    - 486234852809.dkr.ecr.us-east-1.amazonaws.com/remote-docker:latest
  before_script: [ "# noop" ] # Override top level entry
  dependencies: [] # Don't download Gitlab artefacts
  script:
    - apt-get update && apt-get install -y python-pip && pip install awscli
    - aws s3 sync --only-show-errors "s3://dd-ci-artefacts-build-stable/datadog-agent/$CI_PIPELINE_ID" $BUILD_CONTEXT
    - TAG_SUFFIX=${TAG_SUFFIX:-}
    - BUILD_ARG=${BUILD_ARG:-}
    - docker build $BUILD_ARG --pull --tag $IMAGE:v$CI_PIPELINE_ID-${CI_COMMIT_SHA:0:7}$TAG_SUFFIX $BUILD_CONTEXT
    - docker push $IMAGE:v$CI_PIPELINE_ID-${CI_COMMIT_SHA:0:7}$TAG_SUFFIX

# build the agent6 image
build_agent6:
  <<: *docker_build_job_definition
  variables:
    IMAGE: &agent_ecr 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent/agent
    BUILD_CONTEXT: Dockerfiles/agent

# build the agent6 jmx image
build_agent6_jmx:
  <<: *docker_build_job_definition
  variables:
    IMAGE: &agent_ecr 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent/agent
    BUILD_CONTEXT: Dockerfiles/agent
    TAG_SUFFIX: -jmx
    BUILD_ARG: --build-arg WITH_JMX=true

# build the cluster-agent image
build_cluster_agent:
  <<: *docker_build_job_definition
  variables:
    IMAGE: &cluster-agent_ecr 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent/cluster-agent
    BUILD_CONTEXT: Dockerfiles/cluster-agent

# build the dogstatsd image
build_dogstatsd:
  <<: *docker_build_job_definition
  variables:
    IMAGE: &dogstatsd_ecr 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent/dogstatsd
    BUILD_CONTEXT: Dockerfiles/dogstatsd/alpine

#
# Docker dev image deployments
#

.docker_tag_job_definition: &docker_tag_job_definition
  stage: image_deploy
  tags: [ "runner:main", "size:large" ]
  image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/docker:18.03.1
  services:
    - 486234852809.dkr.ecr.us-east-1.amazonaws.com/remote-docker:latest
  before_script:
    - export SRC_TAG=v$CI_PIPELINE_ID-${CI_COMMIT_SHA:0:7}
    - apt-get update && apt-get install -y python-pip && pip install awscli
    - DOCKER_REGISTRY_LOGIN=$(aws ssm get-parameter --region us-east-1 --name ci.datadog-agent.$DOCKER_REGISTRY_LOGIN_SSM_KEY --with-decryption --query "Parameter.Value" --out text)
    - aws ssm get-parameter --region us-east-1 --name ci.datadog-agent.$DOCKER_REGISTRY_PWD_SSM_KEY --with-decryption --query "Parameter.Value" --out text | docker login --username "$DOCKER_REGISTRY_LOGIN" --password-stdin "$DOCKER_REGISTRY_URL"
    - pip install invoke
  dependencies: [] # Don't download Gitlab artefacts

.docker_hub_variables: &docker_hub_variables
  DOCKER_REGISTRY_LOGIN_SSM_KEY: docker_hub_login
  DOCKER_REGISTRY_PWD_SSM_KEY: docker_hub_pwd
  DOCKER_REGISTRY_URL: docker.io
  SRC_AGENT: *agent_ecr
  SRC_DSD: *dogstatsd_ecr
  SRC_DCA: *cluster-agent_ecr

.quay_variables: &quay_variables
  <<: *docker_hub_variables
  DOCKER_REGISTRY_LOGIN_SSM_KEY: quay_login
  DOCKER_REGISTRY_PWD_SSM_KEY: quay_pwd
  DOCKER_REGISTRY_URL: quay.io

dev_branch_docker_hub:
  <<: *docker_tag_job_definition
  when: manual
  except:
    - master
  variables:
    <<: *docker_hub_variables
  script:
    - inv -e docker.publish ${SRC_AGENT}:${SRC_TAG} datadog/agent-dev:${CI_COMMIT_REF_SLUG}
    - inv -e docker.publish ${SRC_AGENT}:${SRC_TAG}-jmx datadog/agent-dev:${CI_COMMIT_REF_SLUG}-jmx
    - inv -e docker.publish ${SRC_DSD}:${SRC_TAG} datadog/dogstatsd-dev:${CI_COMMIT_REF_SLUG}

dev_master_docker_hub:
  <<: *docker_tag_job_definition
  only:
    - master
  variables:
    <<: *docker_hub_variables
  script:
    - inv -e docker.publish ${SRC_AGENT}:${SRC_TAG} datadog/agent-dev:master
    - inv -e docker.publish ${SRC_AGENT}:${SRC_TAG}-jmx datadog/agent-dev:master-jmx
    - inv -e docker.publish ${SRC_DSD}:${SRC_TAG} datadog/dogstatsd-dev:master

dca_dev_branch_docker_hub:
  <<: *docker_tag_job_definition
  when: manual
  except:
    - master
  variables:
    <<: *docker_hub_variables
  script:
    - inv -e docker.publish ${SRC_DCA}:${SRC_TAG} datadog/cluster-agent-dev:${CI_COMMIT_REF_SLUG}

dca_dev_master_docker_hub:
  <<: *docker_tag_job_definition
  <<: *run_when_triggered_on_nightly
  variables:
    <<: *docker_hub_variables
  script:
    - inv -e docker.publish ${SRC_DCA}:${SRC_TAG} datadog/cluster-agent-dev:master

# deploys to quay
# docker images are mirrored on quay.io
# to run security scans on them
dev_master_quay:
  <<: *docker_tag_job_definition
  <<: *run_when_triggered_on_nightly
  variables:
    <<: *quay_variables
  script:
    - inv -e docker.publish ${SRC_AGENT}:${SRC_TAG} quay.io/datawhale/agent-dev:master
    - inv -e docker.publish ${SRC_AGENT}:${SRC_TAG}-jmx quay.io/datawhale/agent-dev:master-jmx
    - inv -e docker.publish ${SRC_DSD}:${SRC_TAG} quay.io/datawhale/dogstatsd-dev:master

#
# Check Deploy
#

# Check that the current version hasn't already been deployed (we don't want to
# overwrite a public package). To update a erroneous package, first remove it
# from our S3 bucket.
check_already_deployed_version:
  <<: *run_when_triggered
  stage: check_deploy
  image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-builders/deploy:latest
  before_script:
    - ls $OMNIBUS_PACKAGE_DIR
  tags: [ "runner:main", "size:large" ]
  script:
    - cd $OMNIBUS_PACKAGE_DIR && /deploy_scripts/fail_deb_is_pkg_already_exists.sh

#
# deploy
#

# deploy debian packages to apt staging repo
deploy_deb:
  <<: *run_when_triggered
  stage: deploy
  image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-builders/deploy:latest
  before_script:
    - ls $OMNIBUS_PACKAGE_DIR
  tags: [ "runner:main", "size:large" ]
  script:
    - source /usr/local/rvm/scripts/rvm
    - rvm use 2.4

    - set +x # make sure we don't output the creds to the build log

    - APT_SIGNING_KEY_ID=$(aws ssm get-parameter --region us-east-1 --name ci.datadog-agent.apt_signing_key_id --with-decryption --query "Parameter.Value" --out text)
    - APT_SIGNING_PRIVATE_KEY_PART1=$(aws ssm get-parameter --region us-east-1 --name ci.datadog-agent.apt_signing_private_key_part1 --with-decryption --query "Parameter.Value" --out text)
    - APT_SIGNING_PRIVATE_KEY_PART2=$(aws ssm get-parameter --region us-east-1 --name ci.datadog-agent.apt_signing_private_key_part2 --with-decryption --query "Parameter.Value" --out text)
    - APT_SIGNING_KEY_PASSPHRASE=$(aws ssm get-parameter --region us-east-1 --name ci.datadog-agent.apt_signing_key_passphrase --with-decryption --query "Parameter.Value" --out text)

    - echo "$APT_SIGNING_KEY_ID"
    - printf -- "$APT_SIGNING_PRIVATE_KEY_PART1\n$APT_SIGNING_PRIVATE_KEY_PART2\n" | gpg --import --batch

    # Release the artifacts to the "6" component
    - echo "$APT_SIGNING_KEY_PASSPHRASE" | deb-s3 upload -c $DEB_RPM_BUCKET_BRANCH -m 6 -b $DEB_S3_BUCKET -a amd64 --sign=$APT_SIGNING_KEY_ID --gpg_options="--passphrase-fd 0 --pinentry-mode loopback --batch --digest-algo SHA512" --preserve_versions --visibility public $OMNIBUS_PACKAGE_DIR/*amd64.deb
    - echo "$APT_SIGNING_KEY_PASSPHRASE" | deb-s3 upload -c $DEB_RPM_BUCKET_BRANCH -m 6 -b $DEB_S3_BUCKET -a x86_64 --sign=$APT_SIGNING_KEY_ID --gpg_options="--passphrase-fd 0 --pinentry-mode loopback --batch --digest-algo SHA512" --preserve_versions --visibility public $OMNIBUS_PACKAGE_DIR/*amd64.deb

# deploy windows packages to a public s3 bucket when pushed on master
deploy_windows_master:
  stage: deploy
  image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-builders/deploy:latest
  before_script:
    - ls $OMNIBUS_PACKAGE_DIR
  <<: *run_when_triggered_on_nightly
  tags: [ "runner:main", "size:large" ]
  script:
    - $S3_CP_CMD --recursive --exclude "*" --include "*.msi" $OMNIBUS_PACKAGE_DIR s3://$WINDOWS_BUILDS_S3_BUCKET/master/ --grants read=uri=http://acs.amazonaws.com/groups/global/AllUsers full=id=3a6e02b08553fd157ae3fb918945dd1eaae5a1aa818940381ef07a430cf25732
    - $S3_CP_CMD $OMNIBUS_PACKAGE_DIR/datadog-agent-*-x86_64.msi "s3://$WINDOWS_BUILDS_S3_BUCKET/master/datadog-agent-6-latest.amd64.msi" --grants read=uri=http://acs.amazonaws.com/groups/global/AllUsers full=id=3a6e02b08553fd157ae3fb918945dd1eaae5a1aa818940381ef07a430cf25732

# deploy windows packages to a public s3 bucket when tagged
deploy_windows_tags:
  stage: deploy
  image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-builders/deploy:latest
  before_script:
    - ls $OMNIBUS_PACKAGE_DIR
  <<: *run_when_triggered_on_tag
  tags: [ "runner:main", "size:large" ]
  script:
    - $S3_CP_CMD --recursive --exclude "*" --include "*.msi" $OMNIBUS_PACKAGE_DIR s3://$WINDOWS_BUILDS_S3_BUCKET/tagged/ --grants read=uri=http://acs.amazonaws.com/groups/global/AllUsers full=id=3a6e02b08553fd157ae3fb918945dd1eaae5a1aa818940381ef07a430cf25732
    # By default we update the "latest" artifacts on our s3 bucket so the
    # staging box can pick it up. Allow the job to skip this step if needed
    # (when building a custom beta for example).
    - if [ "WINDOWS_DO_NOT_UPDATE_LATEST" != "true" ]; then $S3_CP_CMD $OMNIBUS_PACKAGE_DIR/datadog-agent-*-x86_64.msi s3://$WINDOWS_BUILDS_S3_BUCKET/tagged/datadog-agent-6-latest.amd64.msi --grants read=uri=http://acs.amazonaws.com/groups/global/AllUsers full=id=3a6e02b08553fd157ae3fb918945dd1eaae5a1aa818940381ef07a430cf25732; fi

# deploy rpm packages to yum staging repo
deploy_rpm:
  <<: *run_when_triggered
  stage: deploy
  image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-builders/deploy:latest
  before_script:
    - ls $OMNIBUS_PACKAGE_DIR
  tags: [ "runner:main", "size:large" ]
  script:
    - source /usr/local/rvm/scripts/rvm
    - rvm use 2.4
    - mkdir -p ./rpmrepo/6/x86_64/
    - aws s3 sync s3://$RPM_S3_BUCKET/$DEB_RPM_BUCKET_BRANCH/6/ ./rpmrepo/6/

    # add RPMs to new "6" branch
    - cp $OMNIBUS_PACKAGE_DIR/*x86_64.rpm ./rpmrepo/6/x86_64/
    - createrepo --update -v --checksum sha ./rpmrepo/6/x86_64

    # sync to S3
    - aws s3 sync ./rpmrepo/6/ s3://$RPM_S3_BUCKET/$DEB_RPM_BUCKET_BRANCH/6/ --grants read=uri=http://acs.amazonaws.com/groups/global/AllUsers full=id=3a6e02b08553fd157ae3fb918945dd1eaae5a1aa818940381ef07a430cf25732

# deploy suse rpm packages to yum staging repo
deploy_suse_rpm:
  <<: *run_when_triggered
  allow_failure: true
  stage: deploy
  image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-builders/deploy:latest
  before_script:
    - ls $OMNIBUS_PACKAGE_DIR_SUSE
  tags: [ "runner:main", "size:large" ]
  script:
    - source /usr/local/rvm/scripts/rvm
    - rvm use 2.4
    - mkdir -p ./rpmrepo/6/x86_64/
    - aws s3 sync s3://$RPM_S3_BUCKET/suse/$DEB_RPM_BUCKET_BRANCH/6/ ./rpmrepo/6/

    # add RPMs to new "6" branch
    - cp $OMNIBUS_PACKAGE_DIR_SUSE/*x86_64.rpm ./rpmrepo/6/x86_64/
    - createrepo --update -v --checksum sha ./rpmrepo/6/x86_64

    # sync to S3
    - aws s3 sync ./rpmrepo/6/ s3://$RPM_S3_BUCKET/suse/$DEB_RPM_BUCKET_BRANCH/6/ --grants read=uri=http://acs.amazonaws.com/groups/global/AllUsers full=id=3a6e02b08553fd157ae3fb918945dd1eaae5a1aa818940381ef07a430cf25732

# deploy dsd binary to staging bucket
deploy_dsd:
  <<: *run_when_triggered
  stage: deploy
  image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-builders/deploy:latest
  before_script:
    - ls $OMNIBUS_PACKAGE_DIR
  tags: [ "runner:main", "size:large" ]
  script:
    - $S3_CP_CMD $S3_ARTEFACTS_URI/dogstatsd/dogstatsd ./dogstatsd
    - export VERSION=$(inv version --url-safe)
    - aws s3 cp --region us-east-1 ./dogstatsd $S3_DSD6_URI/dogstatsd-$VERSION --grants read=uri=http://acs.amazonaws.com/groups/global/AllUsers full=id=3a6e02b08553fd157ae3fb918945dd1eaae5a1aa818940381ef07a430cf25732

# deploy dsd binary to staging bucket
deploy_puppy:
  <<: *run_when_triggered
  stage: deploy
  image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-builders/deploy:latest
  before_script:
    - ls $OMNIBUS_PACKAGE_DIR
  tags: [ "runner:main", "size:large" ]
  script:
    - $S3_CP_CMD $S3_ARTEFACTS_URI/puppy/agent ./agent
    - export VERSION=$(inv version --url-safe)
    - aws s3 cp --region us-east-1 ./agent $S3_DSD6_URI/puppy/agent-$VERSION --grants read=uri=http://acs.amazonaws.com/groups/global/AllUsers full=id=3a6e02b08553fd157ae3fb918945dd1eaae5a1aa818940381ef07a430cf25732

#
# Docker releases
#

tag_release:
  <<: *docker_tag_job_definition
  <<: *run_when_triggered_on_tag
  stage: deploy
  when: manual
  variables:
    <<: *docker_hub_variables
  script:
    - inv -e docker.publish ${SRC_AGENT}:${SRC_TAG} datadog/agent:${CI_COMMIT_TAG}
    - inv -e docker.publish ${SRC_AGENT}:${SRC_TAG}-jmx datadog/agent:${CI_COMMIT_TAG}-jmx
    - inv -e docker.publish ${SRC_DSD}:${SRC_TAG} datadog/dogstatsd:${CI_COMMIT_TAG}

latest_release:
  <<: *docker_tag_job_definition
  <<: *run_when_triggered_on_tag
  stage: deploy
  when: manual
  variables:
    <<: *docker_hub_variables
  script:
    - inv -e docker.publish ${SRC_AGENT}:${SRC_TAG} datadog/agent:latest
    - inv -e docker.publish ${SRC_AGENT}:${SRC_TAG}-jmx datadog/agent:latest-jmx
    - inv -e docker.publish ${SRC_DSD}:${SRC_TAG} datadog/dogstatsd:latest

dca_tag_release:
  <<: *docker_tag_job_definition
  <<: *run_when_triggered_on_tag
  stage: deploy
  when: manual
  variables:
    <<: *docker_hub_variables
  script:
    - inv -e docker.publish ${SRC_DCA}:${SRC_TAG} datadog/cluster-agent-dev:${CI_COMMIT_TAG}


dca_latest_release:
  <<: *docker_tag_job_definition
  <<: *run_when_triggered_on_tag
  stage: deploy
  when: manual
  variables:
    <<: *docker_hub_variables
  script:
    - inv -e docker.publish ${SRC_DCA}:${SRC_TAG} datadog/cluster-agent-dev:latest

# invalidate cloudfront cache
deploy_cloudfront_invalidate:
  <<: *run_when_triggered
  stage: deploy_invalidate
  image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-builders/deploy:latest
  before_script:
    - ls $OMNIBUS_PACKAGE_DIR
  tags: [ "runner:main", "size:large" ]
  script:
    - cd /deploy_scripts/cloudfront-invalidation
    - "REPO=apt PATTERN_SUBSTRING=/$DEB_RPM_BUCKET_BRANCH/ ./invalidate.sh"
    - "REPO=yum PATTERN_SUBSTRING=/$DEB_RPM_BUCKET_BRANCH/ ./invalidate.sh"

#
# end to end
#

.pupernetes_template: &pupernetes_template
  stage: e2e
  image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-builders/deploy:latest
  tags: [ "runner:main", "size:large" ]
  before_script: [ "# noop" ] # Override top level entry
  script:
  - inv -e e2e-tests --image=datadog/agent-dev:$CI_COMMIT_REF_SLUG

pupernetes-dev:
  <<: *pupernetes_template
  when: manual
  except:
    - master
    - tags

pupernetes-master:
  <<: *pupernetes_template
  only:
    - master
  script:
  - inv -e e2e-tests --image=datadog/agent-dev:master

pupernetes-tags:
  <<: *pupernetes_template
  <<: *run_when_triggered_on_tag
  script:
  # note: it's not the agent-dev
  - inv -e e2e-tests --image=datadog/agent:$CI_COMMIT_TAG