-
Notifications
You must be signed in to change notification settings - Fork 132
/
.gitlab-ci.yml
337 lines (304 loc) · 12 KB
/
.gitlab-ci.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
stages:
- build
- package
- single-step-testing
- publish
- deploy
- benchmarks
include:
- remote: https://gitlab-templates.ddbuild.io/apm/packaging.yml
variables:
GIT_PROFILER_REF: master
DOTNET_PACKAGE_VERSION:
description: "Used by the package stage when triggered manually"
build:
only:
- master
- main
- /^hotfix.*$/
- /^release.*$/
except:
variables:
- $DEPLOY_TO_REL_ENV == "true"
- $CI_COMMIT_TAG # We don't need to build/publish when building a release tag
stage: build
tags: ["runner:windows-docker", "windowsversion:1809"]
script:
- if (Test-Path build-out) { remove-item -recurse -force build-out }
- if (Test-Path artifacts) { remove-item -recurse -force artifacts }
- docker run --rm -m 8192M -v "$(Get-Location):c:\mnt" -e CI_JOB_ID=${CI_JOB_ID} -e ENABLE_MULTIPROCESSOR_COMPILATION=false -e WINDOWS_BUILDER=true -e AWS_NETWORKING=true -e SIGN_WINDOWS=true -e NUGET_CERT_REVOCATION_MODE=offline registry.ddbuild.io/images/mirror/datadog/dd-trace-dotnet-docker-build:latest
- mkdir artifacts
- xcopy /e/s build-out\${CI_JOB_ID}\*.* artifacts
- remove-item -recurse -force build-out\${CI_JOB_ID}
- get-childitem build-out
- get-childitem artifacts
artifacts:
expire_in: 2 weeks
paths:
- artifacts
wait-for-single-step-artifacts:
stage: single-step-testing
image: registry.ddbuild.io/images/ci_docker_base
tags: [ "runner:docker" ]
rules:
# We don't run on commit pushes, because there's no associated build
- if: '$CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+$/'
when: never
- when: delayed
start_in: 20 minutes
script: |
#Create a directory to store the files
target_dir=artifacts
mkdir -p $target_dir
if [ -z "${CI_COMMIT_BRANCH}" ]; then
echo "Not on a branch, skipping test"
exit 0
fi
branchName="refs/heads/$CI_COMMIT_BRANCH"
artifactName="ssi-artifacts"
echo "Waiting for the azure devops build for branch '$branchName' for commit '$CI_COMMIT_SHA' to start"
allBuildsForBranchUrl="https://dev.azure.com/datadoghq/dd-trace-dotnet/_apis/build/builds?api-version=7.1&definitions=54&\$top=10&queryOrder=queueTimeDescending&branchName=$branchName&reasonFilter=manual,individualCI"
# We should _definitely_ have the build by now, so if not, there probably won't be one
buildId=$(curl -sS $allBuildsForBranchUrl | jq --arg version $CI_COMMIT_SHA '.value[] | select(.sourceVersion == $version and .reason != "schedule") | .id' | head -n 1)
if [ -z "${buildId}" ]; then
echo "No build found for commit '$CI_COMMIT_SHA' on branch '$branchName'. Checking for PR builds..."
allBuildsForPrUrl="https://dev.azure.com/datadoghq/dd-trace-dotnet/_apis/build/builds?api-version=7.1&definitions=54&\$top=10&queryOrder=queueTimeDescending&reasonFilter=pullRequest"
buildId=$(curl -sS $allBuildsForPrUrl | jq --arg version $CI_COMMIT_SHA --arg branch $CI_COMMIT_BRANCH '.value[] | select(.triggerInfo["pr.sourceBranch"] == $branch and .triggerInfo["pr.sourceSha"] == $version) | .id' | head -n 1)
fi
if [ -z "${buildId}" ]; then
echo "No build found for commit '$CI_COMMIT_SHA' on branch '$branchName' (including PRs)"
exit 1
fi
echo "Found build with id '$buildId' for commit '$CI_COMMIT_SHA' on branch '$branchName'"
# Now try to download the ssi artifacts from the build
artifactsUrl="https://dev.azure.com/datadoghq/dd-trace-dotnet/_apis/build/builds/$buildId/artifacts?api-version=7.1&artifactName=$artifactName"
# Keep trying to get the artifact for 30 minutes
TIMEOUT=1800
STARTED=0
until (( STARTED == TIMEOUT )) || [ ! -z "${downloadUrl}" ] ; do
echo "Checking for artifacts at '$artifactsUrl'..."
# If the artifact doesn't exist, .resource.downloadUrl will be null, so we filter that out
downloadUrl=$(curl -s $artifactsUrl | jq -r '.resource.downloadUrl | select( . != null )')
sleep 100
(( STARTED += 100 ))
done
(( STARTED < TIMEOUT ))
if [ -z "${downloadUrl}" ]; then
echo "No downloadUrl found after 30 minutes for commit '$CI_COMMIT_SHA' on branch '$branchName'"
exit 1
fi
echo "Downloading artifacts from '$downloadUrl'..."
curl -o $target_dir/artifacts.zip "$downloadUrl"
unzip $target_dir/artifacts.zip -d $target_dir
mv $target_dir/$artifactName/* $target_dir
rm -rf $target_dir/artifacts.zip
rmdir $target_dir/$artifactName
ls -l $target_dir
artifacts:
expire_in: 2 weeks
paths:
- artifacts
compute-ci-package-snapshot-version:
image: registry.ddbuild.io/ci/auto_inject/gitlab:current
tags: ["arch:amd64"]
stage: single-step-testing
needs:
- wait-for-single-step-artifacts
script:
- cp .gitlab/.gitlab-ci-snapshot-package-test-template.yml gitlab-ci-snapshot-package-test.yml
- |
echo "Generating CI DEV configuration from template"
if test -f "artifacts/version.txt"; then
#Set the version of the package
DOTNET_PACKAGE_SPECIFIC_VERSION=$(cat artifacts/version.txt)
DOTNET_PACKAGE_DEV_VERSION=${DOTNET_PACKAGE_SPECIFIC_VERSION}.dev.${CI_COMMIT_SHORT_SHA}
DOTNET_PACKAGE_MAJOR_VERSION=$(cat artifacts/version.txt | cut -d '.' -f 1)
echo "DOTNET_PACKAGE_SPECIFIC_VERSION=${DOTNET_PACKAGE_SPECIFIC_VERSION}"
echo "DOTNET_PACKAGE_DEV_VERSION=${DOTNET_PACKAGE_DEV_VERSION}"
echo "DOTNET_PACKAGE_MAJOR_VERSION=${DOTNET_PACKAGE_MAJOR_VERSION}"
#Set values in the gitlab-ci file
sed -i "s/DOTNET_PACKAGE_SPECIFIC_VERSION:/DOTNET_PACKAGE_SPECIFIC_VERSION: ${DOTNET_PACKAGE_SPECIFIC_VERSION}/g" gitlab-ci-snapshot-package-test.yml
sed -i 's/description: "Three parts version for current build"/##/g' gitlab-ci-snapshot-package-test.yml
sed -i "s/DOTNET_PACKAGE_DEV_VERSION:/DOTNET_PACKAGE_DEV_VERSION: ${DOTNET_PACKAGE_DEV_VERSION}/g" gitlab-ci-snapshot-package-test.yml
sed -i 's/description: "Specific version for current build"/##/g' gitlab-ci-snapshot-package-test.yml
sed -i "s/DOTNET_PACKAGE_MAJOR_VERSION:/DOTNET_PACKAGE_MAJOR_VERSION: ${DOTNET_PACKAGE_MAJOR_VERSION}/g" gitlab-ci-snapshot-package-test.yml
sed -i 's/description: "Major version. By default all builds are generated with the same major version"/##/g' gitlab-ci-snapshot-package-test.yml
fi
echo "----------------------------------"
echo "----------------------------------"
cat gitlab-ci-snapshot-package-test.yml
artifacts:
paths:
- gitlab-ci-snapshot-package-test.yml
trigger-package-snapshot-testing:
stage: single-step-testing
needs:
- compute-ci-package-snapshot-version
variables:
PARENT_PIPELINE_ID: $CI_PIPELINE_ID
trigger:
strategy: depend
include:
- artifact: gitlab-ci-snapshot-package-test.yml
job: compute-ci-package-snapshot-version
publish:
only:
- master
- main
- /^hotfix.*$/
- /^release.*$/
except:
variables:
- $DEPLOY_TO_REL_ENV == "true"
- $CI_COMMIT_TAG # We don't need to build/publish when building a release tag
stage: publish
tags: ["runner:windows-docker", "windowsversion:1809"]
dependencies:
- build
script:
- $result = aws sts assume-role --role-arn "arn:aws:iam::486234852809:role/ci-datadog-windows-filter" --role-session-name AWSCLI-Session
- $resultjson = $result | convertfrom-json
- $credentials = $($resultjson.Credentials)
- $Env:AWS_ACCESS_KEY_ID="$($credentials.AccessKeyId)"
- $Env:AWS_SECRET_ACCESS_KEY="$($credentials.SecretAccessKey)"
- $Env:AWS_SESSION_TOKEN="$($credentials.SessionToken)"
- |
$i = 0
do {
try {
# The grants option at the end is used to allow public access on the files we upload as the acls only aren't enough.
aws s3 cp artifacts/ s3://dd-windowsfilter/builds/tracer/${CI_COMMIT_SHA} --recursive --region us-east-1 --exclude "*" --include "*.zip" --include "*.msi" --grants read=uri=http://acs.amazonaws.com/groups/global/AllUsers full=id=3a6e02b08553fd157ae3fb918945dd1eaae5a1aa818940381ef07a430cf25732
If ($LASTEXITCODE -eq 0) {
return
}
throw "Error uploading artifacts to S3"
} catch {
$msg = $Error[0].Exception.Message
Write-Output "Encountered error during while publishing to S3. Error Message is $msg."
Write-Output "Retrying..."
$i++
Start-Sleep -Milliseconds 100
}
} while ($i -lt 3)
# The 2 jobs below, do not depend on the packaging stage. So they are always run, even if package stage can fail.
generate-lib-init-tag-values:
tags: ["arch:amd64"]
image: registry.ddbuild.io/ci/auto_inject/gitlab:current
stage: deploy
rules:
- if: '$POPULATE_CACHE'
when: never
# We don't tag prerelease versions
- if: '$CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+$/'
when: always
- when: manual
allow_failure: true
variables:
IMG_DESTINATION_BASE: dd-lib-dotnet-init
ADDITIONAL_TAG_SUFFIXES: musl # comma separated list of additional tag suffixes
script:
- ./.gitlab/build-lib-init.sh
artifacts:
reports:
dotenv: build.env
deploy-lib-init-trigger:
stage: deploy
trigger:
# project: DataDog/dd-trace-dotnet-gitlab-test # can be used for testing
project: DataDog/public-images
branch: main
strategy: depend
variables:
IMG_SOURCES: ghcr.io/datadog/dd-trace-dotnet/dd-lib-dotnet-init:$CI_COMMIT_SHA
IMG_DESTINATIONS: $IMG_DESTINATIONS
IMG_SIGNING: "false"
# needs the version from the generate-tag-values job
needs:
- job: generate-lib-init-tag-values
artifacts: true
deploy-musl-lib-init-trigger:
stage: deploy
trigger:
# project: DataDog/dd-trace-dotnet-gitlab-test # can be used for testing
project: DataDog/public-images
branch: main
strategy: depend
variables:
IMG_SOURCES: ghcr.io/datadog/dd-trace-dotnet/dd-lib-dotnet-init:$CI_COMMIT_SHA-musl
IMG_DESTINATIONS: $IMG_DESTINATIONS_musl
IMG_SIGNING: "false"
# needs the version from the generate-tag-values job
needs:
- job: generate-lib-init-tag-values
artifacts: true
package:
extends: .package
rules:
- if: $DOTNET_PACKAGE_VERSION
when: on_success
- if: '$CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+(-prerelease)?$/'
when: manual
allow_failure: false
script:
- ../.gitlab/build-deb-rpm.sh
variables:
ARCH: amd64
package-arm:
extends: .package-arm
rules:
- if: $DOTNET_PACKAGE_VERSION
when: on_success
- if: '$CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+(-prerelease)?$/'
when: manual
allow_failure: false
script:
- ../.gitlab/build-deb-rpm.sh
variables:
ARCH: arm64
package-oci:
stage: package
extends: .package-oci
rules:
- if: $DOTNET_PACKAGE_VERSION
when: on_success
- if: '$CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+(-prerelease)?$/'
when: manual
allow_failure: false
script:
- ../.gitlab/build-oci.sh
parallel:
matrix:
- ARCH:
- arm64
- amd64
.release-package:
stage: deploy
variables:
PRODUCT_NAME: auto_inject-dotnet
benchmark-serverless:
stage: benchmarks
image: registry.ddbuild.io/ci/serverless-tools:1
tags: ["arch:amd64"]
when: on_success
needs:
- benchmark-serverless-trigger
script:
- git clone https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.ddbuild.io/DataDog/serverless-tools.git ./serverless-tools && cd ./serverless-tools
- ./ci/check_trigger_status.sh
benchmark-serverless-trigger:
stage: benchmarks
needs: []
trigger:
project: DataDog/serverless-tools
strategy: depend
allow_failure: true
variables:
UPSTREAM_PIPELINE_ID: $CI_PIPELINE_ID
UPSTREAM_PROJECT_URL: $CI_PROJECT_URL
UPSTREAM_COMMIT_BRANCH: $CI_COMMIT_BRANCH
UPSTREAM_COMMIT_AUTHOR: $CI_COMMIT_AUTHOR
UPSTREAM_COMMIT_TITLE: $CI_COMMIT_TITLE
UPSTREAM_COMMIT_TAG: $CI_COMMIT_TAG
UPSTREAM_PROJECT_NAME: $CI_PROJECT_NAME
UPSTREAM_GITLAB_USER_LOGIN: $GITLAB_USER_LOGIN
UPSTREAM_GITLAB_USER_EMAIL: $GITLAB_USER_EMAIL