-
Notifications
You must be signed in to change notification settings - Fork 1
328 lines (322 loc) · 14.9 KB
/
build-workspace-product-part.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
name: build-workspace-product-part
# will hopefully possible very soon with the new linter
# run-name: build for component ${{ inputs.component }}
on:
workflow_call:
secrets:
DATAVISYN_BOT_REPO_TOKEN:
required: false
DV_BOT_USER:
required: false
NODE_VERSION:
required: false
PYTHON_VERSION:
required: false
GITLAB_TOKEN:
required: false
GITLAB_HOST:
required: false
inputs:
component:
description: "component that should be built"
required: true
type: string
image_tag1:
description: "image tag 1 to push the image"
required: true
type: string
image_tag2:
description: "image tag 2 for labeling"
required: true
type: string
build_time:
description: "actually build time (in RFC 3339)"
required: true
type: string
stage:
description: "stage for the image (develop or production) depending on the branch name"
required: true
type: string
env:
VISYN_SCRIPTS_VERSION: "v7" # visyn_scripts@v7 is the last version with workspace support
TIME_ZONE: "Europe/Vienna"
NODE_VERSION: "20.9"
PYTHON_VERSION: "3.10"
WORKFLOW_BRANCH: "main"
PYTHON_BASE_IMAGE: "python:3.10.8-slim-bullseye"
DATAVISYN_PYTHON_BASE_IMAGE: "188237246440.dkr.ecr.eu-central-1.amazonaws.com/datavisyn/base/python:main"
DATAVISYN_NGINX_BASE_IMAGE: "188237246440.dkr.ecr.eu-central-1.amazonaws.com/datavisyn/base/nginx:main"
permissions:
id-token: write
contents: read
jobs:
build-components:
runs-on: ubuntu-22.04
steps:
# checkout specific repository
- uses: actions/checkout@v3
with:
token: ${{ github.event.repository.private == true && secrets.DATAVISYN_BOT_REPO_TOKEN || github.token }}
# checkout this workflow repository to get actions
- uses: actions/checkout@v3
with:
repository: datavisyn/github-workflows
ref: ${{ env.WORKFLOW_BRANCH }}
path: ./tmp/github-workflows
- name: get parameters
id: get-parameters
run: |
type=$(jq --arg var "${COMPONENT}" -rc '.components | to_entries | .[] | select(.key==$var)| .value.type' ./visyn_product.json)
directory=$(jq --arg var "${COMPONENT}" -rc '.components | to_entries | .[] | select(.key==$var)| .value.directory' ./visyn_product.json)
ecr_repo=$(jq --arg var "${COMPONENT}" -rc '.components | to_entries | .[] | select(.key==$var)| .value.ecr_repo' ./visyn_product.json)
app=$(jq -rc '.app' ./visyn_product.json)
default_app=$(jq -rc '.defaultApp' ./visyn_product.json)
repo=$(jq -rc '.repo' ./visyn_product.json)
branch=$(jq -rc '.branch' ./visyn_product.json)
if [[ ${type} == web ]]; then
image_base_label=$DATAVISYN_NGINX_BASE_IMAGE
elif [[ ${type} == api ]]; then
image_base_label=$DATAVISYN_PYTHON_BASE_IMAGE
fi
image_url="$REPOSITORY_HTML/$directory/Dockerfile"
image_url="${image_url/\/\.\//\/}"
echo "type=$type"
echo "directory=$directory"
echo "ecr_repo=$ecr_repo"
echo "app=$app"
echo "default_app=$default_app"
echo "repo=$repo"
echo "branch=$branch"
echo "image_base_label=$image_base_label"
echo "image_url=$image_url"
# echo to output
# shellcheck disable=SC2129
echo "type=$type" >> "$GITHUB_OUTPUT"
echo "directory=$directory" >> "$GITHUB_OUTPUT"
echo "ecr_repo=$ecr_repo" >> "$GITHUB_OUTPUT"
echo "app=$app" >> "$GITHUB_OUTPUT"
echo "default_app=$default_app" >> "$GITHUB_OUTPUT"
echo "repo=$repo" >> "$GITHUB_OUTPUT"
echo "branch=$branch" >> "$GITHUB_OUTPUT"
echo "image_base_label=$image_base_label" >> "$GITHUB_OUTPUT"
echo "image_url=$image_url" >> "$GITHUB_OUTPUT"
env:
COMPONENT: ${{ inputs.component }}
DATAVISYN_NGINX_BASE_IMAGE: ${{ env.DATAVISYN_NGINX_BASE_IMAGE }}
DATAVISYN_PYTHON_BASE_IMAGE: ${{ env.DATAVISYN_PYTHON_BASE_IMAGE }}
REPOSITORY_HTML: ${{ github.event.repository.html_url }}
- name: setup node
if: ${{ steps.get-parameters.outputs.type == 'web' }}
uses: actions/setup-node@v4
with:
node-version: ${{ secrets.NODE_VERSION || env.NODE_VERSION }}
registry-url: ${{ env.NPM_REGISTRY }}
- name: setup python
if: ${{ steps.get-parameters.outputs.type == 'api' }}
uses: actions/setup-python@v5
with:
python-version: ${{ secrets.PYTHON_VERSION || env.PYTHON_VERSION }}
# TODO: This fails with Error: No file in ..._product matched to [**/requirements.txt]
# cache: 'pip'
- name: Git config
run: |
git config --global url."https://${_GITHUB_TOKEN}@github".insteadOf "https://github"
git config --add --global url."https://[email protected]/".insteadOf ssh://[email protected]/
git config --add --global url."https://[email protected]/".insteadOf [email protected]:
if [[ -n $GITLAB_TOKEN && -n $GITLAB_HOST ]] ; then
echo "set gitlab config"
git config --global url."https://$GITLAB_TOKEN@$GITLAB_HOST/".insteadOf "git@$GITLAB_HOST:"
git config --global url."git+https://$GITLAB_TOKEN@$GITLAB_HOST/".insteadOf "git+ssh://git@$GITLAB_HOST:"
fi
env:
_GITHUB_TOKEN: ${{ github.event.repository.private == true && secrets.DATAVISYN_BOT_REPO_TOKEN || github.token }}
GITLAB_TOKEN: ${{ secrets.GITLAB_TOKEN }}
GITLAB_HOST: ${{ secrets.GITLAB_HOST }}
- name: Install visyn_scripts
run: |
if [[ "$VISYN_SCRIPTS_VERSION" == v[0-9]* ]] ; then
npm install -g "visyn_scripts@$VISYN_SCRIPTS_VERSION"
else
npm install -g "https://github.com/datavisyn/visyn_scripts.git#$VISYN_SCRIPTS_VERSION"
fi
env:
VISYN_SCRIPTS_VERSION: ${{ env.VISYN_SCRIPTS_VERSION }}
- name: create workspace
run: |
mkdir "./tmp/$COMPONENT"
cd "./tmp/$COMPONENT"
repository="$REPO"
if [[ $repository == *"gitlab"* && -n $GITLAB_TOKEN ]] ; then
repository=${repository/"https://"/"https://$GITLAB_TOKEN@"}
else
repository=https://[email protected]/${repository}.git
fi
echo "$repository"
git clone -b "$BRANCH_NAME" --depth 1 "$repository"
readarray -t my_array < <(jq --arg var "${COMPONENT}" -rc '.components | to_entries | .[] | select(.key==$var)| .value.additionals[]?' ../../visyn_product.json)
for additional in "${my_array[@]}"; do
echo "$additional"
name=$(echo "$additional" | jq -rc '.name')
repository=$(echo "$additional" | jq -rc '.repo')
if [[ $repository == *"gitlab"* && -n $GITLAB_TOKEN ]] ; then
repository=${repository/"https://"/"https://$GITLAB_TOKEN@"}
else
repository=https://[email protected]/${repository}.git
fi
branch=$(echo "$additional" | jq -rc '.branch')
git clone -b "$branch" --depth 1 "$repository" "$name"
done
rm -f ./../../.yo-rc.json
visyn_scripts workspace-update --defaultApp="$DEFAULT_APP" --noAdditionals=true --addWorkspaceRepos=false
env:
COMPONENT: ${{ inputs.component }}
REPO: ${{ steps.get-parameters.outputs.repo }}
BRANCH_NAME: ${{ steps.get-parameters.outputs.branch }}
APP: ${{ steps.get-parameters.outputs.app }}
DEFAULT_APP: ${{ steps.get-parameters.outputs.default_app }}
GH_TOKEN: ${{ github.event.repository.private == true && secrets.DATAVISYN_BOT_REPO_TOKEN || github.token }}
GITLAB_TOKEN: ${{ secrets.GITLAB_TOKEN }}
GITLAB_HOST: ${{ secrets.GITLAB_HOST }}
- name: Inject version from product package.json into workspace package.json
run: |
product_version=$(jq -rc '.version' ./package.json)
echo "product_version=$product_version"
if [[ $product_version == *"SNAPSHOT"* ]]; then
echo "replace SNAPSHOT in version with timestamp"
product_version="${product_version//SNAPSHOT/$(date +%Y%m%d-%H%M%S)}"
echo "product_version=$product_version"
fi
workspace_version=$(jq -rc '.version' "./tmp/$COMPONENT/package.json")
echo "workspace_version=$workspace_version"
if [[ "$product_version" != "$workspace_version" ]]; then
echo "update workspace version"
jq --arg version "$product_version" '.version = $version' "./tmp/$COMPONENT/package.json" > "./tmp/$COMPONENT/package.json.tmp"
mv "./tmp/$COMPONENT/package.json.tmp" "./tmp/$COMPONENT/package.json"
echo "workspace version updated to $(jq -rc '.version' './tmp/$COMPONENT/package.json')"
fi
env:
COMPONENT: ${{ inputs.component }}
- name: Copy docker sources
run: |
ls -lah .
ls -lah "$COMPONENT_DIRECTORY"
if [[ -d "$COMPONENT_DIRECTORY/docker" ]]; then
echo "copy docker directory"
cp -r "$COMPONENT_DIRECTORY/docker" "./tmp/$COMPONENT/docker"
ls -lah "./tmp/$COMPONENT/docker"
fi
env:
COMPONENT: ${{ inputs.component }}
COMPONENT_DIRECTORY: ${{ steps.get-parameters.outputs.directory }}
APP: ${{ steps.get-parameters.outputs.app }}
- name: Build web
if: ${{ steps.get-parameters.outputs.type == 'web' }}
run: |
stage_str=".dev"
if [[ "$STAGE" == "production" ]]; then
stage_str=".prod"
elif [[ "$STAGE" == "insight" ]]; then
stage_str=".insight"
fi
echo "$stage_str"
if [[ -f "$COMPONENT_DIRECTORY/.env" ]]; then
echo "copy .env"
cat "$COMPONENT_DIRECTORY/.env"
cp "$COMPONENT_DIRECTORY/.env" "./tmp/$COMPONENT"
fi
if [[ -f "${COMPONENT_DIRECTORY}/${stage_str}.env" ]]; then
cat "${COMPONENT_DIRECTORY}/${stage_str}.env" >> "./tmp/$COMPONENT/.env"
fi
if [[ -f "$COMPONENT_DIRECTORY/workspace.scss" ]]; then
echo "copy workspace.scss"
cp "$COMPONENT_DIRECTORY/workspace.scss" "./tmp/$COMPONENT"
fi
cd "./tmp/$COMPONENT"
ls -lah
touch ./yarn.lock
yarn install --no-immutable
yarn info --name-only
yarn run dist
env:
COMPONENT: ${{ inputs.component }}
COMPONENT_DIRECTORY: ${{ steps.get-parameters.outputs.directory }}
STAGE: ${{ inputs.stage }}
- name: Build api
if: ${{ steps.get-parameters.outputs.type == 'api' }}
run: |
cd "./tmp/$COMPONENT"
ls -lah
python -m pip install --upgrade pip uv
uv pip install setuptools wheel --system
mkdir -p ./build/source
mkdir -p ./dist_python/
cd "$DEFAULT_APP"
if test -f "Makefile"; then
echo "Makefile exists in $DEFAULT_APP, trying to build"
make build
cp -r ./build/lib/* ../build/source/
cp -r "./${DEFAULT_APP,,}.egg-info" "../build/source/${DEFAULT_APP,,}.egg-info"
cp -r ./dist_python/* ../dist_python/
else
echo "$APP has no Makefile, therefore no backend can be built"
fi
cd ..
readarray -t my_array < <(jq --arg var "${COMPONENT}" -rc '.components | to_entries | .[] | select(.key==$var)| .value.additionals[]?' ../../visyn_product.json)
for additional in "${my_array[@]}"; do
echo "$additional"
name=$(echo "$additional" | jq -rc '.name')
# shellcheck disable=SC2001
name=$(echo "$name" | sed -e 's|.*/||')
cd "$name"
make build
cp -r ./build/lib/* ../build/source/
cp -r "./${name,,}.egg-info" "../build/source/${name,,}.egg-info"
cp -r ./dist_python/* ../dist_python/
cd ..
done
env:
COMPONENT: ${{ inputs.component }}
APP: ${{ steps.get-parameters.outputs.app }}
DEFAULT_APP: ${{ steps.get-parameters.outputs.default_app }}
COMPONENT_DIRECTORY: ${{ steps.get-parameters.outputs.directory }}
# checkout this workflow repository to get actions
- uses: ./tmp/github-workflows/.github/actions/build-push-image
with:
aws_role: ${{ vars.DV_AWS_ECR_ROLE }}
aws_region: ${{ vars.DV_AWS_REGION }}
ecr_registry: ${{ vars.DV_AWS_ECR_REGISTRY }}
ecr_repository: ${{ steps.get-parameters.outputs.ecr_repo }}
docker_file: ./tmp/${{ inputs.component }}/docker/Dockerfile
current_directory: ./tmp/${{ inputs.component }}
image_tag: ${{ inputs.image_tag1 }}
build_args: |
PYTHON_BASE_IMAGE=${{ env.PYTHON_BASE_IMAGE }}
DATAVISYN_PYTHON_BASE_IMAGE=${{ env.DATAVISYN_PYTHON_BASE_IMAGE }}
GIT_ACCESS_TOKEN=${{ secrets.DATAVISYN_BOT_REPO_TOKEN }}
DATAVISYN_NGINX_BASE_IMAGE=${{ env.DATAVISYN_NGINX_BASE_IMAGE }}
labels: |
name=${{ steps.get-parameters.outputs.ecr_repo }}
version=${{ inputs.image_tag2 }}
org.opencontainers.image.description=${{ inputs.component }} image for ${{ steps.get-parameters.outputs.app }}
org.opencontainers.image.base.name=${{ steps.get-parameters.outputs.image_base_label }}
org.opencontainers.image.source=${{ steps.get-parameters.outputs.image_url }}
org.opencontainers.image.url=${{ steps.get-parameters.outputs.image_url }}
org.opencontainers.image.title=${{ steps.get-parameters.outputs.ecr_repo }}
org.opencontainers.image.version=${{ inputs.image_tag2 }}
org.opencontainers.image.created=${{ inputs.build_time }}
org.opencontainers.image.revision=${{ github.sha }}
- name: scan image
id: get-ecr-scan-result
uses: ./tmp/github-workflows/.github/actions/get-ecr-scan-result
with:
aws_role: ${{ vars.DV_AWS_ECR_ROLE }}
aws_region: ${{ vars.DV_AWS_REGION }}
ecr_registry: ${{ vars.DV_AWS_ECR_REGISTRY }}
ecr_repository: ${{ steps.get-parameters.outputs.ecr_repo }}
image_tag: ${{ inputs.image_tag1 }}
- name: check scan results
run: |
if [ "${{ steps.get-ecr-scan-result.outputs.critical }}" != "null" ] || [ "${{ steps.get-ecr-scan-result.outputs.high }}" != "null" ]; then
echo "Docker image contains vulnerabilities at critical or high level"
exit 1 #exit execution due to docker image vulnerabilities
fi