-
Notifications
You must be signed in to change notification settings - Fork 23
160 lines (139 loc) · 6.25 KB
/
cicd.yaml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
name: CICD
on:
# Run CICD for non-draft pull request
pull_request:
branches:
- main
# Also run when the pull request merges (which generates a push)
# So that we can tag the docker image appropriately.
push:
branches:
- main
- staging-*
jobs:
CICD:
runs-on: self-hosted
env:
http_proxy: ${{ secrets.PROXY_URL }}
https_proxy: ${{ secrets.PROXY_URL }}
steps:
- name: Checkout branch
uses: actions/checkout@v4
- name: Build docker image
run: docker build --build-arg http_proxy=${{ secrets.PROXY_URL }} --build-arg https_proxy=${{ secrets.PROXY_URL }} -t myria3d .
- name: Run pytest
run: >
docker run
--ipc=host
myria3d
python -m
pytest -rA -v
--ignore=actions-runner
# IMPORTANT: Always run images with --ipc=host and --shm-size=2gb (at least) to enable
# sufficient shared memory when predicting on large files.
- name: Example inference run via Docker with default config and checkpoint
run: >
docker run
-v /var/data/cicd/CICD_github_assets/myria3d_V3.7.0/inputs/:/inputs/
-v /var/data/cicd/CICD_github_assets/myria3d_V3.7.0/outputs/:/outputs/
--ipc=host
--shm-size=2gb
myria3d
python run.py
predict.src_las=/inputs/792000_6272000_subset_buildings.las
datamodule.epsg=2154
predict.output_dir=/outputs/
task.task_name=predict
# predict.subtile_overlap specifies overlap between adjacent samples (in meters).
- name: Example inference run via Docker with inference-time subtiles overlap to smooth-out results.
run: >
docker run
-v /var/data/cicd/CICD_github_assets/myria3d_V3.7.0/inputs/:/inputs/
-v /var/data/cicd/CICD_github_assets/myria3d_V3.7.0/outputs/:/outputs/
--ipc=host
--shm-size=2gb
myria3d
python run.py
--config-path /inputs/
--config-name proto151_V2.0_epoch_100_Myria3DV3.1.0_predict_config_V3.7.0
predict.ckpt_path=/inputs/proto151_V2.0_epoch_100_Myria3DV3.1.0.ckpt
datamodule.epsg=2154
predict.src_las=/inputs/792000_6272000_subset_buildings.las
predict.output_dir=/outputs/
predict.subtile_overlap=25
datamodule.batch_size=10
predict.interpolator.probas_to_save=[building,ground]
task.task_name=predict
- name: Check code neatness (linter)
run: docker run myria3d python -m flake8
# Everything ran so we tag the valid docker image to keep it
# This happens for push events, which are in particular
# triggered when a pull request is merged.
- name: Tag the docker image with branch name
if: github.event_name == 'push'
run: |
docker tag myria3d:latest myria3d:${{github.ref_name}}
docker run myria3d:${{github.ref_name}} bash # Run the new, tagged image at least once so that is it not prunned by mistake when using docker system prune
# docker save myria3d:${{github.ref_name}} -o /var/data/cicd/CICD_github_assets/CICD_docker_images/myria3d_${github.ref_name}.tar # Save the docker image as myria3d_${github.ref_name}.tar
# get version number and date, to tag the image pushed to a private docker registry
- name: get version number
id: tag
run: |
echo "VERSION=$(docker run myria3d python -m myria3d._version)" >> $GITHUB_ENV
echo "DATE=$(date '+%Y.%m.%d')" >> $GITHUB_ENV
# show possible tags, for debugging purpose
- name: Print tags
run: |
echo "${{ env.VERSION }}"
echo "${{ env.DATE }}"
- name: push main docker on nexus (tagged with a date)
# we push on nexus an image from the main branch when it has been updated (push or accepted pull request)
if: ((github.ref_name == 'main') && (github.event_name == 'push'))
run: |
docker tag myria3d ${{ secrets.DOCKER_REGISTRY }}/lidar_hd/myria3d:${{ env.VERSION }}-${{ env.DATE }}
docker login ${{ secrets.DOCKER_REGISTRY }} --username svc_lidarhd --password ${{ secrets.PASSWORD_SVC_LIDARHD }}
docker push ${{ secrets.DOCKER_REGISTRY }}/lidar_hd/myria3d:${{ env.VERSION }}-${{ env.DATE }}
- name: push branch docker on nexus (tagged with the branch name)
# we push on nexus an image from a branch when it's pushed
if: ((github.event_name == 'push') && (github.ref_name != 'main'))
run: |
docker tag myria3d ${{ secrets.DOCKER_REGISTRY }}/lidar_hd/myria3d:${{ env.VERSION }}-${{github.ref_name}}
docker login ${{ secrets.DOCKER_REGISTRY }} --username svc_lidarhd --password ${{ secrets.PASSWORD_SVC_LIDARHD }}
docker push ${{ secrets.DOCKER_REGISTRY }}/lidar_hd/myria3d:${{ env.VERSION }}-${{github.ref_name}}
- name: Clean dangling docker images
if: always() # always do it, even if something failed
run: docker system prune --force # remove dangling docker images, without asking user for confirmation
publish-pypi:
runs-on: ubuntu-latest
if: ((github.ref_name == 'main') && (github.event_name == 'push'))
environment:
name: pypi
url: https://pypi.org/p/myria3d/
permissions:
contents: read
packages: write
id-token: write # IMPORTANT: this permission is mandatory for trusted publishing
steps:
- name: Checkout branch
uses: actions/checkout@v4
# See https://github.com/marketplace/actions/setup-micromamba
- name: setup-micromamba
uses: mamba-org/[email protected]
with:
environment-file: environment.yml
environment-name: myria3d # activate the environment
cache-environment: true
cache-downloads: true
generate-run-shell: true
- name: Run tests with pytest
shell: micromamba-shell {0}
run: python -m pytest ./tests -rA -v
- name: Clean up before building package
run: rm -rf tmp myria3d.egg-info dist
- name: Build pip package
shell: micromamba-shell {0}
run: python -m build
- name: pypi-publish
uses: pypa/[email protected]
with:
skip-existing: true