Skip to content

Commit

Permalink
staging-20230930_60k_basic_targetted_epoch37_Myria3DV3.4.0
Browse files Browse the repository at this point in the history
  • Loading branch information
CharlesGaydon committed Oct 12, 2023
1 parent 0c528d3 commit 98afe19
Show file tree
Hide file tree
Showing 3 changed files with 93 additions and 49 deletions.
75 changes: 75 additions & 0 deletions .github/workflows/staging.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,75 @@
name: staging-model

on:
push:
branches:
- 'staging*'

env:
nexus_server: 10.128.81.69:8082
cicd_path: /var/data/cicd/CICD_github_assets/
model_id: 20230930_60k_basic_targetted_epoch37_Myria3DV3.4.0

jobs:
staging-model:
runs-on: self-hosted

steps:

- name: Checkout branch
uses: actions/checkout@v3

- name: Build docker image
run: docker build -t myria3d .

- name: Inference on validation set of 20211001_building_val-lidar-prod-optimization
run: >
docker run
-v ${cicd_path}:/cicd/
--ipc=host
--shm-size=2gb
myria3d
python run.py
task.task_name=predict
predict.src_las=/cicd/20211001_building_val-lidar-prod-optimization/val/*.las
predict.output_dir=/cicd/lidar_prod_optimization/${model_id}/preds-valset/
predict.interpolator.probas_to_save=[building]
- name: Inference on validation set of 20211001_building_val-lidar-prod-optimization
run: >
docker run
-v ${cicd_path}:/cicd/
--ipc=host
--shm-size=2gb
myria3d
python run.py
task.task_name=predict
predict.src_las=/cicd/20211001_building_val-lidar-prod-optimization/test/*.las
predict.output_dir=/cicd/lidar_prod_optimization/${model_id}/preds-testset/
predict.interpolator.probas_to_save=[building]
# NB: we run the new, tagged image at least once so that is it not prunned
# by mistake when using docker system prune.
- name: Tag the docker image with branch name
run: |
docker tag myria3d:latest myria3d:${{github.ref_name}}
docker run myria3d:${{github.ref_name}} bash
# get version number and date, to tag the image pushed to nexus
- name: get version number
id: tag
run: |
echo "::set-output name=version::$(docker run myria3d grep '__version__' package_metadata.yaml| cut -d\" -f2)"
echo "::set-output name=date::$(date '+%Y.%m.%d')"
- name: push branch docker on nexus (tagged with the branch name)
# we push on nexus an image from a branch when it's pushed
if: ((github.event_name == 'push') && (github.ref_name != 'main'))
run: |
docker tag myria3d $nexus_server/lidar_hd/myria3d:${{steps.tag.outputs.version}}-${{github.ref_name}}
docker login $nexus_server --username svc_lidarhd --password ${{ secrets.PASSWORD_SVC_LIDARHD }}
docker push $nexus_server/lidar_hd/myria3d:${{steps.tag.outputs.version}}-${{github.ref_name}}
- name: Clean dangling docker images
if: always() # always do it, even if something failed
run: docker system prune --force # remove dangling docker images, without asking user for confirmation
Binary file not shown.
Original file line number Diff line number Diff line change
Expand Up @@ -5,20 +5,15 @@ print_config: true
ignore_warnings: true
trainer:
_target_: pytorch_lightning.Trainer
gpus:
- 0
- 1
min_epochs: 100
max_epochs: 150
gpus: 0
min_epochs: 300
max_epochs: 400
log_every_n_steps: 1
weights_summary: null
progress_bar_refresh_rate: 1
auto_lr_find: false
num_sanity_val_steps: 2
accumulate_grad_batches: 3
strategy: ddp
accelerator: gpu
num_processes: 2
datamodule:
transforms:
preparations:
Expand Down Expand Up @@ -118,9 +113,9 @@ datamodule:
preparations_predict_list: '${oc.dict.values: datamodule.transforms.preparations.predict}'
normalizations_list: '${oc.dict.values: datamodule.transforms.normalizations}'
_target_: myria3d.pctl.datamodule.hdf5.HDF5LidarDataModule
data_dir: null
split_csv_path: null
hdf5_file_path: /var/data/cgaydon/data/20220607_151_dalles_proto.myria3d-V2.4.hdf5
data_dir: /mnt/store-lidarhd/projet-LHD/IA/PACASAM-SHARED-WORKSPACE/CGaydon/20230930_60k_basic_targetted/data/
split_csv_path: /mnt/store-lidarhd/projet-LHD/IA/PACASAM-SHARED-WORKSPACE/CGaydon/20230930_60k_basic_targetted/data/split.csv
hdf5_file_path: /var/data/CGaydon/myria3d_datasets/20230930_60k_basic_targetted.hdf5
points_pre_transform:
_target_: functools.partial
_args_:
Expand All @@ -130,7 +125,7 @@ datamodule:
_args_:
- ${get_method:myria3d.pctl.dataset.utils.pre_filter_below_n_points}
min_num_nodes: 1
tile_width: 1000
tile_width: 50
subtile_width: 50
subtile_overlap_train: 0
subtile_overlap_predict: ${predict.subtile_overlap}
Expand All @@ -142,30 +137,10 @@ dataset_description:
classification_preprocessing_dict:
3: 5
4: 5
160: 64
161: 64
162: 64
0: 1
7: 1
46: 1
47: 1
48: 1
49: 1
50: 1
51: 1
52: 1
53: 1
54: 1
55: 1
56: 1
57: 1
58: 1
64: 1
66: 1
67: 1
77: 1
155: 1
204: 1
66: 65
100: 1
101: 1
classification_dict:
1: unclassified
2: ground
Expand All @@ -174,14 +149,6 @@ dataset_description:
9: water
17: bridge
64: lasting_above
class_weights:
- 0.25
- 0.1
- 0.1
- 0.5
- 2.0
- 2.0
- 2.0
d_in: 9
num_classes: 7
callbacks:
Expand Down Expand Up @@ -228,9 +195,11 @@ model:
patience: 20
cooldown: 5
verbose: true
min_lr: 0.001
criterion:
_target_: torch.nn.CrossEntropyLoss
label_smoothing: 0.0
ignore_index: 65
_target_: myria3d.models.model.Model
d_in: ${dataset_description.d_in}
num_classes: ${dataset_description.num_classes}
Expand Down Expand Up @@ -259,21 +228,21 @@ logger:
api_key: ${oc.env:COMET_API_TOKEN}
workspace: ${oc.env:COMET_WORKSPACE}
project_name: ${oc.env:COMET_PROJECT_NAME}
experiment_name: '[V3.0.2-BS10xMAX40000pts] RandLaNet_base_run_FR_pyg_randla_net_NoRS'
experiment_name: 20230930_60k_basic_targetted
auto_log_co2: false
disabled: false
task:
task_name: predict
task_name: fit
predict:
src_las: /path/to/input.las
output_dir: /path/to/output_dir/
ckpt_path: trained_model_assets/proto151_V2.0_epoch_100_Myria3DV3.1.0.ckpt
subtile_overlap: 0
ckpt_path: /path/to/lightning_model.ckpt
gpus: 0
subtile_overlap: 0
interpolator:
_target_: myria3d.models.interpolation.Interpolator
interpolation_k: 10
classification_dict: ${dataset_description.classification_dict}
probas_to_save: [building,ground]
predicted_classification_channel: confidence
probas_to_save: all
predicted_classification_channel: PredictedClassification
entropy_channel: entropy

0 comments on commit 98afe19

Please sign in to comment.