Skip to content

Commit 7406a2a

Browse files
refactor: update dev container and pod (#477)
* fix: remove syntax directive from conda Dockerfile * fix: update devcontainer for compatibility with dp - Update devcontainer name to "pyrovelocity-dev". - Update dockerfile path to "../dockerfiles/Dockerfile.conda". - Remove the "docker-in-docker" feature. * fix: add `.envrc` to `.gitignore` * feat: add dockerignore * fix: update vscode settings and extensions * fix: add make targets to install direnv and just * feat: add example envrc * feat: add justfile for devcontainer --------- Co-authored-by: pyrovelocity[bot] <[email protected]>
1 parent 6f0cda4 commit 7406a2a

File tree

9 files changed

+322
-13
lines changed

9 files changed

+322
-13
lines changed

.devcontainer/devcontainer.json

Lines changed: 2 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,9 @@
11
{
2-
"name": "pyrovelocity",
2+
"name": "pyrovelocity-dev",
33
"build": {
4-
"dockerfile": "../dockerfiles/Dockerfile.conda.cpu",
4+
"dockerfile": "../dockerfiles/Dockerfile.conda",
55
"context": ".."
66
},
7-
"features": {
8-
"ghcr.io/devcontainers/features/docker-in-docker:2": {}
9-
},
107
"customizations": {
118
"vscode": {
129
"extensions": [

.dockerignore

Lines changed: 121 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,121 @@
1+
#
2+
archive/
3+
4+
#
5+
.DS_Store
6+
7+
#
8+
monkeytype.sqlite3
9+
10+
# Byte-compiled / optimized / DLL files
11+
__pycache__/
12+
*.py[cod]
13+
*$py.class
14+
15+
# C extensions
16+
*.so
17+
18+
# Distribution / packaging
19+
.Python
20+
env/
21+
build/
22+
develop-eggs/
23+
dist/
24+
downloads/
25+
eggs/
26+
.eggs/
27+
lib/
28+
lib64/
29+
parts/
30+
sdist/
31+
var/
32+
wheels/
33+
*.egg-info/
34+
.installed.cfg
35+
*.egg
36+
37+
# PyInstaller
38+
# Usually these files are written by a python script from a template
39+
# before PyInstaller builds the exe, so as to inject date/other infos into it.
40+
*.manifest
41+
*.spec
42+
43+
# Installer logs
44+
pip-log.txt
45+
pip-delete-this-directory.txt
46+
47+
# Unit test / coverage reports
48+
htmlcov/
49+
.tox/
50+
.coverage
51+
.coverage.*
52+
.cache
53+
nosetests.xml
54+
coverage.xml
55+
*.cover
56+
.hypothesis/
57+
.pytest_cache/
58+
59+
# Translations
60+
*.mo
61+
*.pot
62+
63+
# Django stuff:
64+
*.log
65+
local_settings.py
66+
67+
# Flask stuff:
68+
instance/
69+
.webassets-cache
70+
71+
# Scrapy stuff:
72+
.scrapy
73+
74+
# Sphinx documentation
75+
docs/_build/
76+
77+
# PyBuilder
78+
target/
79+
80+
# Jupyter Notebook
81+
.ipynb_checkpoints
82+
83+
# pyenv
84+
.python-version
85+
86+
# celery beat schedule file
87+
celerybeat-schedule
88+
89+
# SageMath parsed files
90+
*.sage.py
91+
92+
# dotenv
93+
.env
94+
.envrc
95+
96+
# virtualenv
97+
.venv
98+
venv/
99+
ENV/
100+
101+
# Spyder project settings
102+
.spyderproject
103+
.spyproject
104+
105+
# Rope project settings
106+
.ropeproject
107+
108+
# mkdocs documentation
109+
/site
110+
111+
# mypy
112+
.mypy_cache/
113+
114+
# IDE settings
115+
*.code-workspace
116+
.VSCodeCounter
117+
.idea/
118+
*.pdf
119+
*.tif
120+
*.h5ad
121+
.nox/

.envrc.example

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
export CONTAINER_BUILDER=podman # path to container management executable
2+
export GITHUB_USERNAME=username # github username associated to uploading startup scripts as github gists
3+
export GITHUB_ORG_NAME=pinellolab # name of the github org or user containing the github repository with code for development
4+
export GITHUB_REPO_NAME=pyrovelocity # name of a github repository with a conda environment yaml file
5+
export GITHUB_BRANCH_NAME=master # name of github repository branch to checkout
6+
export POD_DISK_SIZE=400Gi # size of the PVC to mount to the devpod
7+
export POD_ACCELERATOR_TYPE=nvidia-tesla-t4 # cluster-specific node selector for accelerator type | nvidia-l4 | nvidia-tesla-a100
8+
export POD_MIN_CPU=16 # cpu resource request for kubernetes pod
9+
export POD_MIN_MEM=64Gi # ram resource request for kubernetes pod
10+
export POD_MAX_CPU=30 # cpu resource limit for kubernetes pod
11+
export POD_MAX_MEM=96Gi # ram resource limit for kubernetes pod
12+
export POD_MAX_ACCEL=1 # accelerator resource limit for kubernetes pod

.gitignore

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -91,6 +91,7 @@ celerybeat-schedule
9191

9292
# dotenv
9393
.env
94+
.envrc
9495

9596
# virtualenv
9697
.venv
@@ -112,7 +113,6 @@ ENV/
112113

113114
# IDE settings
114115
*.code-workspace
115-
.vscode/
116116
.VSCodeCounter
117117
.idea/
118118
*.pdf

.vscode/extensions.json

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
{
2+
"recommendations": [
3+
"donjayamanne.python-extension-pack",
4+
"redhat.vscode-yaml",
5+
"sclu1034.justfile",
6+
"ms-vscode.makefile-tools",
7+
"hashicorp.terraform",
8+
"ms-python.black-formatter",
9+
"googlecloudtools.cloudcode",
10+
"ms-kubernetes-tools.vscode-kubernetes-tools",
11+
"ms-vsliveshare.vsli",
12+
"eamodio.gitlens",
13+
"GitHub.vscode-pull-request-github",
14+
"github.vscode-github-actions",
15+
"ms-azuretools.vscode-docker",
16+
"ms-toolsai.jupyter",
17+
"iterative.dvc",
18+
"njzy.stats-bar",
19+
"vscode-icons-team.vscode-icons",
20+
"poimandres.theme-poimandres"
21+
]
22+
}

.vscode/settings.json

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
{
2+
"[python]": {
3+
"editor.defaultFormatter": "ms-python.black-formatter",
4+
"editor.formatOnSave": true
5+
}
6+
}

Makefile

Lines changed: 39 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,45 @@ help: ## Display this help. (Default)
1919
help_sort: ## Display alphabetized version of help.
2020
@grep -hE '^[A-Za-z0-9_ \-]*?:.*##.*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'
2121

22+
#-------------
23+
# system / dev
24+
#-------------
25+
26+
install_direnv: ## Install direnv to `/usr/local/bin`. Check script before execution: https://direnv.net/ .
27+
@which direnv > /dev/null || \
28+
(curl -sfL https://direnv.net/install.sh | bash && \
29+
sudo install -c -m 0755 direnv /usr/local/bin && \
30+
rm -f ./direnv)
31+
@echo "see https://direnv.net/docs/hook.html"
32+
33+
install_just: ## Install just. Check script before execution: https://just.systems/ .
34+
@which cargo > /dev/null || (curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh)
35+
@cargo install just
36+
37+
precommit: ## Run pre-commit hooks using nox.
38+
nox -x -rs pre-commit
39+
40+
env_print: ## Print a subset of environment variables defined in ".envrc" file.
41+
env | grep "TF_VAR\|GITHUB\|GH_\|GCP_\|MLFLOW" | sort
42+
43+
approve_prs: ## Approve github pull requests from bots: PR_ENTRIES="2-5 10 12-18"
44+
for entry in $(PR_ENTRIES); do \
45+
if [[ "$$entry" == *-* ]]; then \
46+
start=$${entry%-*}; \
47+
end=$${entry#*-}; \
48+
for pr in $$(seq $$start $$end); do \
49+
@gh pr review $$pr --approve; \
50+
done; \
51+
else \
52+
@gh pr review $$entry --approve; \
53+
fi; \
54+
done
55+
56+
57+
#----------------
58+
# web application
59+
#----------------
60+
2261
st: ## Run streamlit app in local environment.
2362
streamlit run app/app.py \
2463
--server.port=8080 \
@@ -96,9 +135,3 @@ ifdef GCP_RUN_SERVICE_NAME
96135
else
97136
@echo 'Run "make help" and define necessary variables'
98137
endif
99-
100-
precommit: ## Run pre-commit hooks using nox.
101-
nox -x -rs pre-commit
102-
103-
env_print: ## Print a subset of environment variables defined in ".env" file.
104-
env | grep "TF_VAR\|GITHUB\|GH_\|GCP_\|MLFLOW" | sort

dockerfiles/Dockerfile.conda

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,3 @@
1-
# syntax=docker/dockerfile:1
21
FROM condaforge/mambaforge:23.1.0-1
32
LABEL maintainer="pyrovelocity team"
43

justfile

Lines changed: 119 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,119 @@
1+
# Default command when 'just' is run without arguments
2+
# Run 'just <command>' to execute a command.
3+
default: list
4+
5+
# Display help
6+
help:
7+
@printf "\nSee Makefile targets for just and direnv installation."
8+
@printf "\nRun 'just -n <command>' to print what would be executed...\n\n"
9+
@just --list --unsorted
10+
@echo "\n...by running 'just <command>'.\n"
11+
@echo "This message is printed by 'just help'."
12+
@echo "Just 'just' will just list the available recipes.\n"
13+
14+
# List just recipes
15+
list:
16+
@just --list --unsorted
17+
18+
# List evaluated just variables
19+
vars:
20+
@just --evaluate
21+
22+
builder := env_var_or_default('BUILDER', 'podman')
23+
container_user := "runner"
24+
container_home := "/home" / container_user
25+
container_work := container_home / "work"
26+
git_username := env_var_or_default('GITHUB_USERNAME', 'pinellolab')
27+
git_org_name := env_var_or_default('GITHUB_ORG_NAME', 'pinellolab')
28+
git_repo_name := env_var_or_default('GITHUB_REPO_NAME', 'pyrovelocity')
29+
git_branch_name := env_var_or_default('GITHUB_BRANCH_NAME', 'master')
30+
container_registry := "ghcr.io/" + git_org_name + "/"
31+
pod_accelerator_type := env_var_or_default('POD_ACCELERATOR_TYPE', 'nvidia-tesla-t4')
32+
accelerator_node_selector := "gpu-type=" + pod_accelerator_type
33+
34+
container_type := "dev" # or "app"
35+
container_image := if container_type == "dev" {
36+
"pyrovelocitygpu"
37+
} else if container_type == "app" {
38+
"pyrovelocityapp"
39+
} else {
40+
error("container_type must be either 'dev' or 'app'")
41+
}
42+
container_tag := "latest"
43+
44+
pod_source_type := env_var_or_default('POD_SOURCE_TYPE', 'git')
45+
pod_git_provider := env_var_or_default('POD_GIT_PROVIDER', 'github')
46+
pod_disk_size := env_var_or_default('POD_DISK_SIZE', '400Gi')
47+
pod_min_cpu := env_var_or_default('POD_MIN_CPU', '16')
48+
pod_min_mem := env_var_or_default('POD_MIN_MEM', '64Gi')
49+
pod_max_cpu := env_var_or_default('POD_MAX_CPU', '32')
50+
pod_max_mem := env_var_or_default('POD_MAX_MEM', '96Gi')
51+
pod_max_accel := env_var_or_default('POD_MAX_ACCEL', '1')
52+
pod_resources := "requests.cpu=" + pod_min_cpu + ",requests.memory=" + pod_min_mem + ",limits.cpu=" + pod_max_cpu + ",limits.memory=" + pod_max_mem + ",limits.nvidia.com/gpu=" + pod_max_accel
53+
54+
architecture := if arch() == "x86_64" {
55+
"amd64"
56+
} else if arch() == "aarch64" {
57+
"arm64"
58+
} else {
59+
error("unsupported architecture must be amd64 or arm64")
60+
}
61+
62+
opsys := if os() == "macos" {
63+
"darwin"
64+
} else if os() == "linux" {
65+
"linux"
66+
} else {
67+
error("unsupported operating system must be darwin or linux")
68+
}
69+
70+
devpod_release := "latest" # or "v0.3.7" or "v0.4.0-alpha.4"
71+
72+
devpod_binary_url := if devpod_release == "latest" {
73+
"https://github.com/loft-sh/devpod/releases/latest/download/devpod-" + opsys + "-" + architecture
74+
} else {
75+
"https://github.com/loft-sh/devpod/releases/download/" + devpod_release + "/devpod-" + opsys + "-" + architecture
76+
}
77+
78+
# Install devpod (check/set: devpod_release)
79+
[unix]
80+
install-devpod:
81+
curl -L -o devpod {{devpod_binary_url}} && \
82+
sudo install -c -m 0755 devpod /usr/local/bin && \
83+
rm -f devpod
84+
which devpod
85+
devpod version
86+
87+
# Print devpod info
88+
devpod:
89+
devpod version && echo
90+
devpod context list
91+
devpod provider list
92+
devpod list
93+
94+
# Install and use devpod kubernetes provider
95+
provider:
96+
devpod provider add kubernetes --silent || true \
97+
&& devpod provider use kubernetes
98+
99+
# Run latest container_image in current kube context
100+
pod:
101+
devpod up \
102+
--debug \
103+
--devcontainer-image {{container_registry}}{{container_image}}:{{container_tag}} \
104+
--provider kubernetes \
105+
--ide vscode \
106+
--open-ide \
107+
--source {{pod_source_type}}:https://{{pod_git_provider}}.com/{{git_username}}/{{git_repo_name}} \
108+
--provider-option DISK_SIZE={{pod_disk_size}} \
109+
--provider-option NODE_SELECTOR={{accelerator_node_selector}} \
110+
--provider-option RESOURCES={{pod_resources}} \
111+
{{container_image}}
112+
113+
# Stop devpod (check container_image hasn't changed with -n)
114+
stop:
115+
devpod stop {{container_image}}
116+
117+
# Delete devpod (check container_image hasn't changed with -n)
118+
delete:
119+
devpod delete {{container_image}}

0 commit comments

Comments
 (0)