Skip to content
This repository was archived by the owner on Oct 27, 2023. It is now read-only.

Commit d49fbd7

Browse files
RenaudWasTakenRenaudWasTaken
committed
Merge branch 'runtimespec' into 'master'
Add initial runtime container See merge request nvidia/container-toolkit/nvidia-container-runtime!16
2 parents 7e53f88 + e03cdeb commit d49fbd7

File tree

7 files changed

+307
-0
lines changed

7 files changed

+307
-0
lines changed

‎runtimeconfig/Makefile

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
# Copyright (c) 2018-2019, NVIDIA CORPORATION. All rights reserved.
2+
3+
.PHONY: all build builder test
4+
.DEFAULT_GOAL := all
5+
6+
7+
##### Global variables #####
8+
9+
DOCKERFILE ?= $(CURDIR)/docker/Dockerfile
10+
DOCKERDEVEL ?= $(CURDIR)/docker/Dockerfile.builder
11+
IMAGE ?= nvidia/container-toolkit:docker19.03
12+
BUILDER ?= nvidia/container-toolkit:builder
13+
14+
15+
##### Public rules #####
16+
all: build
17+
18+
build:
19+
docker build -f $(DOCKERFILE) -t $(IMAGE) .
20+
21+
builder:
22+
docker build -f $(DOCKERDEVEL) -t $(BUILDER) .
23+
24+
test: build
25+
$(CURDIR)/test/docker_test.sh $(CURDIR)/shared $(IMAGE)

‎runtimeconfig/docker/Dockerfile

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
From ubuntu:18.04
2+
3+
RUN apt-get update && apt-get install -y --no-install-recommends \
4+
apt-transport-https \
5+
ca-certificates \
6+
curl \
7+
gnupg2 \
8+
jq \
9+
moreutils \
10+
software-properties-common
11+
12+
RUN curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - && \
13+
curl -s -L https://nvidia.github.io/nvidia-docker/gpgkey | apt-key add - && \
14+
curl -s -L https://nvidia.github.io/nvidia-docker/ubuntu18.04/nvidia-docker.list | \
15+
tee /etc/apt/sources.list.d/nvidia-docker.list && \
16+
add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu \
17+
$(lsb_release -cs) stable"
18+
19+
RUN apt-get update && apt-get install -y --no-install-recommends \
20+
docker-ce-cli \
21+
nvidia-container-runtime
22+
23+
WORKDIR /work
24+
25+
COPY src/. .
26+
RUN cp /etc/nvidia-container-runtime/config.toml ./
27+
28+
RUN chmod +x /work/run.sh
29+
30+
CMD ["bash"]

‎runtimeconfig/docker/Dockerfile.builder

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
From docker:stable-dind
2+
3+
RUN apk add --update make bash
4+
5+
CMD ["bash"]

‎runtimeconfig/src/common.sh

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
#! /bin/bash
2+
# Copyright (c) 2018-2019, NVIDIA CORPORATION. All rights reserved.
3+
4+
# shellcheck disable=SC2015
5+
[ -t 2 ] && readonly LOG_TTY=1 || readonly LOG_NO_TTY=1
6+
7+
if [ "${LOG_TTY-0}" -eq 1 ] && [ "$(tput colors)" -ge 15 ]; then
8+
readonly FMT_CLEAR=$(tput sgr0)
9+
readonly FMT_BOLD=$(tput bold)
10+
readonly FMT_RED=$(tput setaf 1)
11+
readonly FMT_YELLOW=$(tput setaf 3)
12+
readonly FMT_BLUE=$(tput setaf 12)
13+
fi
14+
15+
log() {
16+
local -r level="$1"; shift
17+
local -r message="$*"
18+
19+
local fmt_on="${FMT_CLEAR-}"
20+
local -r fmt_off="${FMT_CLEAR-}"
21+
22+
case "${level}" in
23+
INFO) fmt_on="${FMT_BLUE-}" ;;
24+
WARN) fmt_on="${FMT_YELLOW-}" ;;
25+
ERROR) fmt_on="${FMT_RED-}" ;;
26+
esac
27+
printf "%s[%s]%s %b\n" "${fmt_on}" "${level}" "${fmt_off}" "${message}" >&2
28+
}
29+

‎runtimeconfig/src/docker.sh

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,75 @@
1+
#! /bin/bash
2+
# Copyright (c) 2018-2019, NVIDIA CORPORATION. All rights reserved.
3+
4+
docker::info() {
5+
local -r docker_socket="${1:-unix:///var/run/docker.socket}"
6+
7+
# Docker in Docker has a startup race
8+
for i in $(seq 1 5); do
9+
# Calling in a subshell so that we can recover from a failure
10+
if [[ ! $(docker -H "${docker_socket}" info -f '{{json .Runtimes}}') ]]; then
11+
sleep 2
12+
continue
13+
fi
14+
15+
docker -H "${docker_socket}" info -f '{{json .Runtimes}}' | jq '.nvidia.path'
16+
return
17+
done
18+
exit 1
19+
}
20+
21+
# Echo an empty config if the config file doesn't exist
22+
docker::daemon_config() {
23+
local -r daemon_file="${1:-"/etc/docker/daemon.json"}"
24+
([[ -f "${daemon_file}" ]] && cat "${daemon_file}") || echo {}
25+
}
26+
27+
docker::refresh_configuration() {
28+
log INFO "Refreshing the docker daemon configuration"
29+
pkill -SIGHUP dockerd
30+
}
31+
32+
docker::update_config_file() {
33+
local -r destination="${1:-/run/nvidia}"
34+
local -r nvcr="${destination}/nvidia-container-runtime"
35+
36+
local config_json
37+
IFS='' read -r config_json
38+
39+
echo "${config_json}" | \
40+
jq -r ".runtimes += {\"nvidia\": {\"path\": \"${nvcr}\"}}" | \
41+
jq -r '. += {"default-runtime": "nvidia"}'
42+
}
43+
44+
docker::ensure_prerequisites() {
45+
# Ensure that the docker config path exists
46+
if [[ ! -d "/etc/docker" ]]; then
47+
log ERROR "Docker directory doesn't exist in container"
48+
log ERROR "Ensure that you have correctly mounted the docker directoy"
49+
exit 1
50+
fi
51+
52+
mount | grep /etc/docker
53+
if [[ ! $? ]]; then
54+
log ERROR "Docker directory isn't mounted in container"
55+
log ERROR "Ensure that you have correctly mounted the docker directoy"
56+
exit 1
57+
fi
58+
}
59+
60+
docker::setup() {
61+
local -r destination="${1:-/run/nvidia}"
62+
log INFO "Setting up the configuration for the docker daemon"
63+
64+
docker::ensure_prerequisites
65+
66+
log INFO "current config: $(docker::daemon_config)"
67+
68+
# Append the nvidia runtime to the docker daemon's configuration
69+
# We use sponge here because the input file is the output file
70+
config=$(docker::daemon_config | docker::update_config_file "${destination}")
71+
echo "${config}" > /etc/docker/daemon.json
72+
73+
log INFO "after: $(docker::daemon_config | jq .)"
74+
docker::refresh_configuration
75+
}

‎runtimeconfig/src/run.sh

Lines changed: 83 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,83 @@
1+
#! /bin/bash
2+
# Copyright (c) 2018-2019, NVIDIA CORPORATION. All rights reserved.
3+
4+
set -euxo pipefail
5+
shopt -s lastpipe
6+
7+
source "common.sh"
8+
source "docker.sh"
9+
10+
install_nvidia_container_runtime_toolkit() {
11+
log INFO "Installing the NVIDIA Container Runtime Toolkit"
12+
13+
local -r destination="${1:-/run/nvidia}"
14+
local -a packages=("/usr/bin/nvidia-container-runtime" \
15+
"/usr/bin/nvidia-container-toolkit" \
16+
"/usr/bin/nvidia-container-cli" \
17+
"/etc/nvidia-container-runtime/config.toml" \
18+
"/usr/lib/x86_64-linux-gnu/libnvidia-container.so.1")
19+
20+
# TODO workaround until we fix the runtime requiring this
21+
# directory and file to exist at that location
22+
cp ./config.toml /etc/nvidia-container-runtime
23+
24+
# Bash variables starts at 0
25+
# ZSH variables starts at 1
26+
for ((i=0; i < ${#packages[@]}; i++)); do
27+
packages[$i]=$(readlink -f ${packages[$i]})
28+
done
29+
30+
if [[ ! -d "${destination}" ]]; then
31+
log ERROR "Destination directory doesn't exist in container"
32+
log ERROR "Ensure that you have correctly mounted the destination directoy"
33+
exit 1
34+
fi
35+
36+
cp "${packages[@]}" "${destination}"
37+
38+
# Setup links to the real binaries to ensure that variables and configs
39+
# are pointing to the right path
40+
mv "${destination}/nvidia-container-toolkit" \
41+
"${destination}/nvidia-container-toolkit.real"
42+
mv "${destination}/nvidia-container-runtime" \
43+
"${destination}/nvidia-container-runtime.real"
44+
45+
46+
# Setup aliases so as to ensure that the path is correctly set
47+
cat <<- EOF > ${destination}/nvidia-container-toolkit
48+
#! /bin/sh
49+
LD_LIBRARY_PATH="${destination}" \
50+
PATH="\$PATH:${destination}" \
51+
${destination}/nvidia-container-toolkit.real \
52+
-config "${destination}/config.toml" \
53+
\$@
54+
EOF
55+
56+
cat <<- EOF > ${destination}/nvidia-container-runtime
57+
#! /bin/sh
58+
LD_LIBRARY_PATH="${destination}" \
59+
PATH="\$PATH:${destination}" \
60+
${destination}/nvidia-container-runtime.real \
61+
\$@
62+
EOF
63+
64+
# Make sure that the alias files are executable
65+
chmod +x "${destination}/nvidia-container-toolkit"
66+
chmod +x "${destination}/nvidia-container-runtime"
67+
}
68+
69+
main() {
70+
local -r destination="${1:-/run/nvidia}"
71+
local -r docker_socket="${2:-/var/run/docker.socket}"
72+
local -r nvidia_runtime="$(docker::info ${docker_socket})"
73+
74+
if [[ "${nvidia_runtime}" = "${destination}/nvidia-container-runtime" ]]; then
75+
exit 0
76+
fi
77+
78+
install_nvidia_container_runtime_toolkit "${destination}"
79+
docker::setup "${destination}"
80+
echo "docker info: $(docker::info ${docker_socket})"
81+
}
82+
83+
main "$@"

‎runtimeconfig/test/docker_test.sh

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
#! /bin/bash
2+
# Copyright (c) 2018-2019, NVIDIA CORPORATION. All rights reserved.
3+
4+
set -eEux
5+
6+
readonly dind_name="nvidia-container-runtime-installer"
7+
8+
# TODO move rm -rf shared to cleanup
9+
testing::cleanup() {
10+
docker kill "${dind_name}" || true &> /dev/null
11+
docker rm "${dind_name}" || true &> /dev/null
12+
13+
return
14+
}
15+
16+
testing::setup() {
17+
local shared_dir=${1:-"./shared"}
18+
19+
rm -rf "${shared_dir}" || true
20+
21+
mkdir -p "${shared_dir}"
22+
mkdir -p "${shared_dir}"/etc/docker
23+
mkdir -p "${shared_dir}"/run/nvidia
24+
mkdir -p "${shared_dir}"/etc/nvidia-container-runtime
25+
}
26+
27+
testing::main() {
28+
local shared_dir="${1:-"./shared"}"
29+
local image="${2:-"nvidia/container-toolkit:docker19.03"}"
30+
31+
testing::setup "${shared_dir}"
32+
33+
# Docker creates /etc/docker when starting
34+
# by default there isn't any config in this directory (even after the daemon starts)
35+
docker run --privileged \
36+
-v "${shared_dir}/etc/docker:/etc/docker" \
37+
-v "${shared_dir}/run/nvidia:/run/nvidia" \
38+
-v "${shared_dir}/etc/nvidia-container-runtime:/etc/nvidia-container-runtime" \
39+
--name "${dind_name}" -d docker:stable-dind -H unix://run/nvidia/docker.sock
40+
41+
# Share the volumes so that we can edit the config file and point to the new runtime
42+
# Share the pid so that we can ask docker to reload its config
43+
docker run -it --privileged \
44+
--volumes-from "${dind_name}" \
45+
--pid "container:${dind_name}" \
46+
"${image}" \
47+
bash -x -c "/work/run.sh /run/nvidia unix:///run/nvidia/docker.sock"
48+
49+
docker run -it --privileged \
50+
--volumes-from "${dind_name}" \
51+
alpine:latest chmod 766 /etc/docker /run/nvidia /etc/nvidia-container-runtime
52+
53+
testing::cleanup
54+
rm -rf "${shared_dir}" || true
55+
}
56+
57+
trap testing::cleanup ERR
58+
59+
testing::cleanup
60+
testing::main "$@"

0 commit comments

Comments
 (0)