Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft: Imex support #149

Open
wants to merge 7 commits into
base: main
Choose a base branch
from
Open

Draft: Imex support #149

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
76faea1
Install and start imex
guptaNswati Oct 24, 2024
aff1d10
Add R560
guptaNswati Oct 24, 2024
3f36077
Fix syntax and driver version
guptaNswati Oct 24, 2024
000e6f8
run imex without systemd
guptaNswati Oct 28, 2024
2cdc307
change nodes_config.cfg path as its conflicting installation
guptaNswati Nov 14, 2024
3785ca3
Merge remote-tracking branch 'origin/main' into imex-support
guptaNswati Nov 14, 2024
ebccc88
update driver version
guptaNswati Nov 14, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions ubuntu22.04/Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,16 @@ RUN if [ "$DRIVER_TYPE" != "vgpu" ] && [ "$TARGETARCH" != "arm64" ]; then \
apt-get install -y --no-install-recommends nvidia-fabricmanager-${DRIVER_BRANCH}=${DRIVER_VERSION}-1 \
libnvidia-nscq-${DRIVER_BRANCH}=${DRIVER_VERSION}-1; fi

# For now install imex only on arm64 and R560+
# https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/sbsa/nvidia-imex_560.28.03-1_arm64.deb
ENV IMEX_DRIVER_VERSION=565
RUN if [ "$(printf '%s\n' "$IMEX_DRIVER_VERSION" "$DRIVER_VERSION" | sort -V | head -n1)" = "$IMEX_DRIVER_VERSION" ] && [ "$TARGETARCH" = "arm64" ]; then \
curl -fsSL -o /tmp/nvidia-imex-${IMEX_DRIVER_VERSION}_${DRIVER_VERSION}-1_${TARGETARCH}.deb https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/sbsa/nvidia-imex-${IMEX_DRIVER_VERSION}_${DRIVER_VERSION}-1_${TARGETARCH}.deb && \
dpkg -i /tmp/nvidia-imex-${IMEX_DRIVER_VERSION}_${DRIVER_VERSION}-1_${TARGETARCH}.deb && \
nvidia-imex --version && \
ls /etc/nvidia-imex && \
rm -rf /tmp/nvidia-imex_${IMEX_DRIVER_VERSION}_${DRIVER_VERSION}-1_${TARGETARCH}.deb; fi

WORKDIR /drivers

ARG PUBLIC_KEY=empty
Expand Down
20 changes: 20 additions & 0 deletions ubuntu22.04/nvidia-driver
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -187,6 +187,11 @@ _create_driver_package() (
mv ${pkg_name} precompiled
)

# Check if imex node config is present
_assert_imex_node_config() {
[ -f /imex/nodes_config.cfg ] || return 1
}

_assert_nvswitch_system() {
[ -d /proc/driver/nvidia-nvswitch/devices ] || return 1
if [ -z "$(ls -A /proc/driver/nvidia-nvswitch/devices)" ]; then
Expand Down Expand Up @@ -326,6 +331,14 @@ _load_driver() {
nv-fabricmanager -c /usr/share/nvidia/nvswitch/fabricmanager.cfg
fi

if _assert_imex_node_config; then
echo "Copying imex nodes_config.cfg to /etc/nvidia-imex"
# folder is created during imex installation
cp /imex/nodes_config.cfg /etc/nvidia-imex/nodes_config.cfg
echo "Starting NVIDIA imex daemon..."
/usr/bin/nvidia-imex -c /etc/nvidia-imex/config.cfg
fi

return 0
}

Expand Down Expand Up @@ -383,6 +396,13 @@ _unload_driver() {
fi
fi

# stop nvidia-imex daemon
local imex_pid=$(pgrep -f /usr/bin/nvidia-imex)
if [ -n "${imex_pid}" ]; then
echo "Stopping NVIDIA imex daemon..."
kill -SIGTERM "${imex_pid}"
fi

echo "Unloading NVIDIA driver kernel modules..."
if [ -f /sys/module/nvidia_modeset/refcnt ]; then
nvidia_modeset_refs=$(< /sys/module/nvidia_modeset/refcnt)
Expand Down
10 changes: 10 additions & 0 deletions ubuntu22.04/precompiled/Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,16 @@ RUN apt-get update && apt-get install -y --no-install-recommends nvidia-driver-$
xserver-xorg-video-nvidia-${DRIVER_BRANCH}-server && \
rm -rf /var/lib/apt/lists/*;

# For now install imex only on arm64 and R560+
# https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/sbsa/nvidia-imex_560.28.03-1_arm64.deb
ENV IMEX_DRIVER_VERSION=565
RUN if [ "$(printf '%s\n' "$IMEX_DRIVER_VERSION" "$DRIVER_VERSION" | sort -V | head -n1)" = "$IMEX_DRIVER_VERSION" ] && [ "$TARGETARCH" = "arm64" ]; then \
curl -fsSL -o /tmp/nvidia-imex-${IMEX_DRIVER_VERSION}_${DRIVER_VERSION}-1_${TARGETARCH}.deb https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/sbsa/nvidia-imex-${IMEX_DRIVER_VERSION}_${DRIVER_VERSION}-1_${TARGETARCH}.deb && \
dpkg -i /tmp/nvidia-imex-${IMEX_DRIVER_VERSION}_${DRIVER_VERSION}-1_${TARGETARCH}.deb && \
nvidia-imex --version && \
ls /etc/nvidia-imex && \
rm -rf /tmp/nvidia-imex_${IMEX_DRIVER_VERSION}_${DRIVER_VERSION}-1_${TARGETARCH}.deb; fi

# update pkg cache and download pkgs for driver module installation during runtime.
# this is done to avoid shipping .ko files.
# avoid cleaning the cache after this to retain these packages during runtime.
Expand Down
18 changes: 18 additions & 0 deletions ubuntu22.04/precompiled/nvidia-driver
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,10 @@ _update_package_cache() {
fi
}

_assert_imex_node_config() {
[ -f /imex/nodes_config.cfg ] || return 1
}

_assert_nvswitch_system() {
[ -d /proc/driver/nvidia-nvswitch/devices ] || return 1
if [ -z "$(ls -A /proc/driver/nvidia-nvswitch/devices)" ]; then
Expand Down Expand Up @@ -129,6 +133,14 @@ _load_driver() {
nv-fabricmanager -c /usr/share/nvidia/nvswitch/fabricmanager.cfg
fi

if _assert_imex_node_config; then
echo "Copying imex nodes_config.cfg to /etc/nvidia-imex"
# folder is created during imex installation
cp /imex/nodes_config.cfg /etc/nvidia-imex/nodes_config.cfg
echo "Starting NVIDIA imex daemon..."
/usr/bin/nvidia-imex -c /etc/nvidia-imex/config.cfg
fi

return 0
}

Expand Down Expand Up @@ -170,6 +182,12 @@ _unload_driver() {
return 1
fi
fi
# stop nvidia-imex daemon
local imex_pid=$(pgrep -f /usr/bin/nvidia-imex)
if [ -n "${imex_pid}" ]; then
echo "Stopping NVIDIA imex daemon..."
kill -SIGTERM "${imex_pid}"
fi

echo "Unloading NVIDIA driver kernel modules..."
if [ -f /sys/module/nvidia_drm/refcnt ]; then
Expand Down