diff --git a/ubuntu22.04/Dockerfile b/ubuntu22.04/Dockerfile index 5ef5041a..58eccf3b 100644 --- a/ubuntu22.04/Dockerfile +++ b/ubuntu22.04/Dockerfile @@ -86,6 +86,16 @@ RUN if [ "$DRIVER_TYPE" != "vgpu" ] && [ "$TARGETARCH" != "arm64" ]; then \ apt-get install -y --no-install-recommends nvidia-fabricmanager-${DRIVER_BRANCH}=${DRIVER_VERSION}-1 \ libnvidia-nscq-${DRIVER_BRANCH}=${DRIVER_VERSION}-1; fi +# For now install imex only on arm64 and R560+ +# https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/sbsa/nvidia-imex_560.28.03-1_arm64.deb +ENV IMEX_DRIVER_VERSION=565 +RUN if [ "$(printf '%s\n' "$IMEX_DRIVER_VERSION" "$DRIVER_VERSION" | sort -V | head -n1)" = "$IMEX_DRIVER_VERSION" ] && [ "$TARGETARCH" = "arm64" ]; then \ + curl -fsSL -o /tmp/nvidia-imex-${IMEX_DRIVER_VERSION}_${DRIVER_VERSION}-1_${TARGETARCH}.deb https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/sbsa/nvidia-imex-${IMEX_DRIVER_VERSION}_${DRIVER_VERSION}-1_${TARGETARCH}.deb && \ + dpkg -i /tmp/nvidia-imex-${IMEX_DRIVER_VERSION}_${DRIVER_VERSION}-1_${TARGETARCH}.deb && \ + nvidia-imex --version && \ + ls /etc/nvidia-imex && \ + rm -rf /tmp/nvidia-imex_${IMEX_DRIVER_VERSION}_${DRIVER_VERSION}-1_${TARGETARCH}.deb; fi + WORKDIR /drivers ARG PUBLIC_KEY=empty diff --git a/ubuntu22.04/nvidia-driver b/ubuntu22.04/nvidia-driver index aedeeea2..d2916f0f 100755 --- a/ubuntu22.04/nvidia-driver +++ b/ubuntu22.04/nvidia-driver @@ -187,6 +187,11 @@ _create_driver_package() ( mv ${pkg_name} precompiled ) +# Check if imex node config is present +_assert_imex_node_config() { + [ -f /imex/nodes_config.cfg ] || return 1 +} + _assert_nvswitch_system() { [ -d /proc/driver/nvidia-nvswitch/devices ] || return 1 if [ -z "$(ls -A /proc/driver/nvidia-nvswitch/devices)" ]; then @@ -326,6 +331,14 @@ _load_driver() { nv-fabricmanager -c /usr/share/nvidia/nvswitch/fabricmanager.cfg fi + if _assert_imex_node_config; then + echo "Copying imex nodes_config.cfg to /etc/nvidia-imex" + # folder is created during imex installation + cp /imex/nodes_config.cfg /etc/nvidia-imex/nodes_config.cfg + echo "Starting NVIDIA imex daemon..." + /usr/bin/nvidia-imex -c /etc/nvidia-imex/config.cfg + fi + return 0 } @@ -383,6 +396,13 @@ _unload_driver() { fi fi + # stop nvidia-imex daemon + local imex_pid=$(pgrep -f /usr/bin/nvidia-imex) + if [ -n "${imex_pid}" ]; then + echo "Stopping NVIDIA imex daemon..." + kill -SIGTERM "${imex_pid}" + fi + echo "Unloading NVIDIA driver kernel modules..." if [ -f /sys/module/nvidia_modeset/refcnt ]; then nvidia_modeset_refs=$(< /sys/module/nvidia_modeset/refcnt) diff --git a/ubuntu22.04/precompiled/Dockerfile b/ubuntu22.04/precompiled/Dockerfile index f582b73d..2bbabdc8 100644 --- a/ubuntu22.04/precompiled/Dockerfile +++ b/ubuntu22.04/precompiled/Dockerfile @@ -59,6 +59,16 @@ RUN apt-get update && apt-get install -y --no-install-recommends nvidia-driver-$ xserver-xorg-video-nvidia-${DRIVER_BRANCH}-server && \ rm -rf /var/lib/apt/lists/*; +# For now install imex only on arm64 and R560+ +# https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/sbsa/nvidia-imex_560.28.03-1_arm64.deb +ENV IMEX_DRIVER_VERSION=565 +RUN if [ "$(printf '%s\n' "$IMEX_DRIVER_VERSION" "$DRIVER_VERSION" | sort -V | head -n1)" = "$IMEX_DRIVER_VERSION" ] && [ "$TARGETARCH" = "arm64" ]; then \ + curl -fsSL -o /tmp/nvidia-imex-${IMEX_DRIVER_VERSION}_${DRIVER_VERSION}-1_${TARGETARCH}.deb https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/sbsa/nvidia-imex-${IMEX_DRIVER_VERSION}_${DRIVER_VERSION}-1_${TARGETARCH}.deb && \ + dpkg -i /tmp/nvidia-imex-${IMEX_DRIVER_VERSION}_${DRIVER_VERSION}-1_${TARGETARCH}.deb && \ + nvidia-imex --version && \ + ls /etc/nvidia-imex && \ + rm -rf /tmp/nvidia-imex_${IMEX_DRIVER_VERSION}_${DRIVER_VERSION}-1_${TARGETARCH}.deb; fi + # update pkg cache and download pkgs for driver module installation during runtime. # this is done to avoid shipping .ko files. # avoid cleaning the cache after this to retain these packages during runtime. diff --git a/ubuntu22.04/precompiled/nvidia-driver b/ubuntu22.04/precompiled/nvidia-driver index 97369be2..6166e64b 100755 --- a/ubuntu22.04/precompiled/nvidia-driver +++ b/ubuntu22.04/precompiled/nvidia-driver @@ -22,6 +22,10 @@ _update_package_cache() { fi } +_assert_imex_node_config() { + [ -f /imex/nodes_config.cfg ] || return 1 +} + _assert_nvswitch_system() { [ -d /proc/driver/nvidia-nvswitch/devices ] || return 1 if [ -z "$(ls -A /proc/driver/nvidia-nvswitch/devices)" ]; then @@ -129,6 +133,14 @@ _load_driver() { nv-fabricmanager -c /usr/share/nvidia/nvswitch/fabricmanager.cfg fi + if _assert_imex_node_config; then + echo "Copying imex nodes_config.cfg to /etc/nvidia-imex" + # folder is created during imex installation + cp /imex/nodes_config.cfg /etc/nvidia-imex/nodes_config.cfg + echo "Starting NVIDIA imex daemon..." + /usr/bin/nvidia-imex -c /etc/nvidia-imex/config.cfg + fi + return 0 } @@ -170,6 +182,12 @@ _unload_driver() { return 1 fi fi + # stop nvidia-imex daemon + local imex_pid=$(pgrep -f /usr/bin/nvidia-imex) + if [ -n "${imex_pid}" ]; then + echo "Stopping NVIDIA imex daemon..." + kill -SIGTERM "${imex_pid}" + fi echo "Unloading NVIDIA driver kernel modules..." if [ -f /sys/module/nvidia_drm/refcnt ]; then