Adjust performance settings for Tesla V100 GPUs.

install-nvidia-coreos.sh

@@ -18,4 +18,7 @@ echo "NVIDIA Drivers built! Trying to install now...\n";
 /bin/bash nvidia_install.sh $1 $2 $3
 echo "NVIDIA Drivers installed!\n"
 
+cp nvidia-configure-settings.sh /opt/bin
+systemctl start --no-block nvidia-configure-settings.service
+
 cd $org_dir

nvidia-configure-settings.service

@@ -0,0 +1,12 @@
+[Unit]
+Description=NVIDIA Settings Configuration for performance
+Wants=local-fs.target
+StartLimitIntervalSec=0
+
+[Service]
+Type=simple
+StandardOutput=journal+console
+StandardError=journal+console
+Restart=on-failure
+RestartSec=1
+ExecStart=/bin/bash /opt/bin/nvidia-configure-settings.sh

nvidia-configure-settings.sh

@@ -0,0 +1,21 @@
+#!/bin/sh
+set -e
+
+PERSISTENCE_MODE=$(/opt/bin/nvidia-smi --query-gpu=persistence_mode --format=csv | sed -n 2p | awk '{print $1}')
+if [ "$PERSISTENCE_MODE" != "Enabled" ]
+then
+    echo "Setting performance settings requires persistence mode to be enabled. Exiting..."
+    exit -1
+fi
+
+GPU_CHIP_NAME=$(/opt/bin/nvidia-smi --query-gpu=gpu_name --format=csv | sed -n 2p | awk '{print $2}')
+if [ "$GPU_CHIP_NAME" = "V100-SXM2-16GB" ]
+then
+    echo "Detected a Tesla V100-SXM2-16GB chip. Setting clocks as suggested by AWS Documentation for P3 instances."
+    /opt/bin/nvidia-smi -ac 877,1530
+else
+    #Not a Tesla V100 GPU. Try turning off auto-boost at least, given we don't know max clocks...
+    /opt/bin/nvidia-smi --auto-boost-default=0
+fi
+
+echo "NVIDIA performance configuration complete."

nvidia-persistenced.service

@@ -1,14 +1,12 @@
 [Unit]
 Description=NVIDIA Persistence Daemon
 Wants=local-fs.target
-After=nvidia-start.service
 
 [Service]
 Type=forking
+StandardOutput=journal+console
+StandardError=journal+console
+PIDFile=/var/run/nvidia-persistenced/nvidia-persistenced.pid
 Restart=always
-RestartSec=2
 ExecStart=/opt/bin/nvidia-persistenced --user nvidia-persistenced --persistence-mode --verbose
-ExecStopPost=/bin/rm -rf /var/run/nvidia-persistenced
-
-[Install]
-WantedBy=multi-user.target
+ExecStopPost=/bin/rm -rf /var/run/nvidia-persistenced

nvidia-start.service

@@ -3,6 +3,9 @@ Description=Load NVIDIA module
 After=local-fs.target
 
 [Service]
+Type=oneshot
+StandardOutput=journal+console
+StandardError=journal+console
 ExecStart=/opt/bin/nvidia-start.sh
 
 [Install]

nvidia_install.sh

@@ -56,5 +56,4 @@ systemctl daemon-reload
 systemctl enable nvidia-start.service
 systemctl start nvidia-start.service
 
-systemctl enable nvidia-persistenced.service
 systemctl start nvidia-persistenced.service