From 0cd266bcd17046c8d308403d4f5d28118a57046e Mon Sep 17 00:00:00 2001 From: Prashant Date: Sat, 16 Nov 2024 15:35:40 +0000 Subject: [PATCH] GPUDirect Storage kernel driver (nvidia-fs) ver-2.24.2 commit --- ChangeLog | 2 ++ src/GDS_VERSION | 2 +- src/Makefile | 8 ++++++++ src/dkms.conf | 13 ++++++++----- src/nvfs-pci.c | 1 + src/nvfs-pci.h | 12 ++++++++---- src/nvfs-vers.h | 4 ++-- 7 files changed, 30 insertions(+), 12 deletions(-) diff --git a/ChangeLog b/ChangeLog index ffeb9bd..1394498 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,5 @@ +nvidia-fs (2.24.2) RELEASE; urgency=low + * Provide support for more rdma devices nvidia-fs (2.23.1) RELEASE; urgency=low * Use persistent_p2p APIs >= nvidia driver version 555 * Fix bug in copy_to_uer through batch API diff --git a/src/GDS_VERSION b/src/GDS_VERSION index 73a567b..902e489 100644 --- a/src/GDS_VERSION +++ b/src/GDS_VERSION @@ -1 +1 @@ -1.12.0.25 +1.13.0.10 diff --git a/src/Makefile b/src/Makefile index c527be9..c66c5bc 100644 --- a/src/Makefile +++ b/src/Makefile @@ -30,6 +30,14 @@ endif KBUILD_MODPOST_WARN=1 obj-m += nvidia-fs.o +# check if variables are defined in the environment +ifneq ($(origin NVFS_MAX_PEER_DEVS), undefined) +ifneq ($(origin NVFS_MAX_PCI_DEPTH), undefined) + ccflags-y += -DNVFS_MAX_PEER_DEVS=$(NVFS_MAX_PEER_DEVS) -DNVFS_MAX_PCI_DEPTH=$(NVFS_MAX_PCI_DEPTH) +endif +endif + + ARCH ?= $(shell uname -m) ifeq ($(ARCH),aarch64) diff --git a/src/dkms.conf b/src/dkms.conf index a9cf007..c7361d8 100644 --- a/src/dkms.conf +++ b/src/dkms.conf @@ -18,6 +18,7 @@ # FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER # DEALINGS IN THE SOFTWARE. +#!/bin/bash # DKMS module name and version PACKAGE_NAME="nvidia-fs" #please update when GDS_VERSION is updated @@ -25,16 +26,18 @@ PACKAGE_VERSION="2.7" kernelver=${kernelver:-$(uname -r)} kernel_source_dir=${kernel_source_dir:-/lib/modules/$kernelver/build} +NVFS_MAX_PEER_DEVS=${NVFS_MAX_PEER_DEVS:-64} +NVFS_MAX_PCI_DEPTH=${NVFS_MAX_PCI_DEPTH:-16} ofedver="" check_ofed=$(which ofed_info) if [ $? -eq 0 ]; then ofed_version=$(ofed_info -s) if [ $? -eq 0 ]; then - testofed_version=$(echo $ofed_version | cut -d '-' -f 2) - major_ver=$(echo $testofed_version | cut -d '.' -f 1) - minor_ver=$(echo $testofed_version | cut -d '.' -f 2) - if [ $testofed_version == "4.6" ] || [ $testofed_version == "4.7" ] || [ $major_ver -gt "4" ]; then + testofed_version=$(echo "$ofed_version" | cut -d '-' -f 2) + major_ver=$(echo "$testofed_version" | cut -d '.' -f 1) + minor_ver=$(echo "$testofed_version" | cut -d '.' -f 2) + if [ "$testofed_version" == "4.6" ] || [ "$testofed_version" == "4.7" ] || [[ "$major_ver" =~ ^[0-9]+$ && "$major_ver" -gt 4 ]]; then check=$(cat /proc/kallsyms | grep -w "__ksymtab_ib_register_peer_memory_client") if [ $? -eq 0 ]; then ofedver=$testofed_version @@ -49,7 +52,7 @@ fi BUILT_MODULE_NAME[0]="nvidia-fs" DEST_MODULE_LOCATION[0]="/kernel/fs/nvidia-fs/" BUILD_DEPENDS[0]="nvidia" -MAKE[0]="'make' -j32 KVER=${kernelver} IGNORE_CC_MISMATCH='1'" +MAKE[0]="NVFS_MAX_PEER_DEVS=${NVFS_MAX_PEER_DEVS} NVFS_MAX_PCI_DEPTH=${NVFS_MAX_PCI_DEPTH} 'make' -j32 KVER=${kernelver} IGNORE_CC_MISMATCH='1'" REMAKE_INITRD=yes # Cleanup command-line diff --git a/src/nvfs-pci.c b/src/nvfs-pci.c index 3ced73b..9934655 100644 --- a/src/nvfs-pci.c +++ b/src/nvfs-pci.c @@ -643,6 +643,7 @@ static void nvfs_get_pci_gpu2peer_distance(void) { * @returns : none */ void nvfs_fill_gpu2peer_distance_table_once(void) { + pr_info("max_peer_devs : %u and max_pci_depth : %u\n", MAX_PEER_DEVS, MAX_PCI_DEPTH); memset ((u8 *)gpu_bdf_map, 0, sizeof(gpu_bdf_map)); memset ((u8 *)peer_bdf_map, 0, sizeof(peer_bdf_map)); memset ((u8 *)gpu_info_table, 0, sizeof(gpu_info_table)); diff --git a/src/nvfs-pci.h b/src/nvfs-pci.h index 13eeb99..77820e4 100644 --- a/src/nvfs-pci.h +++ b/src/nvfs-pci.h @@ -41,11 +41,15 @@ (uint32_t)((uint64_t)(pci_info) >> 32), \ ((uint16_t)(pci_info) >> 8), PCI_SLOT((uint8_t)(pci_info)), PCI_FUNC((uint8_t)(pci_info)) -#define MAX_GPU_DEVS 64U // On DGX-2, 16 GPUS - -#define MAX_PEER_DEVS 64U // On DGX-2, 8 IB Ports +#if defined(NVFS_MAX_PEER_DEVS) && defined(NVFS_MAX_PCI_DEPTH) + #define MAX_PEER_DEVS NVFS_MAX_PEER_DEVS + #define MAX_PCI_DEPTH NVFS_MAX_PCI_DEPTH +#else + #define MAX_PEER_DEVS 64U // On DGX-2, 8 IB Ports + #define MAX_PCI_DEPTH 16U // On DGX-2, max-depth 5 +#endif -#define MAX_PCI_DEPTH 16U // On DGX-2, max-depth 5 +#define MAX_GPU_DEVS 64U // On DGX-2, 16 GPUS // proc limit for pci distance under same Port #define PROC_LIMIT_PCI_DISTANCE_COMMONRP (2 * MAX_PCI_DEPTH) diff --git a/src/nvfs-vers.h b/src/nvfs-vers.h index 76bff4f..5b569a1 100644 --- a/src/nvfs-vers.h +++ b/src/nvfs-vers.h @@ -26,10 +26,10 @@ #define NVFS_DRIVER_MAJOR_VERSION 2 //2-bytes -#define NVFS_DRIVER_MINOR_VERSION 23 //2-bytes +#define NVFS_DRIVER_MINOR_VERSION 24 //2-bytes // template for build version -#define NVFS_DRIVER_PATCH_VERSION 1 +#define NVFS_DRIVER_PATCH_VERSION 2 static inline unsigned int nvfs_driver_version(void) { return (NVFS_DRIVER_MAJOR_VERSION << 16) | NVFS_DRIVER_MINOR_VERSION;