GPUDirect Storage kernel driver (nvidia-fs) ver-2.24.2 commit

ChangeLog

@@ -1,3 +1,5 @@
+nvidia-fs (2.24.2) RELEASE; urgency=low
+  * Provide support for more rdma devices
 nvidia-fs (2.23.1) RELEASE; urgency=low
   * Use persistent_p2p APIs >= nvidia driver version 555
   * Fix bug in copy_to_uer through batch API

src/GDS_VERSION

@@ -1 +1 @@
-1.12.0.25
+1.13.0.10

src/Makefile

@@ -30,6 +30,14 @@ endif
 KBUILD_MODPOST_WARN=1
 obj-m += nvidia-fs.o
 
+# check if variables are defined in the environment
+ifneq ($(origin NVFS_MAX_PEER_DEVS), undefined)
+ifneq ($(origin NVFS_MAX_PCI_DEPTH), undefined)
+    ccflags-y += -DNVFS_MAX_PEER_DEVS=$(NVFS_MAX_PEER_DEVS) -DNVFS_MAX_PCI_DEPTH=$(NVFS_MAX_PCI_DEPTH)
+endif
+endif
+
+
 ARCH ?= $(shell uname -m)
 
 ifeq ($(ARCH),aarch64)

src/dkms.conf

@@ -18,23 +18,26 @@
 # FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
 # DEALINGS IN THE SOFTWARE.
 
+#!/bin/bash
 # DKMS module name and version
 PACKAGE_NAME="nvidia-fs"
 #please update when GDS_VERSION is updated
 PACKAGE_VERSION="2.7"
 
 kernelver=${kernelver:-$(uname -r)}
 kernel_source_dir=${kernel_source_dir:-/lib/modules/$kernelver/build}
+NVFS_MAX_PEER_DEVS=${NVFS_MAX_PEER_DEVS:-64}
+NVFS_MAX_PCI_DEPTH=${NVFS_MAX_PCI_DEPTH:-16}
 
 ofedver=""
 check_ofed=$(which ofed_info)
 if [ $? -eq 0 ]; then
 	ofed_version=$(ofed_info -s)
 	if [ $? -eq 0 ]; then
-		testofed_version=$(echo $ofed_version | cut -d '-' -f 2)
-                major_ver=$(echo $testofed_version | cut -d '.' -f 1)
-                minor_ver=$(echo $testofed_version | cut -d '.' -f 2)
-                if [ $testofed_version == "4.6" ] || [ $testofed_version == "4.7" ] || [ $major_ver -gt "4" ]; then
+		testofed_version=$(echo "$ofed_version" | cut -d '-' -f 2)
+                major_ver=$(echo "$testofed_version" | cut -d '.' -f 1)
+                minor_ver=$(echo "$testofed_version" | cut -d '.' -f 2)
+                if [ "$testofed_version" == "4.6" ] || [ "$testofed_version" == "4.7" ] || [[ "$major_ver" =~ ^[0-9]+$ && "$major_ver" -gt 4 ]]; then
                         check=$(cat /proc/kallsyms | grep -w "__ksymtab_ib_register_peer_memory_client")
                         if [ $? -eq 0 ]; then
                                 ofedver=$testofed_version
@@ -49,7 +52,7 @@ fi
 BUILT_MODULE_NAME[0]="nvidia-fs"
 DEST_MODULE_LOCATION[0]="/kernel/fs/nvidia-fs/"
 BUILD_DEPENDS[0]="nvidia"
-MAKE[0]="'make' -j32 KVER=${kernelver} IGNORE_CC_MISMATCH='1'"
+MAKE[0]="NVFS_MAX_PEER_DEVS=${NVFS_MAX_PEER_DEVS} NVFS_MAX_PCI_DEPTH=${NVFS_MAX_PCI_DEPTH} 'make' -j32 KVER=${kernelver} IGNORE_CC_MISMATCH='1'"
 REMAKE_INITRD=yes
 
 # Cleanup command-line

src/nvfs-pci.c

@@ -643,6 +643,7 @@ static void nvfs_get_pci_gpu2peer_distance(void) {
  *  @returns : none
  */
 void nvfs_fill_gpu2peer_distance_table_once(void) {
+	pr_info("max_peer_devs : %u and max_pci_depth : %u\n", MAX_PEER_DEVS, MAX_PCI_DEPTH);
 	memset ((u8 *)gpu_bdf_map, 0, sizeof(gpu_bdf_map));
 	memset ((u8 *)peer_bdf_map, 0, sizeof(peer_bdf_map));
 	memset ((u8 *)gpu_info_table, 0, sizeof(gpu_info_table));

src/nvfs-pci.h

@@ -41,11 +41,15 @@
 	(uint32_t)((uint64_t)(pci_info) >> 32), \
 	((uint16_t)(pci_info) >> 8), PCI_SLOT((uint8_t)(pci_info)), PCI_FUNC((uint8_t)(pci_info))
 
-#define MAX_GPU_DEVS    64U // On DGX-2, 16 GPUS
-
-#define MAX_PEER_DEVS   64U // On DGX-2, 8 IB Ports
+#if defined(NVFS_MAX_PEER_DEVS) && defined(NVFS_MAX_PCI_DEPTH)
+       #define MAX_PEER_DEVS   NVFS_MAX_PEER_DEVS
+       #define MAX_PCI_DEPTH   NVFS_MAX_PCI_DEPTH
+#else
+       #define MAX_PEER_DEVS   64U // On DGX-2, 8 IB Ports
+       #define MAX_PCI_DEPTH   16U // On DGX-2, max-depth 5
+#endif
 
-#define MAX_PCI_DEPTH   16U // On DGX-2, max-depth 5
+#define MAX_GPU_DEVS    64U // On DGX-2, 16 GPUS
 
 // proc limit for pci distance under same Port
 #define PROC_LIMIT_PCI_DISTANCE_COMMONRP (2 * MAX_PCI_DEPTH)

src/nvfs-vers.h

@@ -26,10 +26,10 @@
 
 #define NVFS_DRIVER_MAJOR_VERSION   2 //2-bytes
 
-#define NVFS_DRIVER_MINOR_VERSION   23 //2-bytes
+#define NVFS_DRIVER_MINOR_VERSION   24 //2-bytes
 
 // template for build version
-#define NVFS_DRIVER_PATCH_VERSION  1
+#define NVFS_DRIVER_PATCH_VERSION  2
 
 static inline unsigned int nvfs_driver_version(void) {
     return (NVFS_DRIVER_MAJOR_VERSION << 16) | NVFS_DRIVER_MINOR_VERSION;