From fe75b1ba88d070ebc0c6f60dd8eb342307bf07e5 Mon Sep 17 00:00:00 2001 From: Kiran Modukuri Date: Wed, 30 Aug 2023 16:03:07 -0700 Subject: [PATCH] GPUDirect Storage kernel driver (nvidia-fs) ver-2.17.5 commit removed the use of symbol_get APIs for nvidia_p2p_* APIs. --- ChangeLog | 4 + src/Makefile | 2 +- src/nvfs-core.c | 9 --- src/nvfs-p2p.c | 194 ------------------------------------------------ src/nvfs-p2p.h | 45 ++--------- src/nvfs-vers.h | 2 +- 6 files changed, 12 insertions(+), 244 deletions(-) delete mode 100644 src/nvfs-p2p.c diff --git a/ChangeLog b/ChangeLog index 01910f6..9edbe71 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,7 @@ +nvidia-fs (2.17.5) RELEASE; urgency=low + * removed the use of symbol_get APIs for nvidia_p2p_* APIs. nvidia-fs is restricted to NVIDIA UNIX Open Kernel Module for x86_64 + +-- Aug 2023 nvidia-fs (2.17.4) RELEASE; urgency=low * Added support for Grace-Hopper platform with 4k and 64k Page support diff --git a/src/Makefile b/src/Makefile index b0e39a6..c527be9 100644 --- a/src/Makefile +++ b/src/Makefile @@ -44,7 +44,7 @@ ccflags-y += -Wall ccflags-y += -I$(NVIDIA_SRC_DIR) ccflags-y += -I/usr/lib/gcc/x86_64-linux-gnu/7/include/ -nvidia-fs-y = nvfs-core.o nvfs-dma.o nvfs-mmap.o nvfs-pci.o nvfs-proc.o nvfs-mod.o nvfs-kernel-interface.o nvfs-p2p.o +nvidia-fs-y = nvfs-core.o nvfs-dma.o nvfs-mmap.o nvfs-pci.o nvfs-proc.o nvfs-mod.o nvfs-kernel-interface.o nvidia-fs-$(CONFIG_NVFS_STATS) += nvfs-stat.o nvidia-fs-$(CONFIG_FAULT_INJECTION) += nvfs-fault.o GDS_VERSION ?= $(shell cat GDS_VERSION) diff --git a/src/nvfs-core.c b/src/nvfs-core.c index 6c8a533..1985028 100644 --- a/src/nvfs-core.c +++ b/src/nvfs-core.c @@ -1061,17 +1061,9 @@ static int nvfs_open(struct inode *inode, struct file *file) mutex_lock(&nvfs_module_mutex); nvfs_get_ops(); - if(nvfs_nvidia_p2p_init()) { - nvfs_err("Could not load nvidia_p2p* symbols\n"); - nvfs_put_ops(); - ret = -EOPNOTSUPP; - goto out; - } - ret = nvfs_blk_register_dma_ops(); if (ret < 0) { nvfs_err("nvfs modules probe failed with error :%d\n", ret); - nvfs_nvidia_p2p_exit(); nvfs_put_ops(); goto out; } @@ -1089,7 +1081,6 @@ static int nvfs_close(struct inode *inode, struct file *file) nvfs_put_ops(); if (nvfs_count_ops() == 0) { nvfs_blk_unregister_dma_ops(); - nvfs_nvidia_p2p_exit(); nvfs_dbg("Unregistering dma ops and nvidia p2p ops\n"); } mutex_unlock(&nvfs_module_mutex); diff --git a/src/nvfs-p2p.c b/src/nvfs-p2p.c deleted file mode 100644 index 3c50b0a..0000000 --- a/src/nvfs-p2p.c +++ /dev/null @@ -1,194 +0,0 @@ -/* - * Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER - * DEALINGS IN THE SOFTWARE. - */ - -#include -#include - -#include "nvfs-p2p.h" -#include "nv-p2p.h" -#include "nvfs-core.h" - -#ifdef HAVE_MODULE_MUTEX -extern struct mutex module_mutex; -#endif - -static nvidia_p2p_dma_unmap_pages_fptr nvidia_p2p_dma_unmap_pages_p = NULL; -static nvidia_p2p_get_pages_fptr nvidia_p2p_get_pages_p = NULL; -static nvidia_p2p_put_pages_fptr nvidia_p2p_put_pages_p = NULL; -static nvidia_p2p_dma_map_pages_fptr nvidia_p2p_dma_map_pages_p = NULL; -static nvidia_p2p_free_dma_mapping_fptr nvidia_p2p_free_dma_mapping_p = NULL; -static nvidia_p2p_free_page_table_fptr nvidia_p2p_free_page_table_p = NULL; - -static inline void nvfs_nvidia_put_symbols(void) { - if(nvidia_p2p_dma_unmap_pages_p) { - __symbol_put("nvidia_p2p_dma_unmap_pages"); - } - if(nvidia_p2p_get_pages_p) { - __symbol_put("nvidia_p2p_get_pages"); - } - if(nvidia_p2p_put_pages_p) { - __symbol_put("nvidia_p2p_put_pages"); - } - if(nvidia_p2p_dma_map_pages_p) { - __symbol_put("nvidia_p2p_dma_map_pages"); - } - if(nvidia_p2p_free_dma_mapping_p) { - __symbol_put("nvidia_p2p_free_dma_mapping"); - } - if(nvidia_p2p_free_page_table_p) { - __symbol_put("nvidia_p2p_free_page_table"); - } - nvidia_p2p_dma_unmap_pages_p = NULL; - nvidia_p2p_get_pages_p = NULL; - nvidia_p2p_put_pages_p = NULL; - nvidia_p2p_dma_map_pages_p = NULL; - nvidia_p2p_free_dma_mapping_p = NULL; - nvidia_p2p_free_page_table_p = NULL; -} - -int nvfs_nvidia_p2p_init() { - -#ifdef HAVE_MODULE_MUTEX - mutex_lock(&module_mutex); -#endif - if(nvidia_p2p_dma_unmap_pages_p == NULL) { - nvidia_p2p_dma_unmap_pages_p = __symbol_get("nvidia_p2p_dma_unmap_pages"); - if(nvidia_p2p_dma_unmap_pages_p == NULL) { - nvfs_err("Unable to find symbol: nvidia_p2p_dma_unmap_pages \n"); - goto error; - } - } - - if(nvidia_p2p_get_pages_p == NULL) { - nvidia_p2p_get_pages_p = __symbol_get("nvidia_p2p_get_pages"); - if(nvidia_p2p_get_pages_p == NULL) { - nvfs_err("Unable to find symbol: nvidia_p2p_get_pages \n"); - goto error; - } - } - - if(nvidia_p2p_put_pages_p == NULL) { - nvidia_p2p_put_pages_p = __symbol_get("nvidia_p2p_put_pages"); - if(nvidia_p2p_put_pages_p == NULL) { - nvfs_err("Unable to find symbol: nvidia_p2p_put_pages \n"); - goto error; - } - } - - if(nvidia_p2p_dma_map_pages_p == NULL) { - nvidia_p2p_dma_map_pages_p = __symbol_get("nvidia_p2p_dma_map_pages"); - if(nvidia_p2p_dma_map_pages_p == NULL) { - nvfs_err("Unable to find symbol: nvidia_p2p_dma_map_pages \n"); - goto error; - } - } - - if(nvidia_p2p_free_dma_mapping_p == NULL) { - nvidia_p2p_free_dma_mapping_p = __symbol_get("nvidia_p2p_free_dma_mapping"); - if(nvidia_p2p_free_dma_mapping_p == NULL) { - nvfs_err("Unable to find symbol: nvidia_p2p_free_dma_mapping \n"); - goto error; - } - } - - if(nvidia_p2p_free_page_table_p == NULL) { - nvidia_p2p_free_page_table_p = __symbol_get("nvidia_p2p_free_page_table"); - if(nvidia_p2p_free_page_table_p == NULL) { - nvfs_err("Unable to find symbol: nvidia_p2p_free_page_table \n"); - goto error; - } - } -#ifdef HAVE_MODULE_MUTEX - mutex_unlock(&module_mutex); -#endif - return 0; -error: -#ifdef HAVE_MODULE_MUTEX - mutex_unlock(&module_mutex); -#endif - nvfs_nvidia_put_symbols(); - return -1; -} - -void nvfs_nvidia_p2p_exit() { -#ifdef HAVE_MODULE_MUTEX - mutex_lock(&module_mutex); -#endif - nvfs_nvidia_put_symbols(); -#ifdef HAVE_MODULE_MUTEX - mutex_unlock(&module_mutex); -#endif -} - -int nvfs_nvidia_p2p_dma_unmap_pages(struct pci_dev *peer, - struct nvidia_p2p_page_table *page_table, - struct nvidia_p2p_dma_mapping *dma_mapping) { - if(nvidia_p2p_dma_unmap_pages_p) { - return nvidia_p2p_dma_unmap_pages_p(peer, page_table, dma_mapping); - } else { - return -ENOMEM; - } -} -int nvfs_nvidia_p2p_get_pages(uint64_t p2p_token, uint32_t va_space, - uint64_t virtual_address, - uint64_t length, - struct nvidia_p2p_page_table **page_table, - void (*free_callback)(void *data), - void *data) { - if(nvidia_p2p_get_pages_p) { - return nvidia_p2p_get_pages_p(p2p_token, va_space, virtual_address, length, page_table, free_callback, data); - } else { - return -ENOMEM; - } -} -int nvfs_nvidia_p2p_put_pages(uint64_t p2p_token, uint32_t va_space, - uint64_t virtual_address, - struct nvidia_p2p_page_table *page_table) { - if(nvidia_p2p_put_pages_p) { - return nvidia_p2p_put_pages_p(p2p_token, va_space, virtual_address, page_table); - } else { - return -ENOMEM; - } -} -int nvfs_nvidia_p2p_dma_map_pages(struct pci_dev *peer, - struct nvidia_p2p_page_table *page_table, - struct nvidia_p2p_dma_mapping **dma_mapping) { - if(nvidia_p2p_dma_map_pages_p) { - return nvidia_p2p_dma_map_pages_p(peer, page_table, dma_mapping); - } else { - return -ENOMEM; - } -} -int nvfs_nvidia_p2p_free_dma_mapping(struct nvidia_p2p_dma_mapping *dma_mapping) { - if(nvidia_p2p_free_dma_mapping_p) { - return nvidia_p2p_free_dma_mapping_p(dma_mapping); - } else { - return -ENOMEM; - } -} -int nvfs_nvidia_p2p_free_page_table(struct nvidia_p2p_page_table *page_table) { - if(nvidia_p2p_free_page_table_p) { - return nvidia_p2p_free_page_table_p(page_table); - } else { - return -ENOMEM; - } -} diff --git a/src/nvfs-p2p.h b/src/nvfs-p2p.h index 43a201d..fa8a33e 100644 --- a/src/nvfs-p2p.h +++ b/src/nvfs-p2p.h @@ -25,44 +25,11 @@ #include "nv-p2p.h" -typedef int (*nvidia_p2p_dma_unmap_pages_fptr) (struct pci_dev*, - struct nvidia_p2p_page_table*, - struct nvidia_p2p_dma_mapping*); -typedef int (*nvidia_p2p_get_pages_fptr) (uint64_t, uint32_t, - uint64_t, - uint64_t , - struct nvidia_p2p_page_table **, - void (*free_callback)(void *data), - void *); -typedef int (*nvidia_p2p_put_pages_fptr)(uint64_t, uint32_t, - uint64_t, - struct nvidia_p2p_page_table *); -typedef int (*nvidia_p2p_dma_map_pages_fptr)(struct pci_dev *, - struct nvidia_p2p_page_table *, - struct nvidia_p2p_dma_mapping **); -typedef int (*nvidia_p2p_free_dma_mapping_fptr)(struct nvidia_p2p_dma_mapping *); -typedef int (*nvidia_p2p_free_page_table_fptr)(struct nvidia_p2p_page_table *); - - -int nvfs_nvidia_p2p_dma_unmap_pages(struct pci_dev *peer, - struct nvidia_p2p_page_table *page_table, - struct nvidia_p2p_dma_mapping *dma_mapping); -int nvfs_nvidia_p2p_get_pages(uint64_t p2p_token, uint32_t va_space, - uint64_t virtual_address, - uint64_t length, - struct nvidia_p2p_page_table **page_table, - void (*free_callback)(void *data), - void *data); -int nvfs_nvidia_p2p_put_pages(uint64_t p2p_token, uint32_t va_space, - uint64_t virtual_address, - struct nvidia_p2p_page_table *page_table); -int nvfs_nvidia_p2p_dma_map_pages(struct pci_dev *peer, - struct nvidia_p2p_page_table *page_table, - struct nvidia_p2p_dma_mapping **dma_mapping); -int nvfs_nvidia_p2p_free_dma_mapping(struct nvidia_p2p_dma_mapping *dma_mapping); -int nvfs_nvidia_p2p_free_page_table(struct nvidia_p2p_page_table *page_table); - -int nvfs_nvidia_p2p_init(void); -void nvfs_nvidia_p2p_exit(void); +#define nvfs_nvidia_p2p_get_pages nvidia_p2p_get_pages +#define nvfs_nvidia_p2p_put_pages nvidia_p2p_put_pages +#define nvfs_nvidia_p2p_dma_map_pages nvidia_p2p_dma_map_pages +#define nvfs_nvidia_p2p_dma_unmap_pages nvidia_p2p_dma_unmap_pages +#define nvfs_nvidia_p2p_free_page_table nvidia_p2p_free_page_table +#define nvfs_nvidia_p2p_free_dma_mapping nvidia_p2p_free_dma_mapping #endif diff --git a/src/nvfs-vers.h b/src/nvfs-vers.h index 1af6f51..ffb9738 100644 --- a/src/nvfs-vers.h +++ b/src/nvfs-vers.h @@ -29,7 +29,7 @@ #define NVFS_DRIVER_MINOR_VERSION 17 //2-bytes // template for build version -#define NVFS_DRIVER_PATCH_VERSION 4 +#define NVFS_DRIVER_PATCH_VERSION 5 static inline unsigned int nvfs_driver_version(void) { return (NVFS_DRIVER_MAJOR_VERSION << 16) | NVFS_DRIVER_MINOR_VERSION;