From 82c02b76c406f1395d758fdea32b6cb35feeb7f1 Mon Sep 17 00:00:00 2001 From: yf711 <109183385+yf711@users.noreply.github.com> Date: Fri, 14 Feb 2025 14:48:25 -0800 Subject: [PATCH 01/10] revert to 10.7-GA --- cmake/deps.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmake/deps.txt b/cmake/deps.txt index 5d8149d5bf61b..2ca666622f202 100644 --- a/cmake/deps.txt +++ b/cmake/deps.txt @@ -36,8 +36,8 @@ microsoft_wil;https://github.com/microsoft/wil/archive/refs/tags/v1.0.230629.1.z mimalloc;https://github.com/microsoft/mimalloc/archive/refs/tags/v2.1.1.zip;d5ee7d34223d0567892db5179849939c8769dc41 mp11;https://github.com/boostorg/mp11/archive/refs/tags/boost-1.82.0.zip;9bc9e01dffb64d9e0773b2e44d2f22c51aace063 onnx;https://github.com/onnx/onnx/archive/refs/tags/v1.17.0.zip;13a60ac5217c104139ce0fd024f48628e7bcf5bc -# Use the latest commit of 10.8-GA -onnx_tensorrt;https://github.com/onnx/onnx-tensorrt/archive/118ed0aea197fa9a7d3ea66180a1d5ddb9deecc3.zip;b78aed3728ad4daf6dc47ea10c1d243dee1d95b1 +# Use the latest commit of 10.7-GA +onnx_tensorrt;https://github.com/onnx/onnx-tensorrt/archive/9c69a24bc2e20c8a511a4e6b06fd49639ec5300a.zip;ff1fe9af78eb129b4a4cdcb7450b7390b4436dd3 protobuf;https://github.com/protocolbuffers/protobuf/archive/refs/tags/v21.12.zip;7cf2733949036c7d52fda017badcab093fe73bfa protoc_win64;https://github.com/protocolbuffers/protobuf/releases/download/v21.12/protoc-21.12-win64.zip;b4521f7ada5b260380f94c4bd7f1b7684c76969a protoc_win32;https://github.com/protocolbuffers/protobuf/releases/download/v21.12/protoc-21.12-win32.zip;3688010318192c46ce73213cdfb6b3e5656da874 From b3e06da9b65cc9b6748ba48c865c6cbf3bb49b7c Mon Sep 17 00:00:00 2001 From: yf711 <109183385+yf711@users.noreply.github.com> Date: Fri, 14 Feb 2025 14:53:41 -0800 Subject: [PATCH 02/10] update --- cmake/onnxruntime_providers_tensorrt.cmake | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cmake/onnxruntime_providers_tensorrt.cmake b/cmake/onnxruntime_providers_tensorrt.cmake index 8723fa1a929ef..98ffcf90b4953 100644 --- a/cmake/onnxruntime_providers_tensorrt.cmake +++ b/cmake/onnxruntime_providers_tensorrt.cmake @@ -39,6 +39,10 @@ HINTS ${TENSORRT_ROOT} PATH_SUFFIXES include) + # TensorRT Python Headers, needed by oss parser + find_path(TENSORRT_PYTHON_INCLUDE_DIR plugin.h + HINTS ${TENSORRT_ROOT} + PATH_SUFFIXES python/include/impl) file(READ ${TENSORRT_INCLUDE_DIR}/NvInferVersion.h NVINFER_VER_CONTENT) string(REGEX MATCH "define NV_TENSORRT_MAJOR * +([0-9]+)" NV_TENSORRT_MAJOR "${NVINFER_VER_CONTENT}") From a81189192093f806b397a62f79221d2c158a5a3b Mon Sep 17 00:00:00 2001 From: yf711 <109183385+yf711@users.noreply.github.com> Date: Fri, 14 Feb 2025 15:04:32 -0800 Subject: [PATCH 03/10] update to newer 10.7-GA with fix --- cmake/deps.txt | 4 ++-- cmake/onnxruntime_providers_tensorrt.cmake | 1 + cmake/onnxruntime_unittests.cmake | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/cmake/deps.txt b/cmake/deps.txt index 2ca666622f202..fc09eb7808d74 100644 --- a/cmake/deps.txt +++ b/cmake/deps.txt @@ -36,8 +36,8 @@ microsoft_wil;https://github.com/microsoft/wil/archive/refs/tags/v1.0.230629.1.z mimalloc;https://github.com/microsoft/mimalloc/archive/refs/tags/v2.1.1.zip;d5ee7d34223d0567892db5179849939c8769dc41 mp11;https://github.com/boostorg/mp11/archive/refs/tags/boost-1.82.0.zip;9bc9e01dffb64d9e0773b2e44d2f22c51aace063 onnx;https://github.com/onnx/onnx/archive/refs/tags/v1.17.0.zip;13a60ac5217c104139ce0fd024f48628e7bcf5bc -# Use the latest commit of 10.7-GA -onnx_tensorrt;https://github.com/onnx/onnx-tensorrt/archive/9c69a24bc2e20c8a511a4e6b06fd49639ec5300a.zip;ff1fe9af78eb129b4a4cdcb7450b7390b4436dd3 +# Use the latest commit of 10.7-GA, with fix towards scatterND +onnx_tensorrt;https://github.com/onnx/onnx-tensorrt/archive/87a6fc947a7a07afcd45eb5adf96a2c57281d18e.zip;70b80ffadd4b79d4ee502a362a480fcc5ddffb45 protobuf;https://github.com/protocolbuffers/protobuf/archive/refs/tags/v21.12.zip;7cf2733949036c7d52fda017badcab093fe73bfa protoc_win64;https://github.com/protocolbuffers/protobuf/releases/download/v21.12/protoc-21.12-win64.zip;b4521f7ada5b260380f94c4bd7f1b7684c76969a protoc_win32;https://github.com/protocolbuffers/protobuf/releases/download/v21.12/protoc-21.12-win32.zip;3688010318192c46ce73213cdfb6b3e5656da874 diff --git a/cmake/onnxruntime_providers_tensorrt.cmake b/cmake/onnxruntime_providers_tensorrt.cmake index 98ffcf90b4953..4efe512d3ccdd 100644 --- a/cmake/onnxruntime_providers_tensorrt.cmake +++ b/cmake/onnxruntime_providers_tensorrt.cmake @@ -156,6 +156,7 @@ endif() include_directories(${TENSORRT_INCLUDE_DIR}) + include_directories(${TENSORRT_PYTHON_INCLUDE_DIR}) # ${TENSORRT_LIBRARY} is empty if we link nvonnxparser_static. # nvonnxparser_static is linked against tensorrt libraries in onnx-tensorrt # See https://github.com/onnx/onnx-tensorrt/blob/8af13d1b106f58df1e98945a5e7c851ddb5f0791/CMakeLists.txt#L121 diff --git a/cmake/onnxruntime_unittests.cmake b/cmake/onnxruntime_unittests.cmake index 789647673e782..024a8112cc1bd 100644 --- a/cmake/onnxruntime_unittests.cmake +++ b/cmake/onnxruntime_unittests.cmake @@ -85,7 +85,7 @@ function(AddTest) endif() if (onnxruntime_USE_TENSORRT) # used for instantiating placeholder TRT builder to mitigate TRT library load/unload overhead - target_include_directories(${_UT_TARGET} PRIVATE ${TENSORRT_INCLUDE_DIR}) + target_include_directories(${_UT_TARGET} PRIVATE ${TENSORRT_INCLUDE_DIR} ${TENSORRT_PYTHON_INCLUDE_DIR}) endif() if(MSVC) From 4f04653e1b00c4b9b7f86ca4899f63a1a7953f07 Mon Sep 17 00:00:00 2001 From: yf711 <109183385+yf711@users.noreply.github.com> Date: Sat, 15 Feb 2025 00:38:06 -0800 Subject: [PATCH 04/10] add TensorRT Python packages --- .../ci_build/github/linux/docker/Dockerfile.manylinux2_28_cuda | 3 ++- .../linux/docker/Dockerfile.package_ubi8_cuda_tensorrt10_0 | 3 ++- .../docker/Dockerfile.package_ubi8_cuda_tensorrt10_0_torch | 3 ++- .../github/linux/docker/Dockerfile.package_ubuntu_2004_gpu | 3 ++- .../linux/docker/Dockerfile.package_ubuntu_2204_gpu_ffmpeg | 3 ++- .../linux/docker/Dockerfile.package_ubuntu_2204_gpu_opencv | 3 ++- .../github/linux/docker/Dockerfile.ubuntu_cuda11_tensorrt10 | 3 ++- .../github/linux/docker/Dockerfile.ubuntu_cuda12_tensorrt10 | 3 ++- 8 files changed, 16 insertions(+), 8 deletions(-) diff --git a/tools/ci_build/github/linux/docker/Dockerfile.manylinux2_28_cuda b/tools/ci_build/github/linux/docker/Dockerfile.manylinux2_28_cuda index c42042b0ec639..81922194a1736 100644 --- a/tools/ci_build/github/linux/docker/Dockerfile.manylinux2_28_cuda +++ b/tools/ci_build/github/linux/docker/Dockerfile.manylinux2_28_cuda @@ -26,7 +26,8 @@ RUN if [ -n "$TRT_VERSION" ]; then \ libnvinfer-vc-plugin-devel-${TRT_VERSION} \ libnvinfer-lean-devel-${TRT_VERSION} \ libnvinfer-dispatch-devel-${TRT_VERSION} \ - libnvinfer-headers-plugin-devel-${TRT_VERSION} && \ + libnvinfer-headers-plugin-devel-${TRT_VERSION} \ + python3-libnvinfer-dev-${TRT_VERSION} && \ dnf clean dbcache ; \ else \ echo "TRT_VERSION is none skipping Tensor RT Installation" ; \ diff --git a/tools/ci_build/github/linux/docker/Dockerfile.package_ubi8_cuda_tensorrt10_0 b/tools/ci_build/github/linux/docker/Dockerfile.package_ubi8_cuda_tensorrt10_0 index 7abc7a6d35ec3..86b4f0e67eb3d 100644 --- a/tools/ci_build/github/linux/docker/Dockerfile.package_ubi8_cuda_tensorrt10_0 +++ b/tools/ci_build/github/linux/docker/Dockerfile.package_ubi8_cuda_tensorrt10_0 @@ -36,7 +36,8 @@ RUN if [ -n "$TRT_VERSION" ]; then \ libnvinfer-vc-plugin-devel-${TRT_VERSION} \ libnvinfer-lean-devel-${TRT_VERSION} \ libnvinfer-dispatch-devel-${TRT_VERSION} \ - libnvinfer-headers-plugin-devel-${TRT_VERSION} && \ + libnvinfer-headers-plugin-devel-${TRT_VERSION} \ + python3-libnvinfer-dev-${TRT_VERSION} && \ dnf clean dbcache ; \ else \ echo "TRT_VERSION is none skipping Tensor RT Installation" ; \ diff --git a/tools/ci_build/github/linux/docker/Dockerfile.package_ubi8_cuda_tensorrt10_0_torch b/tools/ci_build/github/linux/docker/Dockerfile.package_ubi8_cuda_tensorrt10_0_torch index 2df43197b7d39..d2063032384d8 100644 --- a/tools/ci_build/github/linux/docker/Dockerfile.package_ubi8_cuda_tensorrt10_0_torch +++ b/tools/ci_build/github/linux/docker/Dockerfile.package_ubi8_cuda_tensorrt10_0_torch @@ -36,7 +36,8 @@ RUN if [ -n "$TRT_VERSION" ]; then \ libnvinfer-vc-plugin-devel-${TRT_VERSION} \ libnvinfer-lean-devel-${TRT_VERSION} \ libnvinfer-dispatch-devel-${TRT_VERSION} \ - libnvinfer-headers-plugin-devel-${TRT_VERSION} && \ + libnvinfer-headers-plugin-devel-${TRT_VERSION} \ + python3-libnvinfer-dev-${TRT_VERSION} && \ dnf clean dbcache ; \ else \ echo "TRT_VERSION is none skipping Tensor RT Installation" ; \ diff --git a/tools/ci_build/github/linux/docker/Dockerfile.package_ubuntu_2004_gpu b/tools/ci_build/github/linux/docker/Dockerfile.package_ubuntu_2004_gpu index 64186e83f001a..4f060b6cf8ed7 100644 --- a/tools/ci_build/github/linux/docker/Dockerfile.package_ubuntu_2004_gpu +++ b/tools/ci_build/github/linux/docker/Dockerfile.package_ubuntu_2004_gpu @@ -46,7 +46,8 @@ RUN apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/ libnvonnxparsers-dev=${TRT_VERSION} \ libnvonnxparsers10=${TRT_VERSION} \ tensorrt-dev=${TRT_VERSION} \ - libnvinfer-bin=${TRT_VERSION} &&\ + libnvinfer-bin=${TRT_VERSION} \ + python3-libnvinfer-dev=${TRT_VERSION} &&\ if [ $(echo $CUDA_VERSION | cut -d"." -f1) -ge 12 ]; then apt-get install -y cudnn9-cuda-12 ; fi # ^^^^^^^^^^^If cuda version is 12 or higher, install cudnn 9 for cuda 12 diff --git a/tools/ci_build/github/linux/docker/Dockerfile.package_ubuntu_2204_gpu_ffmpeg b/tools/ci_build/github/linux/docker/Dockerfile.package_ubuntu_2204_gpu_ffmpeg index a987d511ddc6d..7c612c1d852eb 100644 --- a/tools/ci_build/github/linux/docker/Dockerfile.package_ubuntu_2204_gpu_ffmpeg +++ b/tools/ci_build/github/linux/docker/Dockerfile.package_ubuntu_2204_gpu_ffmpeg @@ -46,7 +46,8 @@ RUN apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/ libnvonnxparsers-dev=${TRT_VERSION} \ libnvonnxparsers10=${TRT_VERSION} \ tensorrt-dev=${TRT_VERSION} \ - libnvinfer-bin=${TRT_VERSION} + libnvinfer-bin=${TRT_VERSION} \ + python3-libnvinfer-dev=${TRT_VERSION} ADD scripts /tmp/scripts RUN cd /tmp/scripts && /tmp/scripts/install_dotnet.sh && rm -rf /tmp/scripts diff --git a/tools/ci_build/github/linux/docker/Dockerfile.package_ubuntu_2204_gpu_opencv b/tools/ci_build/github/linux/docker/Dockerfile.package_ubuntu_2204_gpu_opencv index 7f5e8f871f415..8735f1ccb7f95 100644 --- a/tools/ci_build/github/linux/docker/Dockerfile.package_ubuntu_2204_gpu_opencv +++ b/tools/ci_build/github/linux/docker/Dockerfile.package_ubuntu_2204_gpu_opencv @@ -48,7 +48,8 @@ RUN apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/ libnvonnxparsers-dev=${TRT_VERSION} \ libnvonnxparsers10=${TRT_VERSION} \ tensorrt-dev=${TRT_VERSION} \ - libnvinfer-bin=${TRT_VERSION} &&\ + libnvinfer-bin=${TRT_VERSION} \ + python3-libnvinfer-dev=${TRT_VERSION} &&\ if [ $(echo $CUDA_VERSION | cut -d"." -f1) -ge 12 ]; then apt-get install -y cudnn9-cuda-12 ; fi # ^^^^^^^^^^^If cuda version is 12 or higher, install cudnn 9 for cuda 12 diff --git a/tools/ci_build/github/linux/docker/Dockerfile.ubuntu_cuda11_tensorrt10 b/tools/ci_build/github/linux/docker/Dockerfile.ubuntu_cuda11_tensorrt10 index f454d21164ac4..e506a9d172f98 100644 --- a/tools/ci_build/github/linux/docker/Dockerfile.ubuntu_cuda11_tensorrt10 +++ b/tools/ci_build/github/linux/docker/Dockerfile.ubuntu_cuda11_tensorrt10 @@ -50,7 +50,8 @@ RUN TRT_VERSION="10.8.0.43-1+cuda11.8" &&\ libnvonnxparsers-dev=${TRT_VERSION} \ libnvonnxparsers10=${TRT_VERSION} \ tensorrt-dev=${TRT_VERSION} \ - libnvinfer-bin=${TRT_VERSION} + libnvinfer-bin=${TRT_VERSION} \ + python3-libnvinfer-dev=${TRT_VERSION} # Compile trtexec if not installed RUN if [ ! -d /usr/src/tensorrt/bin ] || [ ! -f /usr/src/tensorrt/bin/trtexec ]; then \ diff --git a/tools/ci_build/github/linux/docker/Dockerfile.ubuntu_cuda12_tensorrt10 b/tools/ci_build/github/linux/docker/Dockerfile.ubuntu_cuda12_tensorrt10 index a9dbefc6faee0..51905849e90d2 100644 --- a/tools/ci_build/github/linux/docker/Dockerfile.ubuntu_cuda12_tensorrt10 +++ b/tools/ci_build/github/linux/docker/Dockerfile.ubuntu_cuda12_tensorrt10 @@ -50,7 +50,8 @@ RUN TRT_VERSION="10.8.0.43-1+cuda12.8" &&\ libnvonnxparsers-dev=${TRT_VERSION} \ libnvonnxparsers10=${TRT_VERSION} \ tensorrt-dev=${TRT_VERSION} \ - libnvinfer-bin=${TRT_VERSION} + libnvinfer-bin=${TRT_VERSION} \ + python3-libnvinfer-dev=${TRT_VERSION} # Compile trtexec if not installed RUN if [ ! -d /usr/src/tensorrt/bin ] || [ ! -f /usr/src/tensorrt/bin/trtexec ]; then \ From 87523a85b7678df0fe2666f3f1803d50a33b55cb Mon Sep 17 00:00:00 2001 From: yf711 <109183385+yf711@users.noreply.github.com> Date: Tue, 18 Feb 2025 11:58:22 -0800 Subject: [PATCH 05/10] update --- cmake/onnxruntime_providers_tensorrt.cmake | 38 +++++++++++++++------- cmake/onnxruntime_unittests.cmake | 2 +- 2 files changed, 27 insertions(+), 13 deletions(-) diff --git a/cmake/onnxruntime_providers_tensorrt.cmake b/cmake/onnxruntime_providers_tensorrt.cmake index 4efe512d3ccdd..e86a64b8d5860 100644 --- a/cmake/onnxruntime_providers_tensorrt.cmake +++ b/cmake/onnxruntime_providers_tensorrt.cmake @@ -39,11 +39,6 @@ HINTS ${TENSORRT_ROOT} PATH_SUFFIXES include) - # TensorRT Python Headers, needed by oss parser - find_path(TENSORRT_PYTHON_INCLUDE_DIR plugin.h - HINTS ${TENSORRT_ROOT} - PATH_SUFFIXES python/include/impl) - file(READ ${TENSORRT_INCLUDE_DIR}/NvInferVersion.h NVINFER_VER_CONTENT) string(REGEX MATCH "define NV_TENSORRT_MAJOR * +([0-9]+)" NV_TENSORRT_MAJOR "${NVINFER_VER_CONTENT}") string(REGEX REPLACE "define NV_TENSORRT_MAJOR * +([0-9]+)" "\\1" NV_TENSORRT_MAJOR "${NV_TENSORRT_MAJOR}") @@ -61,16 +56,35 @@ MESSAGE(STATUS "Can't find NV_TENSORRT_MAJOR macro") endif() - # Check TRT version >= 10.0.1.6 - if ((NV_TENSORRT_MAJOR_INT GREATER 10) OR - (NV_TENSORRT_MAJOR_INT EQUAL 10 AND NV_TENSORRT_MINOR_INT GREATER 0) OR - (NV_TENSORRT_MAJOR_INT EQUAL 10 AND NV_TENSORRT_PATCH_INT GREATER 0)) - set(TRT_GREATER_OR_EQUAL_TRT_10_GA ON) + # Check TRT version >= requirement + function(check_trt_version req_major req_minor req_patch out_var) + set(version_ok FALSE) + if(NV_TENSORRT_MAJOR_INT GREATER ${req_major}) + set(version_ok TRUE) + elseif(NV_TENSORRT_MAJOR_INT EQUAL ${req_major}) + if(NV_TENSORRT_MINOR_INT GREATER ${req_minor}) + set(version_ok TRUE) + elseif(NV_TENSORRT_MINOR_INT EQUAL ${req_minor}) + if(NV_TENSORRT_PATCH_INT GREATER_EQUAL ${req_patch}) + set(version_ok TRUE) + endif() + endif() + endif() + set(${out_var} ${version_ok} PARENT_SCOPE) + endfunction() + + # TensorRT Python Headers, needed by oss parser since 10.6 + check_trt_version(10 6 0 TRT_VERSION_OK_10_6_0) + if (TRT_VERSION_OK_10_6_0 AND NOT onnxruntime_USE_TENSORRT_BUILTIN_PARSER) + find_path(TENSORRT_PYTHON_INCLUDE_DIR plugin.h + HINTS ${TENSORRT_ROOT} + PATH_SUFFIXES python/include/impl) endif() # TensorRT 10 GA onwards, the TensorRT libraries will have major version appended to the end on Windows, # for example, nvinfer_10.dll, nvinfer_plugin_10.dll, nvonnxparser_10.dll ... - if (WIN32 AND TRT_GREATER_OR_EQUAL_TRT_10_GA) + check_trt_version(10 0 1 TRT_VERSION_OK_10_0_1) + if (WIN32 AND TRT_VERSION_OK_10_0_1) set(NVINFER_LIB "nvinfer_${NV_TENSORRT_MAJOR}") set(NVINFER_PLUGIN_LIB "nvinfer_plugin_${NV_TENSORRT_MAJOR}") set(PARSER_LIB "nvonnxparser_${NV_TENSORRT_MAJOR}") @@ -120,7 +134,7 @@ set(TENSORRT_LIBRARY ${TENSORRT_LIBRARY_INFER} ${TENSORRT_LIBRARY_INFER_PLUGIN} ${TENSORRT_LIBRARY_NVONNXPARSER}) MESSAGE(STATUS "Find TensorRT libs at ${TENSORRT_LIBRARY}") else() - if (TRT_GREATER_OR_EQUAL_TRT_10_GA) + if (TRT_VERSION_OK_10_0_1) set(ONNX_USE_LITE_PROTO ON) endif() onnxruntime_fetchcontent_declare( diff --git a/cmake/onnxruntime_unittests.cmake b/cmake/onnxruntime_unittests.cmake index 024a8112cc1bd..789647673e782 100644 --- a/cmake/onnxruntime_unittests.cmake +++ b/cmake/onnxruntime_unittests.cmake @@ -85,7 +85,7 @@ function(AddTest) endif() if (onnxruntime_USE_TENSORRT) # used for instantiating placeholder TRT builder to mitigate TRT library load/unload overhead - target_include_directories(${_UT_TARGET} PRIVATE ${TENSORRT_INCLUDE_DIR} ${TENSORRT_PYTHON_INCLUDE_DIR}) + target_include_directories(${_UT_TARGET} PRIVATE ${TENSORRT_INCLUDE_DIR}) endif() if(MSVC) From 978b58f4ac850266c2be7cec830605bcac471d19 Mon Sep 17 00:00:00 2001 From: yf711 <109183385+yf711@users.noreply.github.com> Date: Tue, 18 Feb 2025 12:05:47 -0800 Subject: [PATCH 06/10] add log --- cmake/onnxruntime_providers_tensorrt.cmake | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/cmake/onnxruntime_providers_tensorrt.cmake b/cmake/onnxruntime_providers_tensorrt.cmake index e86a64b8d5860..652ce1b43830d 100644 --- a/cmake/onnxruntime_providers_tensorrt.cmake +++ b/cmake/onnxruntime_providers_tensorrt.cmake @@ -71,6 +71,11 @@ endif() endif() set(${out_var} ${version_ok} PARENT_SCOPE) + if(version_ok) + message(STATUS "TensorRT ${NV_TENSORRT_MAJOR_INT}.${NV_TENSORRT_MINOR_INT}.${NV_TENSORRT_PATCH_INT} >= ${req_major}.${req_minor}.${req_patch}.") + else() + message(STATUS "TensorRT ${NV_TENSORRT_MAJOR_INT}.${NV_TENSORRT_MINOR_INT}.${NV_TENSORRT_PATCH_INT} < ${req_major}.${req_minor}.${req_patch}.") + endif() endfunction() # TensorRT Python Headers, needed by oss parser since 10.6 From dbee04caa9b3a09a7e25f4631891e994dc1417bf Mon Sep 17 00:00:00 2001 From: yf711 <109183385+yf711@users.noreply.github.com> Date: Tue, 18 Feb 2025 12:09:17 -0800 Subject: [PATCH 07/10] fix --- cmake/onnxruntime_providers_tensorrt.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/onnxruntime_providers_tensorrt.cmake b/cmake/onnxruntime_providers_tensorrt.cmake index 652ce1b43830d..8b3ec158bd241 100644 --- a/cmake/onnxruntime_providers_tensorrt.cmake +++ b/cmake/onnxruntime_providers_tensorrt.cmake @@ -84,6 +84,7 @@ find_path(TENSORRT_PYTHON_INCLUDE_DIR plugin.h HINTS ${TENSORRT_ROOT} PATH_SUFFIXES python/include/impl) + include_directories(${TENSORRT_PYTHON_INCLUDE_DIR}) endif() # TensorRT 10 GA onwards, the TensorRT libraries will have major version appended to the end on Windows, @@ -175,7 +176,6 @@ endif() include_directories(${TENSORRT_INCLUDE_DIR}) - include_directories(${TENSORRT_PYTHON_INCLUDE_DIR}) # ${TENSORRT_LIBRARY} is empty if we link nvonnxparser_static. # nvonnxparser_static is linked against tensorrt libraries in onnx-tensorrt # See https://github.com/onnx/onnx-tensorrt/blob/8af13d1b106f58df1e98945a5e7c851ddb5f0791/CMakeLists.txt#L121 From 8da27a0f82aa46b043b2382be8249d2516a771c9 Mon Sep 17 00:00:00 2001 From: yf711 <109183385+yf711@users.noreply.github.com> Date: Tue, 18 Feb 2025 13:25:08 -0800 Subject: [PATCH 08/10] revert as not needed --- .../ci_build/github/linux/docker/Dockerfile.manylinux2_28_cuda | 3 +-- .../linux/docker/Dockerfile.package_ubi8_cuda_tensorrt10_0 | 3 +-- .../docker/Dockerfile.package_ubi8_cuda_tensorrt10_0_torch | 3 +-- .../github/linux/docker/Dockerfile.package_ubuntu_2004_gpu | 3 +-- .../linux/docker/Dockerfile.package_ubuntu_2204_gpu_ffmpeg | 3 +-- .../linux/docker/Dockerfile.package_ubuntu_2204_gpu_opencv | 3 +-- .../github/linux/docker/Dockerfile.ubuntu_cuda11_tensorrt10 | 3 +-- .../github/linux/docker/Dockerfile.ubuntu_cuda12_tensorrt10 | 3 +-- 8 files changed, 8 insertions(+), 16 deletions(-) diff --git a/tools/ci_build/github/linux/docker/Dockerfile.manylinux2_28_cuda b/tools/ci_build/github/linux/docker/Dockerfile.manylinux2_28_cuda index 81922194a1736..c42042b0ec639 100644 --- a/tools/ci_build/github/linux/docker/Dockerfile.manylinux2_28_cuda +++ b/tools/ci_build/github/linux/docker/Dockerfile.manylinux2_28_cuda @@ -26,8 +26,7 @@ RUN if [ -n "$TRT_VERSION" ]; then \ libnvinfer-vc-plugin-devel-${TRT_VERSION} \ libnvinfer-lean-devel-${TRT_VERSION} \ libnvinfer-dispatch-devel-${TRT_VERSION} \ - libnvinfer-headers-plugin-devel-${TRT_VERSION} \ - python3-libnvinfer-dev-${TRT_VERSION} && \ + libnvinfer-headers-plugin-devel-${TRT_VERSION} && \ dnf clean dbcache ; \ else \ echo "TRT_VERSION is none skipping Tensor RT Installation" ; \ diff --git a/tools/ci_build/github/linux/docker/Dockerfile.package_ubi8_cuda_tensorrt10_0 b/tools/ci_build/github/linux/docker/Dockerfile.package_ubi8_cuda_tensorrt10_0 index 86b4f0e67eb3d..7abc7a6d35ec3 100644 --- a/tools/ci_build/github/linux/docker/Dockerfile.package_ubi8_cuda_tensorrt10_0 +++ b/tools/ci_build/github/linux/docker/Dockerfile.package_ubi8_cuda_tensorrt10_0 @@ -36,8 +36,7 @@ RUN if [ -n "$TRT_VERSION" ]; then \ libnvinfer-vc-plugin-devel-${TRT_VERSION} \ libnvinfer-lean-devel-${TRT_VERSION} \ libnvinfer-dispatch-devel-${TRT_VERSION} \ - libnvinfer-headers-plugin-devel-${TRT_VERSION} \ - python3-libnvinfer-dev-${TRT_VERSION} && \ + libnvinfer-headers-plugin-devel-${TRT_VERSION} && \ dnf clean dbcache ; \ else \ echo "TRT_VERSION is none skipping Tensor RT Installation" ; \ diff --git a/tools/ci_build/github/linux/docker/Dockerfile.package_ubi8_cuda_tensorrt10_0_torch b/tools/ci_build/github/linux/docker/Dockerfile.package_ubi8_cuda_tensorrt10_0_torch index d2063032384d8..2df43197b7d39 100644 --- a/tools/ci_build/github/linux/docker/Dockerfile.package_ubi8_cuda_tensorrt10_0_torch +++ b/tools/ci_build/github/linux/docker/Dockerfile.package_ubi8_cuda_tensorrt10_0_torch @@ -36,8 +36,7 @@ RUN if [ -n "$TRT_VERSION" ]; then \ libnvinfer-vc-plugin-devel-${TRT_VERSION} \ libnvinfer-lean-devel-${TRT_VERSION} \ libnvinfer-dispatch-devel-${TRT_VERSION} \ - libnvinfer-headers-plugin-devel-${TRT_VERSION} \ - python3-libnvinfer-dev-${TRT_VERSION} && \ + libnvinfer-headers-plugin-devel-${TRT_VERSION} && \ dnf clean dbcache ; \ else \ echo "TRT_VERSION is none skipping Tensor RT Installation" ; \ diff --git a/tools/ci_build/github/linux/docker/Dockerfile.package_ubuntu_2004_gpu b/tools/ci_build/github/linux/docker/Dockerfile.package_ubuntu_2004_gpu index 4f060b6cf8ed7..64186e83f001a 100644 --- a/tools/ci_build/github/linux/docker/Dockerfile.package_ubuntu_2004_gpu +++ b/tools/ci_build/github/linux/docker/Dockerfile.package_ubuntu_2004_gpu @@ -46,8 +46,7 @@ RUN apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/ libnvonnxparsers-dev=${TRT_VERSION} \ libnvonnxparsers10=${TRT_VERSION} \ tensorrt-dev=${TRT_VERSION} \ - libnvinfer-bin=${TRT_VERSION} \ - python3-libnvinfer-dev=${TRT_VERSION} &&\ + libnvinfer-bin=${TRT_VERSION} &&\ if [ $(echo $CUDA_VERSION | cut -d"." -f1) -ge 12 ]; then apt-get install -y cudnn9-cuda-12 ; fi # ^^^^^^^^^^^If cuda version is 12 or higher, install cudnn 9 for cuda 12 diff --git a/tools/ci_build/github/linux/docker/Dockerfile.package_ubuntu_2204_gpu_ffmpeg b/tools/ci_build/github/linux/docker/Dockerfile.package_ubuntu_2204_gpu_ffmpeg index 7c612c1d852eb..a987d511ddc6d 100644 --- a/tools/ci_build/github/linux/docker/Dockerfile.package_ubuntu_2204_gpu_ffmpeg +++ b/tools/ci_build/github/linux/docker/Dockerfile.package_ubuntu_2204_gpu_ffmpeg @@ -46,8 +46,7 @@ RUN apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/ libnvonnxparsers-dev=${TRT_VERSION} \ libnvonnxparsers10=${TRT_VERSION} \ tensorrt-dev=${TRT_VERSION} \ - libnvinfer-bin=${TRT_VERSION} \ - python3-libnvinfer-dev=${TRT_VERSION} + libnvinfer-bin=${TRT_VERSION} ADD scripts /tmp/scripts RUN cd /tmp/scripts && /tmp/scripts/install_dotnet.sh && rm -rf /tmp/scripts diff --git a/tools/ci_build/github/linux/docker/Dockerfile.package_ubuntu_2204_gpu_opencv b/tools/ci_build/github/linux/docker/Dockerfile.package_ubuntu_2204_gpu_opencv index 8735f1ccb7f95..7f5e8f871f415 100644 --- a/tools/ci_build/github/linux/docker/Dockerfile.package_ubuntu_2204_gpu_opencv +++ b/tools/ci_build/github/linux/docker/Dockerfile.package_ubuntu_2204_gpu_opencv @@ -48,8 +48,7 @@ RUN apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/ libnvonnxparsers-dev=${TRT_VERSION} \ libnvonnxparsers10=${TRT_VERSION} \ tensorrt-dev=${TRT_VERSION} \ - libnvinfer-bin=${TRT_VERSION} \ - python3-libnvinfer-dev=${TRT_VERSION} &&\ + libnvinfer-bin=${TRT_VERSION} &&\ if [ $(echo $CUDA_VERSION | cut -d"." -f1) -ge 12 ]; then apt-get install -y cudnn9-cuda-12 ; fi # ^^^^^^^^^^^If cuda version is 12 or higher, install cudnn 9 for cuda 12 diff --git a/tools/ci_build/github/linux/docker/Dockerfile.ubuntu_cuda11_tensorrt10 b/tools/ci_build/github/linux/docker/Dockerfile.ubuntu_cuda11_tensorrt10 index e506a9d172f98..f454d21164ac4 100644 --- a/tools/ci_build/github/linux/docker/Dockerfile.ubuntu_cuda11_tensorrt10 +++ b/tools/ci_build/github/linux/docker/Dockerfile.ubuntu_cuda11_tensorrt10 @@ -50,8 +50,7 @@ RUN TRT_VERSION="10.8.0.43-1+cuda11.8" &&\ libnvonnxparsers-dev=${TRT_VERSION} \ libnvonnxparsers10=${TRT_VERSION} \ tensorrt-dev=${TRT_VERSION} \ - libnvinfer-bin=${TRT_VERSION} \ - python3-libnvinfer-dev=${TRT_VERSION} + libnvinfer-bin=${TRT_VERSION} # Compile trtexec if not installed RUN if [ ! -d /usr/src/tensorrt/bin ] || [ ! -f /usr/src/tensorrt/bin/trtexec ]; then \ diff --git a/tools/ci_build/github/linux/docker/Dockerfile.ubuntu_cuda12_tensorrt10 b/tools/ci_build/github/linux/docker/Dockerfile.ubuntu_cuda12_tensorrt10 index 51905849e90d2..a9dbefc6faee0 100644 --- a/tools/ci_build/github/linux/docker/Dockerfile.ubuntu_cuda12_tensorrt10 +++ b/tools/ci_build/github/linux/docker/Dockerfile.ubuntu_cuda12_tensorrt10 @@ -50,8 +50,7 @@ RUN TRT_VERSION="10.8.0.43-1+cuda12.8" &&\ libnvonnxparsers-dev=${TRT_VERSION} \ libnvonnxparsers10=${TRT_VERSION} \ tensorrt-dev=${TRT_VERSION} \ - libnvinfer-bin=${TRT_VERSION} \ - python3-libnvinfer-dev=${TRT_VERSION} + libnvinfer-bin=${TRT_VERSION} # Compile trtexec if not installed RUN if [ ! -d /usr/src/tensorrt/bin ] || [ ! -f /usr/src/tensorrt/bin/trtexec ]; then \ From 6379fdc8563ae0078b11115b6babf51ab95b1bca Mon Sep 17 00:00:00 2001 From: yf711 <109183385+yf711@users.noreply.github.com> Date: Wed, 19 Feb 2025 14:27:26 -0800 Subject: [PATCH 09/10] clean unused --- cmake/onnxruntime_providers_tensorrt.cmake | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/cmake/onnxruntime_providers_tensorrt.cmake b/cmake/onnxruntime_providers_tensorrt.cmake index 8b3ec158bd241..defc70f81d780 100644 --- a/cmake/onnxruntime_providers_tensorrt.cmake +++ b/cmake/onnxruntime_providers_tensorrt.cmake @@ -77,16 +77,7 @@ message(STATUS "TensorRT ${NV_TENSORRT_MAJOR_INT}.${NV_TENSORRT_MINOR_INT}.${NV_TENSORRT_PATCH_INT} < ${req_major}.${req_minor}.${req_patch}.") endif() endfunction() - - # TensorRT Python Headers, needed by oss parser since 10.6 - check_trt_version(10 6 0 TRT_VERSION_OK_10_6_0) - if (TRT_VERSION_OK_10_6_0 AND NOT onnxruntime_USE_TENSORRT_BUILTIN_PARSER) - find_path(TENSORRT_PYTHON_INCLUDE_DIR plugin.h - HINTS ${TENSORRT_ROOT} - PATH_SUFFIXES python/include/impl) - include_directories(${TENSORRT_PYTHON_INCLUDE_DIR}) - endif() - + # TensorRT 10 GA onwards, the TensorRT libraries will have major version appended to the end on Windows, # for example, nvinfer_10.dll, nvinfer_plugin_10.dll, nvonnxparser_10.dll ... check_trt_version(10 0 1 TRT_VERSION_OK_10_0_1) From f1dcc05187acec6e2d8660954d675ef59472ab58 Mon Sep 17 00:00:00 2001 From: yf711 <109183385+yf711@users.noreply.github.com> Date: Wed, 19 Feb 2025 14:38:30 -0800 Subject: [PATCH 10/10] revert --- cmake/onnxruntime_providers_tensorrt.cmake | 35 +++++++--------------- 1 file changed, 10 insertions(+), 25 deletions(-) diff --git a/cmake/onnxruntime_providers_tensorrt.cmake b/cmake/onnxruntime_providers_tensorrt.cmake index defc70f81d780..8723fa1a929ef 100644 --- a/cmake/onnxruntime_providers_tensorrt.cmake +++ b/cmake/onnxruntime_providers_tensorrt.cmake @@ -39,6 +39,7 @@ HINTS ${TENSORRT_ROOT} PATH_SUFFIXES include) + file(READ ${TENSORRT_INCLUDE_DIR}/NvInferVersion.h NVINFER_VER_CONTENT) string(REGEX MATCH "define NV_TENSORRT_MAJOR * +([0-9]+)" NV_TENSORRT_MAJOR "${NVINFER_VER_CONTENT}") string(REGEX REPLACE "define NV_TENSORRT_MAJOR * +([0-9]+)" "\\1" NV_TENSORRT_MAJOR "${NV_TENSORRT_MAJOR}") @@ -56,32 +57,16 @@ MESSAGE(STATUS "Can't find NV_TENSORRT_MAJOR macro") endif() - # Check TRT version >= requirement - function(check_trt_version req_major req_minor req_patch out_var) - set(version_ok FALSE) - if(NV_TENSORRT_MAJOR_INT GREATER ${req_major}) - set(version_ok TRUE) - elseif(NV_TENSORRT_MAJOR_INT EQUAL ${req_major}) - if(NV_TENSORRT_MINOR_INT GREATER ${req_minor}) - set(version_ok TRUE) - elseif(NV_TENSORRT_MINOR_INT EQUAL ${req_minor}) - if(NV_TENSORRT_PATCH_INT GREATER_EQUAL ${req_patch}) - set(version_ok TRUE) - endif() - endif() - endif() - set(${out_var} ${version_ok} PARENT_SCOPE) - if(version_ok) - message(STATUS "TensorRT ${NV_TENSORRT_MAJOR_INT}.${NV_TENSORRT_MINOR_INT}.${NV_TENSORRT_PATCH_INT} >= ${req_major}.${req_minor}.${req_patch}.") - else() - message(STATUS "TensorRT ${NV_TENSORRT_MAJOR_INT}.${NV_TENSORRT_MINOR_INT}.${NV_TENSORRT_PATCH_INT} < ${req_major}.${req_minor}.${req_patch}.") - endif() - endfunction() - + # Check TRT version >= 10.0.1.6 + if ((NV_TENSORRT_MAJOR_INT GREATER 10) OR + (NV_TENSORRT_MAJOR_INT EQUAL 10 AND NV_TENSORRT_MINOR_INT GREATER 0) OR + (NV_TENSORRT_MAJOR_INT EQUAL 10 AND NV_TENSORRT_PATCH_INT GREATER 0)) + set(TRT_GREATER_OR_EQUAL_TRT_10_GA ON) + endif() + # TensorRT 10 GA onwards, the TensorRT libraries will have major version appended to the end on Windows, # for example, nvinfer_10.dll, nvinfer_plugin_10.dll, nvonnxparser_10.dll ... - check_trt_version(10 0 1 TRT_VERSION_OK_10_0_1) - if (WIN32 AND TRT_VERSION_OK_10_0_1) + if (WIN32 AND TRT_GREATER_OR_EQUAL_TRT_10_GA) set(NVINFER_LIB "nvinfer_${NV_TENSORRT_MAJOR}") set(NVINFER_PLUGIN_LIB "nvinfer_plugin_${NV_TENSORRT_MAJOR}") set(PARSER_LIB "nvonnxparser_${NV_TENSORRT_MAJOR}") @@ -131,7 +116,7 @@ set(TENSORRT_LIBRARY ${TENSORRT_LIBRARY_INFER} ${TENSORRT_LIBRARY_INFER_PLUGIN} ${TENSORRT_LIBRARY_NVONNXPARSER}) MESSAGE(STATUS "Find TensorRT libs at ${TENSORRT_LIBRARY}") else() - if (TRT_VERSION_OK_10_0_1) + if (TRT_GREATER_OR_EQUAL_TRT_10_GA) set(ONNX_USE_LITE_PROTO ON) endif() onnxruntime_fetchcontent_declare(