diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 197117ef2..72fe0a69e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -132,7 +132,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [ubuntu-22.04, windows-2019] + os: [ubuntu-22.04, windows-2022] arch: [auto64] include: - os: ubuntu-22.04 @@ -183,13 +183,13 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [ubuntu-22.04, windows-2019] + os: [ubuntu-22.04, windows-2022] steps: - - name: Set up Python 3.9 + - name: Set up Python 3.10.11 uses: actions/setup-python@v5 with: - python-version: 3.9 + python-version: "3.10.11" - uses: actions/checkout@v4 @@ -208,13 +208,13 @@ jobs: - name: Install wheel if: startsWith(matrix.os, 'ubuntu') run: | - pip install *cp39*manylinux*x86_64.whl + pip install *cp310*manylinux*x86_64.whl - name: Install wheel if: startsWith(matrix.os, 'windows') shell: bash run: | - pip install *cp39*win*.whl + pip install *cp310*win*.whl - name: Run tests shell: bash @@ -228,10 +228,10 @@ jobs: steps: - uses: actions/checkout@v4 - - name: Set up Python 3.9 + - name: Set up Python 3.10.11 uses: actions/setup-python@v5 with: - python-version: 3.9 + python-version: "3.10.11" - name: Install dependencies run: | @@ -307,10 +307,10 @@ jobs: steps: - uses: actions/checkout@v4 - - name: Set up Python 3.9 + - name: Set up Python 3.10.11 uses: actions/setup-python@v5 with: - python-version: 3.9 + python-version: "3.10.11" - name: Download CTranslate2 wheels uses: actions/download-artifact@v4 @@ -321,7 +321,7 @@ jobs: - name: Install CTranslate2 wheel run: | - pip install *cp39*manylinux*x86_64.whl + pip install *cp310*manylinux*x86_64.whl - name: Install dependencies to build docs working-directory: docs diff --git a/python/tools/prepare_build_environment_linux.sh b/python/tools/prepare_build_environment_linux.sh index 61ac171a3..461fb3699 100755 --- a/python/tools/prepare_build_environment_linux.sh +++ b/python/tools/prepare_build_environment_linux.sh @@ -19,20 +19,21 @@ if [ "$CIBW_ARCHS" == "aarch64" ]; then rm -r OpenBLAS-* else - # Install CUDA 12.2: + # Install CUDA 12.4: yum-config-manager --add-repo https://developer.download.nvidia.com/compute/cuda/repos/rhel8/x86_64/cuda-rhel8.repo # error mirrorlist.centos.org doesn't exists anymore. sed -i s/mirror.centos.org/vault.centos.org/g /etc/yum.repos.d/*.repo sed -i s/^#.*baseurl=http/baseurl=http/g /etc/yum.repos.d/*.repo sed -i s/^mirrorlist=http/#mirrorlist=http/g /etc/yum.repos.d/*.repo yum install --setopt=obsoletes=0 -y \ - cuda-nvcc-12-2-12.2.140-1 \ - cuda-cudart-devel-12-2-12.2.140-1 \ - libcurand-devel-12-2-10.3.3.141-1 \ + cuda-nvcc-12-4-12.4.99-1 \ + cuda-cudart-devel-12-4-12.4.99-1 \ + libcurand-devel-12-4-10.3.5.119-1 \ libcudnn9-devel-cuda-12-9.1.0.70-1 \ - libcublas-devel-12-2-12.2.5.6-1 \ - libnccl-devel-2.19.3-1+cuda12.2 - ln -s cuda-12.2 /usr/local/cuda + libcublas-devel-12-4-12.4.2.65-1 \ + libnccl-2.20.5-1+cuda12.4 \ + libnccl-devel-2.20.5-1+cuda12.4 + ln -s cuda-12.4 /usr/local/cuda ONEAPI_VERSION=2023.2.0 yum-config-manager --add-repo https://yum.repos.intel.com/oneapi diff --git a/python/tools/prepare_build_environment_macos.sh b/python/tools/prepare_build_environment_macos.sh index 75d47c91b..c117608af 100755 --- a/python/tools/prepare_build_environment_macos.sh +++ b/python/tools/prepare_build_environment_macos.sh @@ -1,9 +1,24 @@ -#! /bin/bash - +#!/bin/bash set -e set -x pip install "cmake==3.18.4" +brew install libomp + +# Get the actual libomp path +LIBOMP_PREFIX=$(brew --prefix libomp) + +# Set environment variables +export LDFLAGS="-L${LIBOMP_PREFIX}/lib" +export CPPFLAGS="-I${LIBOMP_PREFIX}/include" +export CMAKE_PREFIX_PATH="${LIBOMP_PREFIX}" + +# Critical: Set OpenMP flags explicitly for CMake +export OpenMP_C_FLAGS="-Xpreprocessor -fopenmp -I${LIBOMP_PREFIX}/include" +export OpenMP_C_LIB_NAMES="omp" +export OpenMP_CXX_FLAGS="-Xpreprocessor -fopenmp -I${LIBOMP_PREFIX}/include" +export OpenMP_CXX_LIB_NAMES="omp" +export OpenMP_omp_LIBRARY="${LIBOMP_PREFIX}/lib/libomp.dylib" mkdir build-release && cd build-release @@ -26,7 +41,19 @@ else wget -q https://github.com/oneapi-src/oneDNN/archive/refs/tags/v${ONEDNN_VERSION}.tar.gz tar xf *.tar.gz && rm *.tar.gz cd oneDNN-* - cmake -DCMAKE_BUILD_TYPE=Release -DONEDNN_LIBRARY_TYPE=STATIC -DONEDNN_BUILD_EXAMPLES=OFF -DONEDNN_BUILD_TESTS=OFF -DONEDNN_ENABLE_WORKLOAD=INFERENCE -DONEDNN_ENABLE_PRIMITIVE="CONVOLUTION;REORDER" -DONEDNN_BUILD_GRAPH=OFF . + cmake -DCMAKE_POLICY_VERSION_MINIMUM=3.5 \ + -DCMAKE_BUILD_TYPE=Release \ + -DONEDNN_LIBRARY_TYPE=STATIC \ + -DONEDNN_BUILD_EXAMPLES=OFF \ + -DONEDNN_BUILD_TESTS=OFF \ + -DONEDNN_ENABLE_WORKLOAD=INFERENCE \ + -DONEDNN_ENABLE_PRIMITIVE="CONVOLUTION;REORDER" \ + -DONEDNN_BUILD_GRAPH=OFF \ + -DOpenMP_C_FLAGS="${OpenMP_C_FLAGS}" \ + -DOpenMP_CXX_FLAGS="${OpenMP_CXX_FLAGS}" \ + -DOpenMP_omp_LIBRARY="${OpenMP_omp_LIBRARY}" \ + -DCMAKE_C_FLAGS="${CPPFLAGS}" \ + -DCMAKE_CXX_FLAGS="${CPPFLAGS}" . make -j$(sysctl -n hw.physicalcpu_max) install cd .. rm -r oneDNN-* @@ -35,7 +62,15 @@ else fi -cmake -DCMAKE_BUILD_TYPE=Release -DBUILD_CLI=OFF -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON $CMAKE_EXTRA_OPTIONS .. +cmake -DCMAKE_POLICY_VERSION_MINIMUM=3.5 \ + -DCMAKE_BUILD_TYPE=Release \ + -DBUILD_CLI=OFF \ + -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON \ + -DOpenMP_C_FLAGS="${OpenMP_C_FLAGS}" \ + -DOpenMP_CXX_FLAGS="${OpenMP_CXX_FLAGS}" \ + -DOpenMP_omp_LIBRARY="${OpenMP_omp_LIBRARY}" \ + $CMAKE_EXTRA_OPTIONS .. + VERBOSE=1 make -j$(sysctl -n hw.physicalcpu_max) install cd .. rm -r build-release diff --git a/python/tools/prepare_build_environment_windows.sh b/python/tools/prepare_build_environment_windows.sh index 8b55abf56..f2fcc71dc 100755 --- a/python/tools/prepare_build_environment_windows.sh +++ b/python/tools/prepare_build_environment_windows.sh @@ -3,9 +3,10 @@ set -e set -x -CUDA_ROOT="C:/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v12.2" -curl --netrc-optional -L -nv -o cuda.exe https://developer.download.nvidia.com/compute/cuda/12.2.2/local_installers/cuda_12.2.2_537.13_windows.exe -./cuda.exe -s nvcc_12.2 cudart_12.2 cublas_dev_12.2 curand_dev_12.2 +CUDA_ROOT="C:/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v12.4" +curl --netrc-optional -L -nv -o cuda.exe https://developer.download.nvidia.com/compute/cuda/12.4.0/local_installers/cuda_12.4.0_551.61_windows.exe +./cuda.exe -s nvcc_12.4 cudart_12.4 cublas_dev_12.4 curand_dev_12.4 + rm cuda.exe CUDNN_ROOT="C:/Program Files/NVIDIA/CUDNN/v9.1" @@ -30,7 +31,7 @@ cp -r "$CUDNN_ROOT"/* "$CUDA_ROOT" rm cudnn.exe # See https://github.com/oneapi-src/oneapi-ci for installer URLs -curl --netrc-optional -L -nv -o webimage.exe https://registrationcenter-download.intel.com/akdlm/irc_nas/19078/w_BaseKit_p_2023.0.0.25940_offline.exe +curl --netrc-optional -L -nv -o webimage.exe https://registrationcenter-download.intel.com/akdlm/IRC_NAS/62641e01-1e8d-4ace-91d6-ae03f7f8a71f/w_BaseKit_p_2024.0.0.49563_offline.exe ./webimage.exe -s -x -f webimage_extracted --log extract.log rm webimage.exe ./webimage_extracted/bootstrapper.exe -s --action install --components="intel.oneapi.win.mkl.devel" --eula=accept -p=NEED_VS2017_INTEGRATION=0 -p=NEED_VS2019_INTEGRATION=0 --log-dir=. @@ -53,5 +54,5 @@ rm -r build cp README.md python/ cp $CTRANSLATE2_ROOT/bin/ctranslate2.dll python/ctranslate2/ -cp "C:/Program Files (x86)/Intel/oneAPI/compiler/latest/windows/redist/intel64_win/compiler/libiomp5md.dll" python/ctranslate2/ +cp "C:/Program Files (x86)/Intel/oneAPI/compiler/2024.0/bin/libiomp5md.dll" python/ctranslate2/ cp "$CUDA_ROOT/bin/cudnn64_9.dll" python/ctranslate2/