Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Add GPU support #1056

Merged
merged 5 commits into from
Feb 13, 2024
Merged

Add GPU support #1056

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
53f9862
Update dev.sh
Smartappli Feb 4, 2024
dd278f0
Update deploy.sh
Smartappli Feb 4, 2024
128e76c
Update serge.env
Smartappli Feb 4, 2024
4d09928
Merge branch 'gpu-support' into gpu
gaby Feb 5, 2024
ef4cc67
Merge branch 'gpu-support' into gpu
Smartappli Feb 5, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 7 additions & 2 deletions scripts/deploy.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,14 +19,19 @@ detect_cpu_features() {
echo "basic"
fi
}

# Check if the CPU architecture is aarch64/arm64
if [ "$cpu_arch" = "aarch64" ]; then
pip_command="python -m pip install -v llama-cpp-python==$LLAMA_PYTHON_VERSION --only-binary=:all: --extra-index-url=https://gaby.github.io/arm64-wheels/"
else
# Use @smartappli provided wheels
cpu_feature=$(detect_cpu_features)
pip_command="python -m pip install -v llama-cpp-python==$LLAMA_PYTHON_VERSION --only-binary=:all: --extra-index-url=https://smartappli.github.io/llama-cpp-python-cuBLAS-wheels/$cpu_feature/cpu"
if [ "$SERGE_GPU_NVIDIA_SUPPORT" = true ]; then
pip_command="python -m pip install -v llama-cpp-python==$LLAMA_PYTHON_VERSION --only-binary=:all: --extra-index-url=https://smartappli.github.io/llama-cpp-python-cuBLAS-wheels/$cpu_feature/cu122"
elif [ "$SERGE_GPU_AMD_SUPPORT" = true ]; then
pip_command="python -m pip install -v llama-cpp-python==$LLAMA_PYTHON_VERSION --only-binary=:all: --extra-index-url=https://smartappli.github.io/llama-cpp-python-cuBLAS-wheels/$cpu_feature/rocm5.6.1"
else
pip_command="python -m pip install -v llama-cpp-python==$LLAMA_PYTHON_VERSION --only-binary=:all: --extra-index-url=https://smartappli.github.io/llama-cpp-python-cuBLAS-wheels/$cpu_feature/cpu"
fi
fi

echo "Recommended install command for llama-cpp-python: $pip_command"
Expand Down
8 changes: 7 additions & 1 deletion scripts/dev.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,13 @@ if [ "$cpu_arch" = "aarch64" ]; then
else
# Use @smartappli provided wheels
cpu_feature=$(detect_cpu_features)
pip_command="python -m pip install -v llama-cpp-python==$LLAMA_PYTHON_VERSION --only-binary=:all: --extra-index-url=https://smartappli.github.io/llama-cpp-python-cuBLAS-wheels/$cpu_feature/cpu"
if [ "$SERGE_GPU_NVIDIA_SUPPORT" = true ]; then
pip_command="python -m pip install -v llama-cpp-python==$LLAMA_PYTHON_VERSION --only-binary=:all: --extra-index-url=https://smartappli.github.io/llama-cpp-python-cuBLAS-wheels/$cpu_feature/cu122"
elif [ "$SERGE_GPU_AMD_SUPPORT" = true ]; then
pip_command="python -m pip install -v llama-cpp-python==$LLAMA_PYTHON_VERSION --only-binary=:all: --extra-index-url=https://smartappli.github.io/llama-cpp-python-cuBLAS-wheels/$cpu_feature/rocm5.6.1"
else
pip_command="python -m pip install -v llama-cpp-python==$LLAMA_PYTHON_VERSION --only-binary=:all: --extra-index-url=https://smartappli.github.io/llama-cpp-python-cuBLAS-wheels/$cpu_feature/cpu"
fi
fi

echo "Recommended install command for llama-cpp-python: $pip_command"
Expand Down
3 changes: 2 additions & 1 deletion scripts/serge.env
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@

LLAMA_PYTHON_VERSION=0.2.38
SERGE_GPU_NVIDIA_SUPPORT=false
SERGE_GPU_AMD_SUPPORT=false
SERGE_ENABLE_IPV6=false
Loading