diff --git a/docs/model_support.md b/docs/model_support.md new file mode 100644 index 0000000000..3c67a89f52 --- /dev/null +++ b/docs/model_support.md @@ -0,0 +1,10 @@ +## How to Support a New Model + +To support a new model in SGLang, you only need to add a single file under [SGLang Models Directory](https://github.com/sgl-project/sglang/tree/main/python/sglang/srt/models). + +You can learn from existing model implementations and create new files for the new models. Most models are based on the transformer architecture, making them very similar. + +Another valuable resource is the vLLM model implementations. vLLM has extensive coverage of models, and SGLang has reused vLLM for most parts of the model implementations. This similarity makes it easy to port many models from vLLM to SGLang. + +1. Compare these two files [SGLang LLaMA Implementation](https://github.com/sgl-project/sglang/blob/main/python/sglang/srt/models/llama2.py) and [vLLM LLaMA Implementation](https://github.com/vllm-project/vllm/blob/main/vllm/model_executor/models/llama.py). This comparison will help you understand how to convert a model implementation from vLLM to SGLang. The major difference is the replacement of PagedAttention with RadixAttention. The other parts are almost identical. +2. Convert models from vLLM to SGLang by visiting the [vLLM Models Directory](https://github.com/vllm-project/vllm/tree/main/vllm/model_executor/models). diff --git a/examples/quick_start/srt_example_llava.py b/examples/quick_start/srt_example_llava.py index b2b51622e4..97812ea9f8 100644 --- a/examples/quick_start/srt_example_llava.py +++ b/examples/quick_start/srt_example_llava.py @@ -43,9 +43,11 @@ def batch(): if __name__ == "__main__": - runtime = sgl.Runtime(model_path="liuhaotian/llava-v1.5-7b", + runtime = sgl.Runtime(model_path="liuhaotian/llava-v1.6-vicuna-7b", tokenizer_path="llava-hf/llava-1.5-7b-hf") sgl.set_default_backend(runtime) + print(f"chat template: {runtime.endpoint.chat_template.name}") + # Or you can use API models # sgl.set_default_backend(sgl.OpenAI("gpt-4-vision-preview")) # sgl.set_default_backend(sgl.VertexAI("gemini-pro-vision")) diff --git a/python/pyproject.toml b/python/pyproject.toml index a5513daa76..ea55f93c05 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "sglang" -version = "0.1.11" +version = "0.1.12" description = "A structured generation langauge for LLMs." readme = "README.md" requires-python = ">=3.8" diff --git a/python/sglang/__init__.py b/python/sglang/__init__.py index e39817ab23..1f3304756e 100644 --- a/python/sglang/__init__.py +++ b/python/sglang/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.1.11" +__version__ = "0.1.12" from sglang.api import * from sglang.global_config import global_config