diff --git a/docs/source/basic_tutorials/launcher.md b/docs/source/basic_tutorials/launcher.md index bafe3669219..712b4fc43d8 100644 --- a/docs/source/basic_tutorials/launcher.md +++ b/docs/source/basic_tutorials/launcher.md @@ -354,6 +354,14 @@ Options: [env: NGROK_EDGE=] +``` +## TOKENIZER_CONFIG_PATH +```shell + --tokenizer-config-path + The path to the tokenizer config file. This path is used to load the tokenizer configuration which may include a `chat_template`. If not provided, the default config will be used from the model hub + + [env: TOKENIZER_CONFIG_PATH=] + ``` ## ENV ```shell diff --git a/launcher/src/main.rs b/launcher/src/main.rs index 09657c91550..f0e45141157 100644 --- a/launcher/src/main.rs +++ b/launcher/src/main.rs @@ -368,6 +368,11 @@ struct Args { #[clap(long, env)] ngrok_edge: Option, + /// The path to the tokenizer config file. This path is used to load the tokenizer configuration which may + /// include a `chat_template`. If not provided, the default config will be used from the model hub. + #[clap(long, env)] + tokenizer_config_path: Option, + /// Display a lot of information about your runtime environment #[clap(long, short, action)] env: bool, @@ -1016,6 +1021,12 @@ fn spawn_webserver( args.model_id, ]; + // Tokenizer config path + if let Some(ref tokenizer_config_path) = args.tokenizer_config_path { + router_args.push("--tokenizer-config-path".to_string()); + router_args.push(tokenizer_config_path.to_string()); + } + // Model optional max batch total tokens if let Some(max_batch_total_tokens) = args.max_batch_total_tokens { router_args.push("--max-batch-total-tokens".to_string());