diff --git a/docs/CN/source/tutorial/reasoning_parser.rst b/docs/CN/source/tutorial/reasoning_parser.rst index c00fae32d..11afa5f47 100644 --- a/docs/CN/source/tutorial/reasoning_parser.rst +++ b/docs/CN/source/tutorial/reasoning_parser.rst @@ -175,6 +175,20 @@ GPT-OSS // 答案块 {"choices": [{"delta": {"content": "答案片段"}}]} +启用 ``--reasoning_parser`` 后，Chat Completions 的 usage 会返回生成的推理 +token 数量（不包含推理分隔符）： + +.. code-block:: json + + { + "usage": { + "completion_tokens": 128, + "completion_tokens_details": {"reasoning_tokens": 96} + } + } + +Responses API 会在 ``usage.output_tokens_details.reasoning_tokens`` 中返回相同计数。 + 高级功能 -------- diff --git a/docs/EN/source/tutorial/reasoning_parser.rst b/docs/EN/source/tutorial/reasoning_parser.rst index 156dff4d2..0b572c65e 100644 --- a/docs/EN/source/tutorial/reasoning_parser.rst +++ b/docs/EN/source/tutorial/reasoning_parser.rst @@ -175,6 +175,21 @@ Response Format // Answer chunk {"choices": [{"delta": {"content": "Answer fragment"}}]} +When ``--reasoning_parser`` is enabled, Chat Completions usage reports the +number of generated reasoning tokens (excluding reasoning delimiters): + +.. code-block:: json + + { + "usage": { + "completion_tokens": 128, + "completion_tokens_details": {"reasoning_tokens": 96} + } + } + +The Responses API exposes the same count as +``usage.output_tokens_details.reasoning_tokens``. + Advanced Features ----------------- diff --git a/lightllm/server/api_models.py b/lightllm/server/api_models.py index 2246149af..bfb19ff0e 100644 --- a/lightllm/server/api_models.py +++ b/lightllm/server/api_models.py @@ -292,11 +292,16 @@ class PromptTokensDetails(BaseModel): audio_tokens: int = 0 +class CompletionTokensDetails(BaseModel): + reasoning_tokens: int = 0 + + class UsageInfo(BaseModel): prompt_tokens: int = 0 completion_tokens: Optional[int] = 0 total_tokens: int = 0 prompt_tokens_details: Optional[PromptTokensDetails] = None + completion_tokens_details: Optional[CompletionTokensDetails] = None class ChatMessage(BaseModel): diff --git a/lightllm/server/api_openai.py b/lightllm/server/api_openai.py index e89ec5c5a..1878e60f1 100644 --- a/lightllm/server/api_openai.py +++ b/lightllm/server/api_openai.py @@ -51,6 +51,7 @@ ToolCall, UsageInfo, PromptTokensDetails, + CompletionTokensDetails, ChatMessage, ChatCompletionResponseChoice, ChatCompletionResponse, @@ -169,19 +170,21 @@ def _process_reasoning_stream( index: int, delta: str, reasoning_parser_dict: Dict[int, ReasoningParser], - content: Dict[str, Any], + metadata: Dict[str, Any], request: ChatCompletionRequest, ) -> tuple[Optional[str], str]: - """Process reasoning content in streaming response""" + """Process reasoning content and update its token usage.""" if index not in reasoning_parser_dict: - request_enable_reasoning = _is_force_thinking_mode(request) reasoning_parser_dict[index] = ReasoningParser( get_env_start_args().reasoning_parser, request.stream_reasoning, - request_enable_reasoning, + _is_force_thinking_mode(request), ) - reasoning_parser = reasoning_parser_dict[index] - return reasoning_parser.parse_stream_chunk(delta) + parser = reasoning_parser_dict[index] + token_id = metadata.get("id") + if token_id is not None: + parser.update_reasoning_token_count(int(token_id)) + return parser.parse_stream_chunk(delta) def _process_tools_stream(index: int, delta: str, parser_dict: Dict, request: ChatCompletionRequest): @@ -357,18 +360,33 @@ async def chat_completions_impl(request: ChatCompletionRequest, raw_request: Req # Non-streaming case if not request.stream: + reasoning_parser = get_env_start_args().reasoning_parser + request_enable_reasoning = _is_force_thinking_mode(request) if reasoning_parser else False final_output_dict = collections.defaultdict(list) count_output_tokens_dict = collections.defaultdict(lambda: 0) finish_reason_dict = {} prompt_tokens_dict = {} prompt_cache_len_dict = {} completion_tokens = 0 + reasoning_parser_dict: Dict[int, ReasoningParser] = {} async for sub_req_id, request_output, metadata, finish_status in results_generator: from .req_id_generator import convert_sub_id_to_group_id group_request_id = convert_sub_id_to_group_id(sub_req_id) count_output_tokens_dict[sub_req_id] += 1 final_output_dict[sub_req_id].append(request_output) + if reasoning_parser: + parser = reasoning_parser_dict.get(sub_req_id) + if parser is None: + parser = ReasoningParser( + reasoning_parser, + stream_reasoning=False, + force_reasoning=request_enable_reasoning, + ) + reasoning_parser_dict[sub_req_id] = parser + token_id = metadata.get("id") + if token_id is not None: + parser.update_reasoning_token_count(int(token_id)) if finish_status.is_finished(): finish_reason_dict[sub_req_id] = finish_status.get_finish_reason() prompt_tokens_dict[sub_req_id] = metadata["prompt_tokens"] @@ -378,12 +396,7 @@ async def chat_completions_impl(request: ChatCompletionRequest, raw_request: Req prompt_tokens = prompt_tokens_dict[sub_ids[0]] completion_tokens = sum(count_output_tokens_dict[sub_req_id] for sub_req_id in sub_ids) cached_tokens = prompt_cache_len_dict.get(sub_ids[0], 0) - usage = UsageInfo( - prompt_tokens=prompt_tokens, - completion_tokens=completion_tokens, - total_tokens=prompt_tokens + completion_tokens, - prompt_tokens_details=PromptTokensDetails(cached_tokens=cached_tokens), - ) + reasoning_tokens = sum(reasoning_parser_dict[sub_req_id].reasoning_tokens for sub_req_id in sub_ids) for i in range(request.n): sub_req_id = sub_ids[i] @@ -392,15 +405,9 @@ async def chat_completions_impl(request: ChatCompletionRequest, raw_request: Req # Handle reasoning content reasoning_text = None - reasoning_parser = get_env_start_args().reasoning_parser if reasoning_parser: - request_enable_reasoning = _is_force_thinking_mode(request) try: - parser = ReasoningParser( - model_type=reasoning_parser, - stream_reasoning=False, - force_reasoning=request_enable_reasoning, - ) + parser = reasoning_parser_dict[sub_req_id] reasoning_text, text = parser.parse_non_stream(text) except Exception as e: logger.error(f"Reasoning parsing error: {e}") @@ -454,6 +461,16 @@ async def chat_completions_impl(request: ChatCompletionRequest, raw_request: Req finish_reason=finish_reason, ) choices.append(choice) + completion_tokens_details = None + if reasoning_parser: + completion_tokens_details = CompletionTokensDetails(reasoning_tokens=reasoning_tokens) + usage = UsageInfo( + prompt_tokens=prompt_tokens, + completion_tokens=completion_tokens, + total_tokens=prompt_tokens + completion_tokens, + prompt_tokens_details=PromptTokensDetails(cached_tokens=cached_tokens), + completion_tokens_details=completion_tokens_details, + ) resp = ChatCompletionResponse( id=group_request_id, created=created_time, model=request.model, choices=choices, usage=usage ) @@ -511,7 +528,7 @@ async def stream_results() -> AsyncGenerator[bytes, None]: # Handle reasoning content if get_env_start_args().reasoning_parser: reasoning_text, delta = _process_reasoning_stream( - choice_index, delta, reasoning_parser_dict, request_output, request + choice_index, delta, reasoning_parser_dict, metadata, request ) if reasoning_text: if request.separate_reasoning: @@ -747,11 +764,17 @@ async def stream_results() -> AsyncGenerator[bytes, None]: ) yield f"data: {_serialize_sse_chunk(final_chunk, _final_choice_nulls)}\n\n" + reasoning_parser = get_env_start_args().reasoning_parser + completion_tokens_details = None + if reasoning_parser: + reasoning_tokens = sum(parser.reasoning_tokens for parser in reasoning_parser_dict.values()) + completion_tokens_details = CompletionTokensDetails(reasoning_tokens=reasoning_tokens) usage = UsageInfo( prompt_tokens=prompt_tokens, completion_tokens=completion_tokens, total_tokens=prompt_tokens + completion_tokens, prompt_tokens_details=PromptTokensDetails(cached_tokens=cached_tokens), + completion_tokens_details=completion_tokens_details, ) usage_chunk = ChatCompletionStreamResponse( id=chat_completion_id, diff --git a/lightllm/server/api_responses.py b/lightllm/server/api_responses.py index cad3f4fe3..e1b79acc7 100644 --- a/lightllm/server/api_responses.py +++ b/lightllm/server/api_responses.py @@ -269,11 +269,12 @@ def _usage_to_responses(usage: Dict[str, Any]) -> Dict[str, Any]: input_tokens = int(usage.get("prompt_tokens", 0)) output_tokens = int(usage.get("completion_tokens", 0)) cached = int((usage.get("prompt_tokens_details") or {}).get("cached_tokens", 0) or 0) + reasoning = int((usage.get("completion_tokens_details") or {}).get("reasoning_tokens", 0) or 0) return { "input_tokens": input_tokens, "input_tokens_details": {"cached_tokens": cached}, "output_tokens": output_tokens, - "output_tokens_details": {"reasoning_tokens": 0}, + "output_tokens_details": {"reasoning_tokens": reasoning}, "total_tokens": input_tokens + output_tokens, } diff --git a/lightllm/server/reasoning_parser.py b/lightllm/server/reasoning_parser.py index 8a8d07355..cfd2e5a3b 100644 --- a/lightllm/server/reasoning_parser.py +++ b/lightllm/server/reasoning_parser.py @@ -15,6 +15,8 @@ from dataclasses import dataclass from typing import Iterator, List, Tuple, Dict, Optional, Type +from lightllm.utils.config_utils import get_token_id + @dataclass class Event: @@ -926,6 +928,7 @@ def __init__( if not model_type: raise ValueError("Model type must be specified") + requested_force_reasoning = force_reasoning detector_class = self.DetectorMap.get(model_type.lower()) if not detector_class: raise ValueError(f"Unsupported model type: {model_type}") @@ -943,6 +946,20 @@ def __init__( kwargs["force_reasoning"] = force_reasoning self.detector = detector_class(**kwargs) + self.reasoning_tokens = 0 + reasoning_enabled = self.detector._in_reasoning or requested_force_reasoning is True + self._counting_reasoning = reasoning_enabled and model_type.lower() != "minimax-append-think" + self._think_end_token_id = get_token_id(self.detector.think_end_token) + + def update_reasoning_token_count(self, token_id: int) -> None: + """Count one generated token until the reasoning closing delimiter.""" + if not self._counting_reasoning: + return + + if token_id == self._think_end_token_id: + self._counting_reasoning = False + else: + self.reasoning_tokens += 1 def parse_non_stream(self, full_text: str) -> Tuple[Optional[str], Optional[str]]: """Non-streaming call: one-time parsing""" diff --git a/lightllm/utils/config_utils.py b/lightllm/utils/config_utils.py index bf247d090..6695f0ec4 100644 --- a/lightllm/utils/config_utils.py +++ b/lightllm/utils/config_utils.py @@ -319,6 +319,12 @@ def get_eos_token_ids(model_path: str) -> Optional[List[int]]: return +def get_token_id(token: str) -> int: + from lightllm.server.build_prompt import tokenizer + + return int(tokenizer.convert_tokens_to_ids(token)) + + def get_model_architectures(model_path: str): try: config_json = get_config_json(model_path) diff --git a/unit_tests/server/test_reasoning_token_usage.py b/unit_tests/server/test_reasoning_token_usage.py new file mode 100644 index 000000000..438525ebf --- /dev/null +++ b/unit_tests/server/test_reasoning_token_usage.py @@ -0,0 +1,54 @@ +from unittest.mock import patch + +from lightllm.server.reasoning_parser import ReasoningParser + + +def _create_parser(model_type: str, force_reasoning: bool) -> ReasoningParser: + with patch("lightllm.server.reasoning_parser.get_token_id", return_value=99): + return ReasoningParser(model_type, force_reasoning=force_reasoning) + + +def _count_tokens(parser: ReasoningParser, token_ids: list[int]) -> int: + for token_id in token_ids: + parser.update_reasoning_token_count(token_id) + return parser.reasoning_tokens + + +def test_counts_tokens_until_single_token_closing_marker(): + parser = _create_parser("qwen3", force_reasoning=True) + + reasoning_tokens = _count_tokens(parser, [1, 2, 3, 99, 4]) + + assert reasoning_tokens == 3 + + +def test_counts_all_tokens_when_generation_is_truncated_before_closing_marker(): + parser = _create_parser("qwen3", force_reasoning=True) + + reasoning_tokens = _count_tokens(parser, [1, 2]) + + assert reasoning_tokens == 2 + + +def test_does_not_count_when_reasoning_is_disabled(): + parser = _create_parser("qwen3", force_reasoning=False) + + reasoning_tokens = _count_tokens(parser, [1, 2, 99]) + + assert reasoning_tokens == 0 + + +def test_counts_for_always_reasoning_detector(): + parser = _create_parser("deepseek-r1", force_reasoning=False) + + reasoning_tokens = _count_tokens(parser, [1, 2, 99]) + + assert reasoning_tokens == 2 + + +def test_minimax_append_think_output_is_not_counted_as_reasoning(): + parser = _create_parser("minimax-append-think", force_reasoning=True) + + reasoning_tokens = _count_tokens(parser, [1, 2, 99]) + + assert reasoning_tokens == 0