@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / f4d97e4fc2
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 15, 23:30 UTC

[Bug] [V1] Try fetching stop_reason from EngineOutput before checking the request (#13108)

bnellnm committed Feb 12, 2025 at 05:39 UTC f4d97e4fc276b13e1a4ec18f35239fd48695667d
1 file changed +6 -4
vllm/v1/engine/output_processor.py
+6 -4
@@ -2,7 +2,7 @@
2
3 import asyncio
4 from dataclasses import dataclass
5 -from typing import Dict, List, Optional
5 +from typing import Dict, List, Optional, Union
6
7 from vllm.outputs import RequestOutput
8 from vllm.sampling_params import RequestOutputKind
@@ -164,6 +164,7 @@ class OutputProcessor:
164
165 new_token_ids = engine_core_output.new_token_ids
166 finish_reason = engine_core_output.finish_reason
167 + stop_reason = engine_core_output.stop_reason
168
169 # TODO(andy): prompt logprobs + chunked prefill can
170 # result in engine core returning an output for a
@@ -181,9 +182,10 @@ class OutputProcessor:
182
183 # 2) Detokenize the token ids into text and check for stop
184 # strings.
184 - stop_reason = req_state.detokenizer.update(new_token_ids)
185 - if stop_reason:
185 + stop_string = req_state.detokenizer.update(new_token_ids)
186 + if stop_string and finish_reason != FinishReason.STOP:
187 finish_reason = FinishReason.STOP
188 + stop_reason = stop_string
189
190 # 3) Compute sample and prompt logprobs for request,
191 # if required.
@@ -250,7 +252,7 @@ class OutputProcessor:
252 request_state: RequestState,
253 new_token_ids: List[int],
254 finish_reason: Optional[FinishReason],
253 - stop_reason: Optional[str],
255 + stop_reason: Union[int, str, None],
256 ) -> Optional[RequestOutput]:
257
258 finished = finish_reason is not None