A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 23:30 UTC
[Bug] [V1] Try fetching stop_reason from EngineOutput before checking the request (#13108)
bnellnm committed
Feb 12, 2025 at 05:39 UTC
f4d97e4fc276b13e1a4ec18f35239fd48695667d
1 file changed
+6
-4
vllm/v1/engine/output_processor.py
+6
-4
@@ -2,7 +2,7 @@
2
3
import asyncio
4
from dataclasses import dataclass
5
-from typing import Dict, List, Optional
5
+from typing import Dict, List, Optional, Union
6
7
from vllm.outputs import RequestOutput
8
from vllm.sampling_params import RequestOutputKind
@@ -164,6 +164,7 @@ class OutputProcessor:
164
165
new_token_ids = engine_core_output.new_token_ids
166
finish_reason = engine_core_output.finish_reason
167
+ stop_reason = engine_core_output.stop_reason
168
169
# TODO(andy): prompt logprobs + chunked prefill can
170
# result in engine core returning an output for a
@@ -181,9 +182,10 @@ class OutputProcessor:
182
183
# 2) Detokenize the token ids into text and check for stop
184
# strings.
184
- stop_reason = req_state.detokenizer.update(new_token_ids)
185
- if stop_reason:
185
+ stop_string = req_state.detokenizer.update(new_token_ids)
186
+ if stop_string and finish_reason != FinishReason.STOP:
187
finish_reason = FinishReason.STOP
188
+ stop_reason = stop_string
189
190
# 3) Compute sample and prompt logprobs for request,
191
# if required.
@@ -250,7 +252,7 @@ class OutputProcessor:
252
request_state: RequestState,
253
new_token_ids: List[int],
254
finish_reason: Optional[FinishReason],
253
- stop_reason: Optional[str],
255
+ stop_reason: Union[int, str, None],
256
) -> Optional[RequestOutput]:
257
258
finished = finish_reason is not None