A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 18:30 UTC
[Bugfix][V0] Multi-sequence logprobs streaming edge case (#15259)
Signed-off-by: Andy Lo <andy@mistral.ai>
Andy Lo committed
Mar 22, 2025 at 05:35 UTC
df1430265c0bda5fe02c43005352cce7a8aa9562
1 file changed
+6
-1
vllm/outputs.py
+6
-1
@@ -223,7 +223,12 @@ class RequestOutput:
223
if delta:
224
# Slice logprobs delta if applicable
225
if output_logprobs:
226
- output_logprobs = output_logprobs[-num_output_tokens:]
226
+ # num_output_tokens can be 0 when n > 1 and request finishes
227
+ # before the others
228
+ if num_output_tokens > 0:
229
+ output_logprobs = output_logprobs[-num_output_tokens:]
230
+ else:
231
+ output_logprobs = None
232
# Don't include prompt if this is after the first output
233
# containing decode token ids
234
if include_prompt and seq.get_output_len() > num_output_tokens: