A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 09:30 UTC
[BugFix] Fix and simplify completion API usage streaming (#9475)
Nick Hill committed
Oct 18, 2024 at 15:10 UTC
25aeb7d4c9e1b2b8d4a28c2797569a1f8edfccc5
1 file changed
+61
-62
vllm/entrypoints/openai/serving_completion.py
+61
-62
@@ -258,6 +258,14 @@ class OpenAIServingCompletion(OpenAIServing):
258
has_echoed = [False] * num_choices * num_prompts
259
num_prompt_tokens = [0] * num_prompts
260
261
+ stream_options = request.stream_options
262
+ if stream_options:
263
+ include_usage = stream_options.include_usage
264
+ include_continuous_usage = include_usage and \
265
+ stream_options.continuous_usage_stats
266
+ else:
267
+ include_usage, include_continuous_usage = False, False
268
+
269
try:
270
async for prompt_idx, res in result_generator:
271
prompt_token_ids = res.prompt_token_ids
@@ -276,28 +284,25 @@ class OpenAIServingCompletion(OpenAIServing):
284
i = output.index + prompt_idx * num_choices
285
286
assert request.max_tokens is not None
279
- if request.echo and request.max_tokens == 0:
287
+ if request.echo and not has_echoed[i]:
288
assert prompt_token_ids is not None
289
assert prompt_text is not None
282
- # only return the prompt
283
- delta_text = prompt_text
284
- delta_token_ids = prompt_token_ids
285
- out_logprobs = prompt_logprobs
286
- has_echoed[i] = True
287
- elif (request.echo and request.max_tokens > 0
288
- and not has_echoed[i]):
289
- assert prompt_token_ids is not None
290
- assert prompt_text is not None
291
- assert prompt_logprobs is not None
292
- # echo the prompt and first token
293
- delta_text = prompt_text + output.text
294
- delta_token_ids = [
295
- *prompt_token_ids, *output.token_ids
296
- ]
297
- out_logprobs = [
298
- *prompt_logprobs,
299
- *(output.logprobs or []),
300
- ]
290
+ if request.max_tokens == 0:
291
+ # only return the prompt
292
+ delta_text = prompt_text
293
+ delta_token_ids = prompt_token_ids
294
+ out_logprobs = prompt_logprobs
295
+ else:
296
+ assert prompt_logprobs is not None
297
+ # echo the prompt and first token
298
+ delta_text = prompt_text + output.text
299
+ delta_token_ids = [
300
+ *prompt_token_ids, *output.token_ids
301
+ ]
302
+ out_logprobs = [
303
+ *prompt_logprobs,
304
+ *(output.logprobs or []),
305
+ ]
306
has_echoed[i] = True
307
else:
308
# return just the delta
@@ -341,45 +346,39 @@ class OpenAIServingCompletion(OpenAIServing):
346
stop_reason=stop_reason,
347
)
348
])
344
- if (request.stream_options
345
- and request.stream_options.include_usage):
346
- if (request.stream_options.continuous_usage_stats
347
- or output.finish_reason is not None):
348
- prompt_tokens = num_prompt_tokens[prompt_idx]
349
- completion_tokens = previous_num_tokens[i]
350
- usage = UsageInfo(
351
- prompt_tokens=prompt_tokens,
352
- completion_tokens=completion_tokens,
353
- total_tokens=prompt_tokens + completion_tokens,
354
- )
355
- if request.stream_options.continuous_usage_stats:
356
- chunk.usage = usage
357
- else:
358
- chunk.usage = None
349
+ if include_continuous_usage:
350
+ prompt_tokens = num_prompt_tokens[prompt_idx]
351
+ completion_tokens = previous_num_tokens[i]
352
+ chunk.usage = UsageInfo(
353
+ prompt_tokens=prompt_tokens,
354
+ completion_tokens=completion_tokens,
355
+ total_tokens=prompt_tokens + completion_tokens,
356
+ )
357
358
response_json = chunk.model_dump_json(exclude_unset=False)
359
yield f"data: {response_json}\n\n"
360
363
- if (request.stream_options
364
- and request.stream_options.include_usage):
361
+ total_prompt_tokens = sum(num_prompt_tokens)
362
+ total_completion_tokens = sum(previous_num_tokens)
363
+ final_usage_info = UsageInfo(
364
+ prompt_tokens=total_prompt_tokens,
365
+ completion_tokens=total_completion_tokens,
366
+ total_tokens=total_prompt_tokens + total_completion_tokens)
367
+
368
+ if include_usage:
369
final_usage_chunk = CompletionStreamResponse(
370
id=request_id,
371
created=created_time,
372
model=model_name,
373
choices=[],
370
- usage=usage,
374
+ usage=final_usage_info,
375
)
376
final_usage_data = (final_usage_chunk.model_dump_json(
377
exclude_unset=False, exclude_none=True))
378
yield f"data: {final_usage_data}\n\n"
379
380
# report to FastAPI middleware aggregate usage across all choices
377
- total_prompt_tokens = sum(num_prompt_tokens)
378
- total_completion_tokens = sum(previous_num_tokens)
379
- request_metadata.final_usage_info = UsageInfo(
380
- prompt_tokens=total_prompt_tokens,
381
- completion_tokens=total_completion_tokens,
382
- total_tokens=total_prompt_tokens + total_completion_tokens)
381
+ request_metadata.final_usage_info = final_usage_info
382
383
except ValueError as e:
384
# TODO: Use a vllm-specific Validation Error
@@ -413,26 +412,26 @@ class OpenAIServingCompletion(OpenAIServing):
412
413
for output in final_res.outputs:
414
assert request.max_tokens is not None
416
- if request.echo and request.max_tokens == 0:
417
- assert prompt_text is not None
418
- token_ids = prompt_token_ids
419
- out_logprobs = prompt_logprobs
420
- output_text = prompt_text
421
- elif request.echo and request.max_tokens > 0:
415
+ if request.echo:
416
assert prompt_text is not None
423
- token_ids = [*prompt_token_ids, *output.token_ids]
424
-
425
- if request.logprobs is None:
426
- out_logprobs = None
417
+ if request.max_tokens == 0:
418
+ token_ids = prompt_token_ids
419
+ out_logprobs = prompt_logprobs
420
+ output_text = prompt_text
421
else:
428
- assert prompt_logprobs is not None
429
- assert output.logprobs is not None
430
- out_logprobs = [
431
- *prompt_logprobs,
432
- *output.logprobs,
433
- ]
434
-
435
- output_text = prompt_text + output.text
422
+ token_ids = [*prompt_token_ids, *output.token_ids]
423
+
424
+ if request.logprobs is None:
425
+ out_logprobs = None
426
+ else:
427
+ assert prompt_logprobs is not None
428
+ assert output.logprobs is not None
429
+ out_logprobs = [
430
+ *prompt_logprobs,
431
+ *output.logprobs,
432
+ ]
433
+
434
+ output_text = prompt_text + output.text
435
else:
436
token_ids = output.token_ids
437
out_logprobs = output.logprobs