@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 25aeb7d4c9
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 09:30 UTC

[BugFix] Fix and simplify completion API usage streaming (#9475)

Nick Hill committed Oct 18, 2024 at 15:10 UTC 25aeb7d4c9e1b2b8d4a28c2797569a1f8edfccc5
1 file changed +61 -62
vllm/entrypoints/openai/serving_completion.py
+61 -62
@@ -258,6 +258,14 @@ class OpenAIServingCompletion(OpenAIServing):
258 has_echoed = [False] * num_choices * num_prompts
259 num_prompt_tokens = [0] * num_prompts
260
261 + stream_options = request.stream_options
262 + if stream_options:
263 + include_usage = stream_options.include_usage
264 + include_continuous_usage = include_usage and \
265 + stream_options.continuous_usage_stats
266 + else:
267 + include_usage, include_continuous_usage = False, False
268 +
269 try:
270 async for prompt_idx, res in result_generator:
271 prompt_token_ids = res.prompt_token_ids
@@ -276,28 +284,25 @@ class OpenAIServingCompletion(OpenAIServing):
284 i = output.index + prompt_idx * num_choices
285
286 assert request.max_tokens is not None
279 - if request.echo and request.max_tokens == 0:
287 + if request.echo and not has_echoed[i]:
288 assert prompt_token_ids is not None
289 assert prompt_text is not None
282 - # only return the prompt
283 - delta_text = prompt_text
284 - delta_token_ids = prompt_token_ids
285 - out_logprobs = prompt_logprobs
286 - has_echoed[i] = True
287 - elif (request.echo and request.max_tokens > 0
288 - and not has_echoed[i]):
289 - assert prompt_token_ids is not None
290 - assert prompt_text is not None
291 - assert prompt_logprobs is not None
292 - # echo the prompt and first token
293 - delta_text = prompt_text + output.text
294 - delta_token_ids = [
295 - *prompt_token_ids, *output.token_ids
296 - ]
297 - out_logprobs = [
298 - *prompt_logprobs,
299 - *(output.logprobs or []),
300 - ]
290 + if request.max_tokens == 0:
291 + # only return the prompt
292 + delta_text = prompt_text
293 + delta_token_ids = prompt_token_ids
294 + out_logprobs = prompt_logprobs
295 + else:
296 + assert prompt_logprobs is not None
297 + # echo the prompt and first token
298 + delta_text = prompt_text + output.text
299 + delta_token_ids = [
300 + *prompt_token_ids, *output.token_ids
301 + ]
302 + out_logprobs = [
303 + *prompt_logprobs,
304 + *(output.logprobs or []),
305 + ]
306 has_echoed[i] = True
307 else:
308 # return just the delta
@@ -341,45 +346,39 @@ class OpenAIServingCompletion(OpenAIServing):
346 stop_reason=stop_reason,
347 )
348 ])
344 - if (request.stream_options
345 - and request.stream_options.include_usage):
346 - if (request.stream_options.continuous_usage_stats
347 - or output.finish_reason is not None):
348 - prompt_tokens = num_prompt_tokens[prompt_idx]
349 - completion_tokens = previous_num_tokens[i]
350 - usage = UsageInfo(
351 - prompt_tokens=prompt_tokens,
352 - completion_tokens=completion_tokens,
353 - total_tokens=prompt_tokens + completion_tokens,
354 - )
355 - if request.stream_options.continuous_usage_stats:
356 - chunk.usage = usage
357 - else:
358 - chunk.usage = None
349 + if include_continuous_usage:
350 + prompt_tokens = num_prompt_tokens[prompt_idx]
351 + completion_tokens = previous_num_tokens[i]
352 + chunk.usage = UsageInfo(
353 + prompt_tokens=prompt_tokens,
354 + completion_tokens=completion_tokens,
355 + total_tokens=prompt_tokens + completion_tokens,
356 + )
357
358 response_json = chunk.model_dump_json(exclude_unset=False)
359 yield f"data: {response_json}\n\n"
360
363 - if (request.stream_options
364 - and request.stream_options.include_usage):
361 + total_prompt_tokens = sum(num_prompt_tokens)
362 + total_completion_tokens = sum(previous_num_tokens)
363 + final_usage_info = UsageInfo(
364 + prompt_tokens=total_prompt_tokens,
365 + completion_tokens=total_completion_tokens,
366 + total_tokens=total_prompt_tokens + total_completion_tokens)
367 +
368 + if include_usage:
369 final_usage_chunk = CompletionStreamResponse(
370 id=request_id,
371 created=created_time,
372 model=model_name,
373 choices=[],
370 - usage=usage,
374 + usage=final_usage_info,
375 )
376 final_usage_data = (final_usage_chunk.model_dump_json(
377 exclude_unset=False, exclude_none=True))
378 yield f"data: {final_usage_data}\n\n"
379
380 # report to FastAPI middleware aggregate usage across all choices
377 - total_prompt_tokens = sum(num_prompt_tokens)
378 - total_completion_tokens = sum(previous_num_tokens)
379 - request_metadata.final_usage_info = UsageInfo(
380 - prompt_tokens=total_prompt_tokens,
381 - completion_tokens=total_completion_tokens,
382 - total_tokens=total_prompt_tokens + total_completion_tokens)
381 + request_metadata.final_usage_info = final_usage_info
382
383 except ValueError as e:
384 # TODO: Use a vllm-specific Validation Error
@@ -413,26 +412,26 @@ class OpenAIServingCompletion(OpenAIServing):
412
413 for output in final_res.outputs:
414 assert request.max_tokens is not None
416 - if request.echo and request.max_tokens == 0:
417 - assert prompt_text is not None
418 - token_ids = prompt_token_ids
419 - out_logprobs = prompt_logprobs
420 - output_text = prompt_text
421 - elif request.echo and request.max_tokens > 0:
415 + if request.echo:
416 assert prompt_text is not None
423 - token_ids = [*prompt_token_ids, *output.token_ids]
424 -
425 - if request.logprobs is None:
426 - out_logprobs = None
417 + if request.max_tokens == 0:
418 + token_ids = prompt_token_ids
419 + out_logprobs = prompt_logprobs
420 + output_text = prompt_text
421 else:
428 - assert prompt_logprobs is not None
429 - assert output.logprobs is not None
430 - out_logprobs = [
431 - *prompt_logprobs,
432 - *output.logprobs,
433 - ]
434 -
435 - output_text = prompt_text + output.text
422 + token_ids = [*prompt_token_ids, *output.token_ids]
423 +
424 + if request.logprobs is None:
425 + out_logprobs = None
426 + else:
427 + assert prompt_logprobs is not None
428 + assert output.logprobs is not None
429 + out_logprobs = [
430 + *prompt_logprobs,
431 + *output.logprobs,
432 + ]
433 +
434 + output_text = prompt_text + output.text
435 else:
436 token_ids = output.token_ids
437 out_logprobs = output.logprobs