@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / f1e15da6fe
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 22:00 UTC

[Frontend] Continuous usage stats in OpenAI completion API (#5742)

jvlunteren committed Jul 5, 2024 at 19:37 UTC f1e15da6fe20ff17d5b8c28f37487cee38f08b83
3 files changed +110 -31
tests/entrypoints/openai/test_completion.py
+94 -18
@@ -295,25 +295,49 @@ async def test_completion_stream_options(client: openai.AsyncOpenAI,
295 model_name: str):
296 prompt = "What is the capital of France?"
297
298 - # Test stream=True, stream_options={"include_usage": False}
299 - stream = await client.completions.create(
300 - model=model_name,
301 - prompt=prompt,
302 - max_tokens=5,
303 - temperature=0.0,
304 - stream=True,
305 - stream_options={"include_usage": False})
298 + # Test stream=True, stream_options=
299 + # {"include_usage": False, "continuous_usage_stats": False}
300 + stream = await client.completions.create(model=model_name,
301 + prompt=prompt,
302 + max_tokens=5,
303 + temperature=0.0,
304 + stream=True,
305 + stream_options={
306 + "include_usage": False,
307 + "continuous_usage_stats":
308 + False,
309 + })
310 +
311 async for chunk in stream:
312 assert chunk.usage is None
313
309 - # Test stream=True, stream_options={"include_usage": True}
310 - stream = await client.completions.create(
311 - model=model_name,
312 - prompt=prompt,
313 - max_tokens=5,
314 - temperature=0.0,
315 - stream=True,
316 - stream_options={"include_usage": True})
314 + # Test stream=True, stream_options=
315 + # {"include_usage": False, "continuous_usage_stats": True}
316 + stream = await client.completions.create(model=model_name,
317 + prompt=prompt,
318 + max_tokens=5,
319 + temperature=0.0,
320 + stream=True,
321 + stream_options={
322 + "include_usage": False,
323 + "continuous_usage_stats":
324 + True,
325 + })
326 + async for chunk in stream:
327 + assert chunk.usage is None
328 +
329 + # Test stream=True, stream_options=
330 + # {"include_usage": True, "continuous_usage_stats": False}
331 + stream = await client.completions.create(model=model_name,
332 + prompt=prompt,
333 + max_tokens=5,
334 + temperature=0.0,
335 + stream=True,
336 + stream_options={
337 + "include_usage": True,
338 + "continuous_usage_stats":
339 + False,
340 + })
341 async for chunk in stream:
342 if chunk.choices[0].finish_reason is None:
343 assert chunk.usage is None
@@ -328,7 +352,36 @@ async def test_completion_stream_options(client: openai.AsyncOpenAI,
352 final_chunk.usage.completion_tokens)
353 assert final_chunk.choices == []
354
331 - # Test stream=False, stream_options={"include_usage": None}
355 + # Test stream=True, stream_options=
356 + # {"include_usage": True, "continuous_usage_stats": True}
357 + stream = await client.completions.create(model=model_name,
358 + prompt=prompt,
359 + max_tokens=5,
360 + temperature=0.0,
361 + stream=True,
362 + stream_options={
363 + "include_usage": True,
364 + "continuous_usage_stats":
365 + True,
366 + })
367 + async for chunk in stream:
368 + assert chunk.usage is not None
369 + assert chunk.usage.prompt_tokens > 0
370 + assert chunk.usage.completion_tokens > 0
371 + assert chunk.usage.total_tokens == (chunk.usage.prompt_tokens +
372 + chunk.usage.completion_tokens)
373 + if chunk.choices[0].finish_reason is not None:
374 + final_chunk = await stream.__anext__()
375 + assert final_chunk.usage is not None
376 + assert final_chunk.usage.prompt_tokens > 0
377 + assert final_chunk.usage.completion_tokens > 0
378 + assert final_chunk.usage.total_tokens == (
379 + final_chunk.usage.prompt_tokens +
380 + final_chunk.usage.completion_tokens)
381 + assert final_chunk.choices == []
382 +
383 + # Test stream=False, stream_options=
384 + # {"include_usage": None}
385 with pytest.raises(BadRequestError):
386 await client.completions.create(model=model_name,
387 prompt=prompt,
@@ -337,7 +390,8 @@ async def test_completion_stream_options(client: openai.AsyncOpenAI,
390 stream=False,
391 stream_options={"include_usage": None})
392
340 - # Test stream=False, stream_options={"include_usage": True}
393 + # Test stream=False, stream_options=
394 + # {"include_usage": True}
395 with pytest.raises(BadRequestError):
396 await client.completions.create(model=model_name,
397 prompt=prompt,
@@ -346,6 +400,28 @@ async def test_completion_stream_options(client: openai.AsyncOpenAI,
400 stream=False,
401 stream_options={"include_usage": True})
402
403 + # Test stream=False, stream_options=
404 + # {"continuous_usage_stats": None}
405 + with pytest.raises(BadRequestError):
406 + await client.completions.create(
407 + model=model_name,
408 + prompt=prompt,
409 + max_tokens=5,
410 + temperature=0.0,
411 + stream=False,
412 + stream_options={"continuous_usage_stats": None})
413 +
414 + # Test stream=False, stream_options=
415 + # {"continuous_usage_stats": True}
416 + with pytest.raises(BadRequestError):
417 + await client.completions.create(
418 + model=model_name,
419 + prompt=prompt,
420 + max_tokens=5,
421 + temperature=0.0,
422 + stream=False,
423 + stream_options={"continuous_usage_stats": True})
424 +
425
426 @pytest.mark.asyncio
427 @pytest.mark.parametrize(
vllm/entrypoints/openai/protocol.py
+2 -1
@@ -103,7 +103,8 @@ class ResponseFormat(OpenAIBaseModel):
103
104
105 class StreamOptions(OpenAIBaseModel):
106 - include_usage: Optional[bool]
106 + include_usage: Optional[bool] = True
107 + continuous_usage_stats: Optional[bool] = True
108
109
110 class FunctionDefinition(OpenAIBaseModel):
vllm/entrypoints/openai/serving_completion.py
+14 -12
@@ -271,16 +271,6 @@ class OpenAIServingCompletion(OpenAIServing):
271 previous_num_tokens[i] = len(output.token_ids)
272 finish_reason = output.finish_reason
273 stop_reason = output.stop_reason
274 - if output.finish_reason is not None: # return final usage
275 - prompt_tokens = len(res.prompt_token_ids)
276 - completion_tokens = len(output.token_ids)
277 - final_usage = UsageInfo(
278 - prompt_tokens=prompt_tokens,
279 - completion_tokens=completion_tokens,
280 - total_tokens=prompt_tokens + completion_tokens,
281 - )
282 - else:
283 - final_usage = None
274
275 chunk = CompletionStreamResponse(
276 id=request_id,
@@ -297,7 +287,19 @@ class OpenAIServingCompletion(OpenAIServing):
287 ])
288 if (request.stream_options
289 and request.stream_options.include_usage):
300 - chunk.usage = None
290 + if (request.stream_options.continuous_usage_stats
291 + or output.finish_reason is not None):
292 + prompt_tokens = len(res.prompt_token_ids)
293 + completion_tokens = len(output.token_ids)
294 + usage = UsageInfo(
295 + prompt_tokens=prompt_tokens,
296 + completion_tokens=completion_tokens,
297 + total_tokens=prompt_tokens + completion_tokens,
298 + )
299 + if request.stream_options.continuous_usage_stats:
300 + chunk.usage = usage
301 + else:
302 + chunk.usage = None
303
304 response_json = chunk.model_dump_json(exclude_unset=True)
305 yield f"data: {response_json}\n\n"
@@ -309,7 +311,7 @@ class OpenAIServingCompletion(OpenAIServing):
311 created=created_time,
312 model=model_name,
313 choices=[],
312 - usage=final_usage,
314 + usage=usage,
315 )
316 final_usage_data = (final_usage_chunk.model_dump_json(
317 exclude_unset=True, exclude_none=True))