@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 774d1035e4
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 00:30 UTC

[Feature][Frontend]: Continued `stream_options` implementation also in CompletionRequest (#5319)

Itay Etelis committed Jun 10, 2024 at 17:22 UTC 774d1035e4000fe3a40e01ccbf017a0f763fd6f2
4 files changed +180 -126
tests/entrypoints/test_openai_server.py
+132 -104
@@ -478,8 +478,6 @@ async def test_completion_streaming(server, client: openai.AsyncOpenAI,
478 temperature=0.0,
479 )
480 single_output = single_completion.choices[0].text
481 - single_usage = single_completion.usage
482 -
481 stream = await client.completions.create(model=model_name,
482 prompt=prompt,
483 max_tokens=5,
@@ -495,7 +493,6 @@ async def test_completion_streaming(server, client: openai.AsyncOpenAI,
493 assert finish_reason_count == 1
494 assert chunk.choices[0].finish_reason == "length"
495 assert chunk.choices[0].text
498 - assert chunk.usage == single_usage
496 assert "".join(chunks) == single_output
497
498
@@ -550,6 +547,138 @@ async def test_chat_streaming(server, client: openai.AsyncOpenAI,
547 assert "".join(chunks) == output
548
549
550 +@pytest.mark.asyncio
551 +@pytest.mark.parametrize(
552 + "model_name",
553 + ["HuggingFaceH4/zephyr-7b-beta", "zephyr-lora"],
554 +)
555 +async def test_chat_completion_stream_options(server,
556 + client: openai.AsyncOpenAI,
557 + model_name: str):
558 + messages = [{
559 + "role": "system",
560 + "content": "You are a helpful assistant."
561 + }, {
562 + "role": "user",
563 + "content": "What is the capital of France?"
564 + }]
565 +
566 + # Test stream=True, stream_options={"include_usage": False}
567 + stream = await client.chat.completions.create(
568 + model=model_name,
569 + messages=messages,
570 + max_tokens=10,
571 + temperature=0.0,
572 + stream=True,
573 + stream_options={"include_usage": False})
574 + async for chunk in stream:
575 + assert chunk.usage is None
576 +
577 + # Test stream=True, stream_options={"include_usage": True}
578 + stream = await client.chat.completions.create(
579 + model=model_name,
580 + messages=messages,
581 + max_tokens=10,
582 + temperature=0.0,
583 + stream=True,
584 + stream_options={"include_usage": True})
585 +
586 + async for chunk in stream:
587 + if chunk.choices[0].finish_reason is None:
588 + assert chunk.usage is None
589 + else:
590 + assert chunk.usage is None
591 + final_chunk = await stream.__anext__()
592 + assert final_chunk.usage is not None
593 + assert final_chunk.usage.prompt_tokens > 0
594 + assert final_chunk.usage.completion_tokens > 0
595 + assert final_chunk.usage.total_tokens == (
596 + final_chunk.usage.prompt_tokens +
597 + final_chunk.usage.completion_tokens)
598 + assert final_chunk.choices == []
599 +
600 + # Test stream=False, stream_options={"include_usage": None}
601 + with pytest.raises(BadRequestError):
602 + await client.chat.completions.create(
603 + model=model_name,
604 + messages=messages,
605 + max_tokens=10,
606 + temperature=0.0,
607 + stream=False,
608 + stream_options={"include_usage": None})
609 +
610 + # Test stream=False, stream_options={"include_usage": True}
611 + with pytest.raises(BadRequestError):
612 + await client.chat.completions.create(
613 + model=model_name,
614 + messages=messages,
615 + max_tokens=10,
616 + temperature=0.0,
617 + stream=False,
618 + stream_options={"include_usage": True})
619 +
620 +
621 +@pytest.mark.asyncio
622 +@pytest.mark.parametrize(
623 + "model_name",
624 + ["HuggingFaceH4/zephyr-7b-beta", "zephyr-lora"],
625 +)
626 +async def test_completion_stream_options(server, client: openai.AsyncOpenAI,
627 + model_name: str):
628 + prompt = "What is the capital of France?"
629 +
630 + # Test stream=True, stream_options={"include_usage": False}
631 + stream = await client.completions.create(
632 + model=model_name,
633 + prompt=prompt,
634 + max_tokens=5,
635 + temperature=0.0,
636 + stream=True,
637 + stream_options={"include_usage": False})
638 + async for chunk in stream:
639 + assert chunk.usage is None
640 +
641 + # Test stream=True, stream_options={"include_usage": True}
642 + stream = await client.completions.create(
643 + model=model_name,
644 + prompt=prompt,
645 + max_tokens=5,
646 + temperature=0.0,
647 + stream=True,
648 + stream_options={"include_usage": True})
649 + async for chunk in stream:
650 + if chunk.choices[0].finish_reason is None:
651 + assert chunk.usage is None
652 + else:
653 + assert chunk.usage is None
654 + final_chunk = await stream.__anext__()
655 + assert final_chunk.usage is not None
656 + assert final_chunk.usage.prompt_tokens > 0
657 + assert final_chunk.usage.completion_tokens > 0
658 + assert final_chunk.usage.total_tokens == (
659 + final_chunk.usage.prompt_tokens +
660 + final_chunk.usage.completion_tokens)
661 + assert final_chunk.choices == []
662 +
663 + # Test stream=False, stream_options={"include_usage": None}
664 + with pytest.raises(BadRequestError):
665 + await client.completions.create(model=model_name,
666 + prompt=prompt,
667 + max_tokens=5,
668 + temperature=0.0,
669 + stream=False,
670 + stream_options={"include_usage": None})
671 +
672 + # Test stream=False, stream_options={"include_usage": True}
673 + with pytest.raises(BadRequestError):
674 + await client.completions.create(model=model_name,
675 + prompt=prompt,
676 + max_tokens=5,
677 + temperature=0.0,
678 + stream=False,
679 + stream_options={"include_usage": True})
680 +
681 +
682 @pytest.mark.asyncio
683 @pytest.mark.parametrize(
684 # just test 1 lora hereafter
@@ -1343,106 +1472,5 @@ async def test_batch_embedding(embedding_server, client: openai.AsyncOpenAI,
1472 assert embeddings.usage.total_tokens == 17
1473
1474
1346 -@pytest.mark.parametrize(
1347 - "model_name",
1348 - [MODEL_NAME],
1349 -)
1350 -async def test_stream_options(server, client: openai.AsyncOpenAI,
1351 - model_name: str):
1352 - prompt = "What is the capital of France?"
1353 -
1354 - # Test stream=True, stream_options=None
1355 - stream = await client.completions.create(
1356 - model=model_name,
1357 - prompt=prompt,
1358 - max_tokens=5,
1359 - temperature=0.0,
1360 - stream=True,
1361 - stream_options=None,
1362 - )
1363 - chunks = []
1364 - async for chunk in stream:
1365 - chunks.append(chunk.choices[0].text)
1366 - assert len(chunks) > 0
1367 - assert "usage" not in chunk
1368 -
1369 - # Test stream=True, stream_options={"include_usage": False}
1370 - stream = await client.completions.create(
1371 - model=model_name,
1372 - prompt=prompt,
1373 - max_tokens=5,
1374 - temperature=0.0,
1375 - stream=True,
1376 - stream_options={"include_usage": False},
1377 - )
1378 - chunks = []
1379 - async for chunk in stream:
1380 - chunks.append(chunk.choices[0].text)
1381 - assert len(chunks) > 0
1382 - assert "usage" not in chunk
1383 -
1384 - # Test stream=True, stream_options={"include_usage": True}
1385 - stream = await client.completions.create(
1386 - model=model_name,
1387 - prompt=prompt,
1388 - max_tokens=5,
1389 - temperature=0.0,
1390 - stream=True,
1391 - stream_options={"include_usage": True},
1392 - )
1393 - chunks = []
1394 - finish_reason_count = 0
1395 - async for chunk in stream:
1396 - if chunk.choices[0].finish_reason is None:
1397 - assert chunk.usage is None
1398 - chunks.append(chunk.choices[0].text)
1399 - else:
1400 - assert chunk.usage is None
1401 - finish_reason_count += 1
1402 -
1403 - # The last message should have usage and no choices
1404 - last_message = await stream.__anext__()
1405 - assert last_message.usage is not None
1406 - assert last_message.usage.prompt_tokens > 0
1407 - assert last_message.usage.completion_tokens > 0
1408 - assert last_message.usage.total_tokens == (
1409 - last_message.usage.prompt_tokens +
1410 - last_message.usage.completion_tokens)
1411 - assert last_message.choices == []
1412 -
1413 - # Test stream=False, stream_options={"include_usage": None}
1414 - with pytest.raises(BadRequestError):
1415 - await client.completions.create(
1416 - model=model_name,
1417 - prompt=prompt,
1418 - max_tokens=5,
1419 - temperature=0.0,
1420 - stream=False,
1421 - stream_options={"include_usage": None},
1422 - )
1423 -
1424 - # Test stream=False, stream_options={"include_usage": False}
1425 - with pytest.raises(BadRequestError):
1426 - await client.completions.create(
1427 - model=model_name,
1428 - prompt=prompt,
1429 - max_tokens=5,
1430 - temperature=0.0,
1431 - stream=False,
1432 - stream_options={"include_usage": False},
1433 - )
1434 -
1435 - # Test stream=False, stream_options={"include_usage": True}
1436 - with pytest.raises(BadRequestError):
1437 - await client.completions.create(
1438 - model=model_name,
1439 - prompt=prompt,
1440 - max_tokens=5,
1441 - temperature=0.0,
1442 - stream=False,
1443 - stream_options={"include_usage": True},
1444 - )
1445 -
1446 -
1475 if __name__ == "__main__":
1476 pytest.main([__file__])
vllm/entrypoints/openai/protocol.py
+9
@@ -346,6 +346,7 @@ class CompletionRequest(OpenAIBaseModel):
346 le=torch.iinfo(torch.long).max)
347 stop: Optional[Union[str, List[str]]] = Field(default_factory=list)
348 stream: Optional[bool] = False
349 + stream_options: Optional[StreamOptions] = None
350 suffix: Optional[str] = None
351 temperature: Optional[float] = 1.0
352 top_p: Optional[float] = 1.0
@@ -482,6 +483,14 @@ class CompletionRequest(OpenAIBaseModel):
483 " in the interval [0, 5]."))
484 return data
485
486 + @model_validator(mode="before")
487 + @classmethod
488 + def validate_stream_options(cls, data):
489 + if data.get("stream_options") and not data.get("stream"):
490 + raise ValueError(
491 + "Stream options can only be defined when stream is True.")
492 + return data
493 +
494
495 class EmbeddingRequest(BaseModel):
496 # Ordered by official OpenAI API documentation
vllm/entrypoints/openai/serving_chat.py
+17 -18
@@ -441,25 +441,24 @@ class OpenAIServingChat(OpenAIServing):
441 yield f"data: {data}\n\n"
442 finish_reason_sent[i] = True
443
444 - if (request.stream_options
445 - and request.stream_options.include_usage):
446 - final_usage = UsageInfo(
447 - prompt_tokens=prompt_tokens,
448 - completion_tokens=previous_num_tokens[i],
449 - total_tokens=prompt_tokens +
450 - previous_num_tokens[i],
451 - )
444 + if (request.stream_options
445 + and request.stream_options.include_usage):
446 + final_usage = UsageInfo(
447 + prompt_tokens=prompt_tokens,
448 + completion_tokens=previous_num_tokens[i],
449 + total_tokens=prompt_tokens + previous_num_tokens[i],
450 + )
451
453 - final_usage_chunk = ChatCompletionStreamResponse(
454 - id=request_id,
455 - object=chunk_object_type,
456 - created=created_time,
457 - choices=[],
458 - model=model_name,
459 - usage=final_usage)
460 - final_usage_data = (final_usage_chunk.model_dump_json(
461 - exclude_unset=True, exclude_none=True))
462 - yield f"data: {final_usage_data}\n\n"
452 + final_usage_chunk = ChatCompletionStreamResponse(
453 + id=request_id,
454 + object=chunk_object_type,
455 + created=created_time,
456 + choices=[],
457 + model=model_name,
458 + usage=final_usage)
459 + final_usage_data = (final_usage_chunk.model_dump_json(
460 + exclude_unset=True, exclude_none=True))
461 + yield f"data: {final_usage_data}\n\n"
462
463 except ValueError as e:
464 # TODO: Use a vllm-specific Validation Error
vllm/entrypoints/openai/serving_completion.py
+22 -4
@@ -264,7 +264,8 @@ class OpenAIServingCompletion(OpenAIServing):
264 )
265 else:
266 final_usage = None
267 - response_json = CompletionStreamResponse(
267 +
268 + chunk = CompletionStreamResponse(
269 id=request_id,
270 created=created_time,
271 model=model_name,
@@ -276,10 +277,27 @@ class OpenAIServingCompletion(OpenAIServing):
277 finish_reason=finish_reason,
278 stop_reason=stop_reason,
279 )
279 - ],
280 - usage=final_usage,
281 - ).model_dump_json(exclude_unset=True)
280 + ])
281 + if (request.stream_options
282 + and request.stream_options.include_usage):
283 + chunk.usage = None
284 +
285 + response_json = chunk.model_dump_json(exclude_unset=True)
286 yield f"data: {response_json}\n\n"
287 +
288 + if (request.stream_options
289 + and request.stream_options.include_usage):
290 + final_usage_chunk = CompletionStreamResponse(
291 + id=request_id,
292 + created=created_time,
293 + model=model_name,
294 + choices=[],
295 + usage=final_usage,
296 + )
297 + final_usage_data = (final_usage_chunk.model_dump_json(
298 + exclude_unset=True, exclude_none=True))
299 + yield f"data: {final_usage_data}\n\n"
300 +
301 except ValueError as e:
302 # TODO: Use a vllm-specific Validation Error
303 data = self.create_streaming_error_response(str(e))