@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / baa15a9ec3
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 00:30 UTC

[Feature][Frontend]: Add support for `stream_options` in `ChatCompletionRequest` (#5135)

Itay Etelis committed Jun 7, 2024 at 06:29 UTC baa15a9ec320a6b90222df0aaed13b89e3bafc9c
3 files changed +149 -10
tests/entrypoints/test_openai_server.py
+101
@@ -1343,5 +1343,106 @@ async def test_batch_embedding(embedding_server, client: openai.AsyncOpenAI,
1343 assert embeddings.usage.total_tokens == 17
1344
1345
1346 +@pytest.mark.parametrize(
1347 + "model_name",
1348 + [MODEL_NAME],
1349 +)
1350 +async def test_stream_options(server, client: openai.AsyncOpenAI,
1351 + model_name: str):
1352 + prompt = "What is the capital of France?"
1353 +
1354 + # Test stream=True, stream_options=None
1355 + stream = await client.completions.create(
1356 + model=model_name,
1357 + prompt=prompt,
1358 + max_tokens=5,
1359 + temperature=0.0,
1360 + stream=True,
1361 + stream_options=None,
1362 + )
1363 + chunks = []
1364 + async for chunk in stream:
1365 + chunks.append(chunk.choices[0].text)
1366 + assert len(chunks) > 0
1367 + assert "usage" not in chunk
1368 +
1369 + # Test stream=True, stream_options={"include_usage": False}
1370 + stream = await client.completions.create(
1371 + model=model_name,
1372 + prompt=prompt,
1373 + max_tokens=5,
1374 + temperature=0.0,
1375 + stream=True,
1376 + stream_options={"include_usage": False},
1377 + )
1378 + chunks = []
1379 + async for chunk in stream:
1380 + chunks.append(chunk.choices[0].text)
1381 + assert len(chunks) > 0
1382 + assert "usage" not in chunk
1383 +
1384 + # Test stream=True, stream_options={"include_usage": True}
1385 + stream = await client.completions.create(
1386 + model=model_name,
1387 + prompt=prompt,
1388 + max_tokens=5,
1389 + temperature=0.0,
1390 + stream=True,
1391 + stream_options={"include_usage": True},
1392 + )
1393 + chunks = []
1394 + finish_reason_count = 0
1395 + async for chunk in stream:
1396 + if chunk.choices[0].finish_reason is None:
1397 + assert chunk.usage is None
1398 + chunks.append(chunk.choices[0].text)
1399 + else:
1400 + assert chunk.usage is None
1401 + finish_reason_count += 1
1402 +
1403 + # The last message should have usage and no choices
1404 + last_message = await stream.__anext__()
1405 + assert last_message.usage is not None
1406 + assert last_message.usage.prompt_tokens > 0
1407 + assert last_message.usage.completion_tokens > 0
1408 + assert last_message.usage.total_tokens == (
1409 + last_message.usage.prompt_tokens +
1410 + last_message.usage.completion_tokens)
1411 + assert last_message.choices == []
1412 +
1413 + # Test stream=False, stream_options={"include_usage": None}
1414 + with pytest.raises(BadRequestError):
1415 + await client.completions.create(
1416 + model=model_name,
1417 + prompt=prompt,
1418 + max_tokens=5,
1419 + temperature=0.0,
1420 + stream=False,
1421 + stream_options={"include_usage": None},
1422 + )
1423 +
1424 + # Test stream=False, stream_options={"include_usage": False}
1425 + with pytest.raises(BadRequestError):
1426 + await client.completions.create(
1427 + model=model_name,
1428 + prompt=prompt,
1429 + max_tokens=5,
1430 + temperature=0.0,
1431 + stream=False,
1432 + stream_options={"include_usage": False},
1433 + )
1434 +
1435 + # Test stream=False, stream_options={"include_usage": True}
1436 + with pytest.raises(BadRequestError):
1437 + await client.completions.create(
1438 + model=model_name,
1439 + prompt=prompt,
1440 + max_tokens=5,
1441 + temperature=0.0,
1442 + stream=False,
1443 + stream_options={"include_usage": True},
1444 + )
1445 +
1446 +
1447 if __name__ == "__main__":
1448 pytest.main([__file__])
vllm/entrypoints/openai/protocol.py
+14
@@ -102,6 +102,10 @@ class ResponseFormat(OpenAIBaseModel):
102 type: Literal["text", "json_object"]
103
104
105 +class StreamOptions(OpenAIBaseModel):
106 + include_usage: Optional[bool]
107 +
108 +
109 class FunctionDefinition(OpenAIBaseModel):
110 name: str
111 description: Optional[str] = None
@@ -140,6 +144,7 @@ class ChatCompletionRequest(OpenAIBaseModel):
144 le=torch.iinfo(torch.long).max)
145 stop: Optional[Union[str, List[str]]] = Field(default_factory=list)
146 stream: Optional[bool] = False
147 + stream_options: Optional[StreamOptions] = None
148 temperature: Optional[float] = 0.7
149 top_p: Optional[float] = 1.0
150 tools: Optional[List[ChatCompletionToolsParam]] = None
@@ -269,6 +274,15 @@ class ChatCompletionRequest(OpenAIBaseModel):
274 logits_processors=logits_processors,
275 )
276
277 + @model_validator(mode='before')
278 + @classmethod
279 + def validate_stream_options(cls, values):
280 + if (values.get('stream_options') is not None
281 + and not values.get('stream')):
282 + raise ValueError(
283 + "stream_options can only be set if stream is true")
284 + return values
285 +
286 @model_validator(mode="before")
287 @classmethod
288 def check_guided_decoding_count(cls, data):
vllm/entrypoints/openai/serving_chat.py
+34 -10
@@ -247,6 +247,9 @@ class OpenAIServingChat(OpenAIServing):
247 created=created_time,
248 choices=[choice_data],
249 model=model_name)
250 + if (request.stream_options
251 + and request.stream_options.include_usage):
252 + chunk.usage = None
253 data = chunk.model_dump_json(exclude_unset=True)
254 yield f"data: {data}\n\n"
255
@@ -274,6 +277,9 @@ class OpenAIServingChat(OpenAIServing):
277 choices=[choice_data],
278 logprobs=None,
279 model=model_name)
280 + if (request.stream_options and
281 + request.stream_options.include_usage):
282 + chunk.usage = None
283 data = chunk.model_dump_json(
284 exclude_unset=True)
285 yield f"data: {data}\n\n"
@@ -327,17 +333,14 @@ class OpenAIServingChat(OpenAIServing):
333 created=created_time,
334 choices=[choice_data],
335 model=model_name)
336 + if (request.stream_options
337 + and request.stream_options.include_usage):
338 + chunk.usage = None
339 data = chunk.model_dump_json(exclude_unset=True)
340 yield f"data: {data}\n\n"
341 else:
342 # Send the finish response for each request.n only once
343 prompt_tokens = len(res.prompt_token_ids)
335 - final_usage = UsageInfo(
336 - prompt_tokens=prompt_tokens,
337 - completion_tokens=previous_num_tokens[i],
338 - total_tokens=prompt_tokens +
339 - previous_num_tokens[i],
340 - )
344 choice_data = ChatCompletionResponseStreamChoice(
345 index=i,
346 delta=delta_message,
@@ -350,12 +353,33 @@ class OpenAIServingChat(OpenAIServing):
353 created=created_time,
354 choices=[choice_data],
355 model=model_name)
353 - if final_usage is not None:
354 - chunk.usage = final_usage
355 - data = chunk.model_dump_json(exclude_unset=True,
356 - exclude_none=True)
356 + if (request.stream_options
357 + and request.stream_options.include_usage):
358 + chunk.usage = None
359 + data = chunk.model_dump_json(exclude_unset=True)
360 yield f"data: {data}\n\n"
361 finish_reason_sent[i] = True
362 +
363 + if (request.stream_options
364 + and request.stream_options.include_usage):
365 + final_usage = UsageInfo(
366 + prompt_tokens=prompt_tokens,
367 + completion_tokens=previous_num_tokens[i],
368 + total_tokens=prompt_tokens +
369 + previous_num_tokens[i],
370 + )
371 +
372 + final_usage_chunk = ChatCompletionStreamResponse(
373 + id=request_id,
374 + object=chunk_object_type,
375 + created=created_time,
376 + choices=[],
377 + model=model_name,
378 + usage=final_usage)
379 + final_usage_data = (final_usage_chunk.model_dump_json(
380 + exclude_unset=True, exclude_none=True))
381 + yield f"data: {final_usage_data}\n\n"
382 +
383 except ValueError as e:
384 # TODO: Use a vllm-specific Validation Error
385 data = self.create_streaming_error_response(str(e))