A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 00:30 UTC
[Feature][Frontend]: Add support for `stream_options` in `ChatCompletionRequest` (#5135)
Itay Etelis committed
Jun 7, 2024 at 06:29 UTC
baa15a9ec320a6b90222df0aaed13b89e3bafc9c
3 files changed
+149
-10
tests/entrypoints/test_openai_server.py
+101
@@ -1343,5 +1343,106 @@ async def test_batch_embedding(embedding_server, client: openai.AsyncOpenAI,
1343
assert embeddings.usage.total_tokens == 17
1344
1345
1346
+@pytest.mark.parametrize(
1347
+ "model_name",
1348
+ [MODEL_NAME],
1349
+)
1350
+async def test_stream_options(server, client: openai.AsyncOpenAI,
1351
+ model_name: str):
1352
+ prompt = "What is the capital of France?"
1353
+
1354
+ # Test stream=True, stream_options=None
1355
+ stream = await client.completions.create(
1356
+ model=model_name,
1357
+ prompt=prompt,
1358
+ max_tokens=5,
1359
+ temperature=0.0,
1360
+ stream=True,
1361
+ stream_options=None,
1362
+ )
1363
+ chunks = []
1364
+ async for chunk in stream:
1365
+ chunks.append(chunk.choices[0].text)
1366
+ assert len(chunks) > 0
1367
+ assert "usage" not in chunk
1368
+
1369
+ # Test stream=True, stream_options={"include_usage": False}
1370
+ stream = await client.completions.create(
1371
+ model=model_name,
1372
+ prompt=prompt,
1373
+ max_tokens=5,
1374
+ temperature=0.0,
1375
+ stream=True,
1376
+ stream_options={"include_usage": False},
1377
+ )
1378
+ chunks = []
1379
+ async for chunk in stream:
1380
+ chunks.append(chunk.choices[0].text)
1381
+ assert len(chunks) > 0
1382
+ assert "usage" not in chunk
1383
+
1384
+ # Test stream=True, stream_options={"include_usage": True}
1385
+ stream = await client.completions.create(
1386
+ model=model_name,
1387
+ prompt=prompt,
1388
+ max_tokens=5,
1389
+ temperature=0.0,
1390
+ stream=True,
1391
+ stream_options={"include_usage": True},
1392
+ )
1393
+ chunks = []
1394
+ finish_reason_count = 0
1395
+ async for chunk in stream:
1396
+ if chunk.choices[0].finish_reason is None:
1397
+ assert chunk.usage is None
1398
+ chunks.append(chunk.choices[0].text)
1399
+ else:
1400
+ assert chunk.usage is None
1401
+ finish_reason_count += 1
1402
+
1403
+ # The last message should have usage and no choices
1404
+ last_message = await stream.__anext__()
1405
+ assert last_message.usage is not None
1406
+ assert last_message.usage.prompt_tokens > 0
1407
+ assert last_message.usage.completion_tokens > 0
1408
+ assert last_message.usage.total_tokens == (
1409
+ last_message.usage.prompt_tokens +
1410
+ last_message.usage.completion_tokens)
1411
+ assert last_message.choices == []
1412
+
1413
+ # Test stream=False, stream_options={"include_usage": None}
1414
+ with pytest.raises(BadRequestError):
1415
+ await client.completions.create(
1416
+ model=model_name,
1417
+ prompt=prompt,
1418
+ max_tokens=5,
1419
+ temperature=0.0,
1420
+ stream=False,
1421
+ stream_options={"include_usage": None},
1422
+ )
1423
+
1424
+ # Test stream=False, stream_options={"include_usage": False}
1425
+ with pytest.raises(BadRequestError):
1426
+ await client.completions.create(
1427
+ model=model_name,
1428
+ prompt=prompt,
1429
+ max_tokens=5,
1430
+ temperature=0.0,
1431
+ stream=False,
1432
+ stream_options={"include_usage": False},
1433
+ )
1434
+
1435
+ # Test stream=False, stream_options={"include_usage": True}
1436
+ with pytest.raises(BadRequestError):
1437
+ await client.completions.create(
1438
+ model=model_name,
1439
+ prompt=prompt,
1440
+ max_tokens=5,
1441
+ temperature=0.0,
1442
+ stream=False,
1443
+ stream_options={"include_usage": True},
1444
+ )
1445
+
1446
+
1447
if __name__ == "__main__":
1448
pytest.main([__file__])
vllm/entrypoints/openai/protocol.py
+14
@@ -102,6 +102,10 @@ class ResponseFormat(OpenAIBaseModel):
102
type: Literal["text", "json_object"]
103
104
105
+class StreamOptions(OpenAIBaseModel):
106
+ include_usage: Optional[bool]
107
+
108
+
109
class FunctionDefinition(OpenAIBaseModel):
110
name: str
111
description: Optional[str] = None
@@ -140,6 +144,7 @@ class ChatCompletionRequest(OpenAIBaseModel):
144
le=torch.iinfo(torch.long).max)
145
stop: Optional[Union[str, List[str]]] = Field(default_factory=list)
146
stream: Optional[bool] = False
147
+ stream_options: Optional[StreamOptions] = None
148
temperature: Optional[float] = 0.7
149
top_p: Optional[float] = 1.0
150
tools: Optional[List[ChatCompletionToolsParam]] = None
@@ -269,6 +274,15 @@ class ChatCompletionRequest(OpenAIBaseModel):
274
logits_processors=logits_processors,
275
)
276
277
+ @model_validator(mode='before')
278
+ @classmethod
279
+ def validate_stream_options(cls, values):
280
+ if (values.get('stream_options') is not None
281
+ and not values.get('stream')):
282
+ raise ValueError(
283
+ "stream_options can only be set if stream is true")
284
+ return values
285
+
286
@model_validator(mode="before")
287
@classmethod
288
def check_guided_decoding_count(cls, data):
vllm/entrypoints/openai/serving_chat.py
+34
-10
@@ -247,6 +247,9 @@ class OpenAIServingChat(OpenAIServing):
247
created=created_time,
248
choices=[choice_data],
249
model=model_name)
250
+ if (request.stream_options
251
+ and request.stream_options.include_usage):
252
+ chunk.usage = None
253
data = chunk.model_dump_json(exclude_unset=True)
254
yield f"data: {data}\n\n"
255
@@ -274,6 +277,9 @@ class OpenAIServingChat(OpenAIServing):
277
choices=[choice_data],
278
logprobs=None,
279
model=model_name)
280
+ if (request.stream_options and
281
+ request.stream_options.include_usage):
282
+ chunk.usage = None
283
data = chunk.model_dump_json(
284
exclude_unset=True)
285
yield f"data: {data}\n\n"
@@ -327,17 +333,14 @@ class OpenAIServingChat(OpenAIServing):
333
created=created_time,
334
choices=[choice_data],
335
model=model_name)
336
+ if (request.stream_options
337
+ and request.stream_options.include_usage):
338
+ chunk.usage = None
339
data = chunk.model_dump_json(exclude_unset=True)
340
yield f"data: {data}\n\n"
341
else:
342
# Send the finish response for each request.n only once
343
prompt_tokens = len(res.prompt_token_ids)
335
- final_usage = UsageInfo(
336
- prompt_tokens=prompt_tokens,
337
- completion_tokens=previous_num_tokens[i],
338
- total_tokens=prompt_tokens +
339
- previous_num_tokens[i],
340
- )
344
choice_data = ChatCompletionResponseStreamChoice(
345
index=i,
346
delta=delta_message,
@@ -350,12 +353,33 @@ class OpenAIServingChat(OpenAIServing):
353
created=created_time,
354
choices=[choice_data],
355
model=model_name)
353
- if final_usage is not None:
354
- chunk.usage = final_usage
355
- data = chunk.model_dump_json(exclude_unset=True,
356
- exclude_none=True)
356
+ if (request.stream_options
357
+ and request.stream_options.include_usage):
358
+ chunk.usage = None
359
+ data = chunk.model_dump_json(exclude_unset=True)
360
yield f"data: {data}\n\n"
361
finish_reason_sent[i] = True
362
+
363
+ if (request.stream_options
364
+ and request.stream_options.include_usage):
365
+ final_usage = UsageInfo(
366
+ prompt_tokens=prompt_tokens,
367
+ completion_tokens=previous_num_tokens[i],
368
+ total_tokens=prompt_tokens +
369
+ previous_num_tokens[i],
370
+ )
371
+
372
+ final_usage_chunk = ChatCompletionStreamResponse(
373
+ id=request_id,
374
+ object=chunk_object_type,
375
+ created=created_time,
376
+ choices=[],
377
+ model=model_name,
378
+ usage=final_usage)
379
+ final_usage_data = (final_usage_chunk.model_dump_json(
380
+ exclude_unset=True, exclude_none=True))
381
+ yield f"data: {final_usage_data}\n\n"
382
+
383
except ValueError as e:
384
# TODO: Use a vllm-specific Validation Error
385
data = self.create_streaming_error_response(str(e))