A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 23:15 UTC
[Bugfix][Frontend] Cleanup "fix chat logprobs" (#5026)
Cyrus Leung committed
Jun 11, 2024 at 13:36 UTC
640052b0698d64d03806c98bc118a425afc53eff
6 files changed
+122
-123
tests/async_engine/test_openapi_server_ray.py
+11
-14
@@ -55,9 +55,8 @@ async def test_single_completion(server, client: openai.AsyncOpenAI):
55
temperature=0.0)
56
57
assert completion.id is not None
58
- assert completion.choices is not None and len(completion.choices) == 1
59
- assert completion.choices[0].text is not None and len(
60
- completion.choices[0].text) >= 5
58
+ assert len(completion.choices) == 1
59
+ assert len(completion.choices[0].text) >= 5
60
assert completion.choices[0].finish_reason == "length"
61
assert completion.usage == openai.types.CompletionUsage(
62
completion_tokens=5, prompt_tokens=6, total_tokens=11)
@@ -69,8 +68,7 @@ async def test_single_completion(server, client: openai.AsyncOpenAI):
68
max_tokens=5,
69
temperature=0.0,
70
)
72
- assert completion.choices[0].text is not None and len(
73
- completion.choices[0].text) >= 5
71
+ assert len(completion.choices[0].text) >= 5
72
73
74
@pytest.mark.asyncio
@@ -90,15 +88,14 @@ async def test_single_chat_session(server, client: openai.AsyncOpenAI):
88
logprobs=True,
89
top_logprobs=5)
90
assert chat_completion.id is not None
93
- assert chat_completion.choices is not None and len(
94
- chat_completion.choices) == 1
95
- assert chat_completion.choices[0].message is not None
96
- assert chat_completion.choices[0].logprobs is not None
97
- assert chat_completion.choices[0].logprobs.content[
98
- 0].top_logprobs is not None
99
- assert len(
100
- chat_completion.choices[0].logprobs.content[0].top_logprobs) == 5
101
- message = chat_completion.choices[0].message
91
+ assert len(chat_completion.choices) == 1
92
+
93
+ choice = chat_completion.choices[0]
94
+ assert choice.finish_reason == "length"
95
+ assert chat_completion.usage == openai.types.CompletionUsage(
96
+ completion_tokens=10, prompt_tokens=13, total_tokens=23)
97
+
98
+ message = choice.message
99
assert message.content is not None and len(message.content) >= 10
100
assert message.role == "assistant"
101
messages.append({"role": "assistant", "content": message.content})
tests/entrypoints/test_openai_server.py
+84
-85
@@ -167,9 +167,10 @@ async def test_single_completion(server, client: openai.AsyncOpenAI,
167
168
assert completion.id is not None
169
assert completion.choices is not None and len(completion.choices) == 1
170
- assert completion.choices[0].text is not None and len(
171
- completion.choices[0].text) >= 5
172
- assert completion.choices[0].finish_reason == "length"
170
+
171
+ choice = completion.choices[0]
172
+ assert len(choice.text) >= 5
173
+ assert choice.finish_reason == "length"
174
assert completion.usage == openai.types.CompletionUsage(
175
completion_tokens=5, prompt_tokens=6, total_tokens=11)
176
@@ -180,8 +181,7 @@ async def test_single_completion(server, client: openai.AsyncOpenAI,
181
max_tokens=5,
182
temperature=0.0,
183
)
183
- assert completion.choices[0].text is not None and len(
184
- completion.choices[0].text) >= 5
184
+ assert len(completion.choices[0].text) >= 5
185
186
187
@pytest.mark.asyncio
@@ -206,9 +206,9 @@ async def test_no_logprobs(server, client: openai.AsyncOpenAI,
206
207
@pytest.mark.asyncio
208
@pytest.mark.parametrize(
209
- # first test base model, then test loras
209
+ # just test 1 lora hereafter
210
"model_name",
211
- [MODEL_NAME, "zephyr-lora", "zephyr-lora2"],
211
+ [MODEL_NAME, "zephyr-lora"],
212
)
213
async def test_zero_logprobs(server, client: openai.AsyncOpenAI,
214
model_name: str):
@@ -291,55 +291,7 @@ async def test_too_many_completion_logprobs(server, client: openai.AsyncOpenAI,
291
max_tokens=5,
292
temperature=0.0,
293
)
294
- completion = completion.choices[0].text
295
- assert completion is not None and len(completion) >= 0
296
-
297
-
298
-@pytest.mark.asyncio
299
-@pytest.mark.parametrize(
300
- # just test 1 lora hereafter
301
- "model_name",
302
- [MODEL_NAME, "zephyr-lora"],
303
-)
304
-async def test_single_chat_session(server, client: openai.AsyncOpenAI,
305
- model_name: str):
306
- messages = [{
307
- "role": "system",
308
- "content": "you are a helpful assistant"
309
- }, {
310
- "role": "user",
311
- "content": "what is 1+1?"
312
- }]
313
-
314
- # test single completion
315
- chat_completion = await client.chat.completions.create(model=model_name,
316
- messages=messages,
317
- max_tokens=10,
318
- logprobs=True,
319
- top_logprobs=5)
320
- assert chat_completion.id is not None
321
- assert chat_completion.choices is not None and len(
322
- chat_completion.choices) == 1
323
- assert chat_completion.choices[0].message is not None
324
- assert chat_completion.choices[0].logprobs is not None
325
- assert chat_completion.choices[0].logprobs.content[
326
- 0].top_logprobs is not None
327
- assert len(
328
- chat_completion.choices[0].logprobs.content[0].top_logprobs) == 5
329
- message = chat_completion.choices[0].message
330
- assert message.content is not None and len(message.content) >= 10
331
- assert message.role == "assistant"
332
- messages.append({"role": "assistant", "content": message.content})
333
-
334
- # test multi-turn dialogue
335
- messages.append({"role": "user", "content": "express your result in json"})
336
- chat_completion = await client.chat.completions.create(
337
- model=model_name,
338
- messages=messages,
339
- max_tokens=10,
340
- )
341
- message = chat_completion.choices[0].message
342
- assert message.content is not None and len(message.content) >= 0
294
+ assert len(completion.choices[0].text) >= 0
295
296
297
@pytest.mark.asyncio
@@ -394,7 +346,7 @@ async def test_zero_logprobs_chat(server, client: openai.AsyncOpenAI,
346
choice = chat_completion.choices[0]
347
assert choice.logprobs is not None
348
assert choice.logprobs.content is not None
397
- assert len(choice.logprobs.content[0].top_logprobs) <= 1
349
+ assert len(choice.logprobs.content[0].top_logprobs) == 0
350
351
352
@pytest.mark.asyncio
@@ -422,11 +374,14 @@ async def test_some_logprobs_chat(server, client: openai.AsyncOpenAI,
374
choice = chat_completion.choices[0]
375
assert choice.logprobs is not None
376
assert choice.logprobs.content is not None
425
- assert len(choice.logprobs.content[0].top_logprobs) <= 6
377
+ assert len(choice.logprobs.content[0].top_logprobs) == 5
378
379
380
@pytest.mark.asyncio
429
-@pytest.mark.parametrize("model_name", [MODEL_NAME])
381
+@pytest.mark.parametrize(
382
+ "model_name",
383
+ [MODEL_NAME, "zephyr-lora"],
384
+)
385
async def test_too_many_chat_logprobs(server, client: openai.AsyncOpenAI,
386
model_name: str):
387
messages = [{
@@ -467,7 +422,51 @@ async def test_too_many_chat_logprobs(server, client: openai.AsyncOpenAI,
422
423
@pytest.mark.asyncio
424
@pytest.mark.parametrize(
470
- # just test 1 lora hereafter
425
+ "model_name",
426
+ [MODEL_NAME, "zephyr-lora"],
427
+)
428
+async def test_single_chat_session(server, client: openai.AsyncOpenAI,
429
+ model_name: str):
430
+ messages = [{
431
+ "role": "system",
432
+ "content": "you are a helpful assistant"
433
+ }, {
434
+ "role": "user",
435
+ "content": "what is 1+1?"
436
+ }]
437
+
438
+ # test single completion
439
+ chat_completion = await client.chat.completions.create(model=model_name,
440
+ messages=messages,
441
+ max_tokens=10,
442
+ logprobs=True,
443
+ top_logprobs=5)
444
+ assert chat_completion.id is not None
445
+ assert len(chat_completion.choices) == 1
446
+
447
+ choice = chat_completion.choices[0]
448
+ assert choice.finish_reason == "length"
449
+ assert chat_completion.usage == openai.types.CompletionUsage(
450
+ completion_tokens=10, prompt_tokens=37, total_tokens=47)
451
+
452
+ message = choice.message
453
+ assert message.content is not None and len(message.content) >= 10
454
+ assert message.role == "assistant"
455
+ messages.append({"role": "assistant", "content": message.content})
456
+
457
+ # test multi-turn dialogue
458
+ messages.append({"role": "user", "content": "express your result in json"})
459
+ chat_completion = await client.chat.completions.create(
460
+ model=model_name,
461
+ messages=messages,
462
+ max_tokens=10,
463
+ )
464
+ message = chat_completion.choices[0].message
465
+ assert message.content is not None and len(message.content) >= 0
466
+
467
+
468
+@pytest.mark.asyncio
469
+@pytest.mark.parametrize(
470
"model_name",
471
[MODEL_NAME, "zephyr-lora"],
472
)
@@ -753,8 +752,7 @@ async def test_logits_bias(server, client: openai.AsyncOpenAI):
752
logit_bias={str(token_id): 100},
753
seed=42,
754
)
756
- assert completion.choices[0].text is not None and len(
757
- completion.choices[0].text) >= 5
755
+ assert len(completion.choices[0].text) >= 5
756
response_tokens = tokenizer(completion.choices[0].text,
757
add_special_tokens=False)["input_ids"]
758
expected_tokens = tokenizer(tokenizer.decode([token_id] * 5),
@@ -801,9 +799,8 @@ async def test_guided_json_completion(server, client: openai.AsyncOpenAI,
799
guided_decoding_backend=guided_decoding_backend))
800
801
assert completion.id is not None
804
- assert completion.choices is not None and len(completion.choices) == 3
802
+ assert len(completion.choices) == 3
803
for i in range(3):
806
- assert completion.choices[i].text is not None
804
output_json = json.loads(completion.choices[i].text)
805
jsonschema.validate(instance=output_json, schema=TEST_SCHEMA)
806
@@ -870,9 +867,8 @@ async def test_guided_regex_completion(server, client: openai.AsyncOpenAI,
867
guided_decoding_backend=guided_decoding_backend))
868
869
assert completion.id is not None
873
- assert completion.choices is not None and len(completion.choices) == 3
870
+ assert len(completion.choices) == 3
871
for i in range(3):
875
- assert completion.choices[i].text is not None
872
assert re.fullmatch(TEST_REGEX, completion.choices[i].text) is not None
873
874
@@ -929,7 +925,7 @@ async def test_guided_choice_completion(server, client: openai.AsyncOpenAI,
925
guided_decoding_backend=guided_decoding_backend))
926
927
assert completion.id is not None
932
- assert completion.choices is not None and len(completion.choices) == 2
928
+ assert len(completion.choices) == 2
929
for i in range(2):
930
assert completion.choices[i].text in TEST_CHOICE
931
@@ -1031,12 +1027,14 @@ async def test_guided_choice_chat_logprobs(server, client: openai.AsyncOpenAI,
1027
top_logprobs=5,
1028
extra_body=dict(guided_choice=TEST_CHOICE,
1029
guided_decoding_backend=guided_decoding_backend))
1030
+
1031
+ assert chat_completion.choices[0].logprobs is not None
1032
+ assert chat_completion.choices[0].logprobs.content is not None
1033
top_logprobs = chat_completion.choices[0].logprobs.content[0].top_logprobs
1034
1035
# -9999.0 is the minimum logprob returned by OpenAI
1037
- assert all(
1038
- isinstance(token.logprob, float) and token.logprob >= -9999.0
1039
- for token in top_logprobs)
1036
+ for item in top_logprobs:
1037
+ assert item.logprob >= -9999.0, f"Failed (top_logprobs={top_logprobs})"
1038
1039
1040
@pytest.mark.asyncio
@@ -1238,6 +1236,8 @@ async def test_response_format_json_object(server, client: openai.AsyncOpenAI):
1236
response_format={"type": "json_object"})
1237
1238
content = resp.choices[0].message.content
1239
+ assert content is not None
1240
+
1241
loaded = json.loads(content)
1242
assert loaded == {"result": 2}, loaded
1243
@@ -1365,8 +1365,7 @@ async def test_echo_logprob_completion(server, client: openai.AsyncOpenAI,
1365
1366
prompt_text = tokenizer.decode(prompt) if isinstance(prompt,
1367
list) else prompt
1368
- assert (completion.choices[0].text is not None
1369
- and re.search(r"^" + prompt_text, completion.choices[0].text))
1368
+ assert re.search(r"^" + prompt_text, completion.choices[0].text)
1369
logprobs = completion.choices[0].logprobs
1370
assert logprobs is not None
1371
assert len(logprobs.text_offset) > 5
@@ -1407,32 +1406,32 @@ async def test_long_seed(server, client: openai.AsyncOpenAI):
1406
)
1407
async def test_single_embedding(embedding_server, client: openai.AsyncOpenAI,
1408
model_name: str):
1410
- input = [
1409
+ input_texts = [
1410
"The chef prepared a delicious meal.",
1411
]
1412
1413
# test single embedding
1414
embeddings = await client.embeddings.create(
1415
model=model_name,
1417
- input=input,
1416
+ input=input_texts,
1417
encoding_format="float",
1418
)
1419
assert embeddings.id is not None
1421
- assert embeddings.data is not None and len(embeddings.data) == 1
1420
+ assert len(embeddings.data) == 1
1421
assert len(embeddings.data[0].embedding) == 4096
1422
assert embeddings.usage.completion_tokens == 0
1423
assert embeddings.usage.prompt_tokens == 9
1424
assert embeddings.usage.total_tokens == 9
1425
1426
# test using token IDs
1428
- input = [1, 1, 1, 1, 1]
1427
+ input_tokens = [1, 1, 1, 1, 1]
1428
embeddings = await client.embeddings.create(
1429
model=model_name,
1431
- input=input,
1430
+ input=input_tokens,
1431
encoding_format="float",
1432
)
1433
assert embeddings.id is not None
1435
- assert embeddings.data is not None and len(embeddings.data) == 1
1434
+ assert len(embeddings.data) == 1
1435
assert len(embeddings.data[0].embedding) == 4096
1436
assert embeddings.usage.completion_tokens == 0
1437
assert embeddings.usage.prompt_tokens == 5
@@ -1447,29 +1446,29 @@ async def test_single_embedding(embedding_server, client: openai.AsyncOpenAI,
1446
async def test_batch_embedding(embedding_server, client: openai.AsyncOpenAI,
1447
model_name: str):
1448
# test List[str]
1450
- inputs = [
1449
+ input_texts = [
1450
"The cat sat on the mat.", "A feline was resting on a rug.",
1451
"Stars twinkle brightly in the night sky."
1452
]
1453
embeddings = await client.embeddings.create(
1454
model=model_name,
1456
- input=inputs,
1455
+ input=input_texts,
1456
encoding_format="float",
1457
)
1458
assert embeddings.id is not None
1460
- assert embeddings.data is not None and len(embeddings.data) == 3
1459
+ assert len(embeddings.data) == 3
1460
assert len(embeddings.data[0].embedding) == 4096
1461
1462
# test List[List[int]]
1464
- inputs = [[4, 5, 7, 9, 20], [15, 29, 499], [24, 24, 24, 24, 24],
1465
- [25, 32, 64, 77]]
1463
+ input_tokens = [[4, 5, 7, 9, 20], [15, 29, 499], [24, 24, 24, 24, 24],
1464
+ [25, 32, 64, 77]]
1465
embeddings = await client.embeddings.create(
1466
model=model_name,
1468
- input=inputs,
1467
+ input=input_tokens,
1468
encoding_format="float",
1469
)
1470
assert embeddings.id is not None
1472
- assert embeddings.data is not None and len(embeddings.data) == 4
1471
+ assert len(embeddings.data) == 4
1472
assert len(embeddings.data[0].embedding) == 4096
1473
assert embeddings.usage.completion_tokens == 0
1474
assert embeddings.usage.prompt_tokens == 17
tests/tensorizer_loader/test_tensorizer.py
+2
-3
@@ -209,9 +209,8 @@ def test_openai_apiserver_with_tensorizer(vllm_runner, tmp_path):
209
temperature=0.0)
210
211
assert completion.id is not None
212
- assert completion.choices is not None and len(completion.choices) == 1
213
- assert completion.choices[0].text is not None and len(
214
- completion.choices[0].text) >= 5
212
+ assert len(completion.choices) == 1
213
+ assert len(completion.choices[0].text) >= 5
214
assert completion.choices[0].finish_reason == "length"
215
assert completion.usage == openai.types.CompletionUsage(
216
completion_tokens=5, prompt_tokens=6, total_tokens=11)
vllm/entrypoints/openai/protocol.py
+4
-3
@@ -513,7 +513,8 @@ class CompletionLogProbs(OpenAIBaseModel):
513
text_offset: List[int] = Field(default_factory=list)
514
token_logprobs: List[Optional[float]] = Field(default_factory=list)
515
tokens: List[str] = Field(default_factory=list)
516
- top_logprobs: Optional[List[Optional[Dict[str, float]]]] = None
516
+ top_logprobs: List[Optional[Dict[str,
517
+ float]]] = Field(default_factory=list)
518
519
520
class CompletionResponseChoice(OpenAIBaseModel):
@@ -612,7 +613,7 @@ class ChatCompletionResponseChoice(OpenAIBaseModel):
613
index: int
614
message: ChatMessage
615
logprobs: Optional[ChatCompletionLogProbs] = None
615
- finish_reason: Optional[Literal["stop", "length", "tool_calls"]] = None
616
+ finish_reason: Optional[str] = None
617
stop_reason: Optional[Union[int, str]] = None
618
619
@@ -635,7 +636,7 @@ class ChatCompletionResponseStreamChoice(OpenAIBaseModel):
636
index: int
637
delta: DeltaMessage
638
logprobs: Optional[ChatCompletionLogProbs] = None
638
- finish_reason: Optional[Literal["stop", "length", "tool_calls"]] = None
639
+ finish_reason: Optional[str] = None
640
stop_reason: Optional[Union[int, str]] = None
641
642
vllm/entrypoints/openai/serving_chat.py
+9
-6
@@ -373,13 +373,15 @@ class OpenAIServingChat(OpenAIServing):
373
continue
374
375
delta_token_ids = output.token_ids[previous_num_tokens[i]:]
376
- top_logprobs = output.logprobs[
376
+ out_logprobs = output.logprobs[
377
previous_num_tokens[i]:] if output.logprobs else None
378
379
- if request.logprobs:
379
+ if request.logprobs and request.top_logprobs is not None:
380
+ assert out_logprobs is not None, (
381
+ "Did not output logprobs")
382
logprobs = self._create_chat_logprobs(
383
token_ids=delta_token_ids,
382
- top_logprobs=top_logprobs,
384
+ top_logprobs=out_logprobs,
385
num_output_top_logprobs=request.top_logprobs,
386
)
387
else:
@@ -490,12 +492,13 @@ class OpenAIServingChat(OpenAIServing):
492
role = self.get_chat_request_role(request)
493
for output in final_res.outputs:
494
token_ids = output.token_ids
493
- top_logprobs = output.logprobs
495
+ out_logprobs = output.logprobs
496
495
- if request.logprobs:
497
+ if request.logprobs and request.top_logprobs is not None:
498
+ assert out_logprobs is not None, "Did not output logprobs"
499
logprobs = self._create_chat_logprobs(
500
token_ids=token_ids,
498
- top_logprobs=top_logprobs,
501
+ top_logprobs=out_logprobs,
502
num_output_top_logprobs=request.top_logprobs,
503
)
504
else:
vllm/entrypoints/openai/serving_completion.py
+12
-12
@@ -8,6 +8,7 @@ from fastapi import Request
8
9
from vllm.config import ModelConfig
10
from vllm.engine.async_llm_engine import AsyncLLMEngine
11
+# yapf conflicts with isort for this block
12
# yapf: disable
13
from vllm.entrypoints.openai.protocol import (CompletionLogProbs,
14
CompletionRequest,
@@ -16,7 +17,6 @@ from vllm.entrypoints.openai.protocol import (CompletionLogProbs,
17
CompletionResponseStreamChoice,
18
CompletionStreamResponse,
19
UsageInfo)
19
-# yapf: enable
20
from vllm.entrypoints.openai.serving_engine import (LoRAModulePath,
21
OpenAIServing)
22
from vllm.logger import init_logger
@@ -221,7 +221,7 @@ class OpenAIServingCompletion(OpenAIServing):
221
# only return the prompt
222
delta_text = res.prompt
223
delta_token_ids = res.prompt_token_ids
224
- top_logprobs = res.prompt_logprobs
224
+ out_logprobs = res.prompt_logprobs
225
has_echoed[i] = True
226
elif (request.echo and request.max_tokens > 0
227
and not has_echoed[i]):
@@ -229,7 +229,7 @@ class OpenAIServingCompletion(OpenAIServing):
229
delta_text = res.prompt + output.text
230
delta_token_ids = (res.prompt_token_ids +
231
output.token_ids)
232
- top_logprobs = res.prompt_logprobs + (output.logprobs
232
+ out_logprobs = res.prompt_logprobs + (output.logprobs
233
or [])
234
has_echoed[i] = True
235
else:
@@ -237,13 +237,15 @@ class OpenAIServingCompletion(OpenAIServing):
237
delta_text = output.text[len(previous_texts[i]):]
238
delta_token_ids = output.token_ids[
239
previous_num_tokens[i]:]
240
- top_logprobs = output.logprobs[previous_num_tokens[
240
+ out_logprobs = output.logprobs[previous_num_tokens[
241
i]:] if output.logprobs else None
242
243
if request.logprobs is not None:
244
+ assert out_logprobs is not None, (
245
+ "Did not output logprobs")
246
logprobs = self._create_completion_logprobs(
247
token_ids=delta_token_ids,
246
- top_logprobs=top_logprobs,
248
+ top_logprobs=out_logprobs,
249
num_output_top_logprobs=request.logprobs,
250
initial_text_offset=len(previous_texts[i]),
251
)
@@ -325,25 +327,23 @@ class OpenAIServingCompletion(OpenAIServing):
327
assert request.max_tokens is not None
328
if request.echo and request.max_tokens == 0:
329
token_ids = prompt_token_ids
328
- top_logprobs = prompt_logprobs
330
+ out_logprobs = prompt_logprobs
331
output_text = prompt_text
332
elif request.echo and request.max_tokens > 0:
333
token_ids = prompt_token_ids + output.token_ids
332
- top_logprobs = (prompt_logprobs + output.logprobs
334
+ out_logprobs = (prompt_logprobs + output.logprobs
335
if request.logprobs is not None else None)
336
output_text = prompt_text + output.text
337
else:
338
token_ids = output.token_ids
337
- top_logprobs = output.logprobs
339
+ out_logprobs = output.logprobs
340
output_text = output.text
341
342
if request.logprobs is not None:
341
- assert top_logprobs is not None, (
342
- "top_logprobs must be provided when logprobs "
343
- "is requested")
343
+ assert out_logprobs is not None, "Did not output logprobs"
344
logprobs = self._create_completion_logprobs(
345
token_ids=token_ids,
346
- top_logprobs=top_logprobs,
346
+ top_logprobs=out_logprobs,
347
num_output_top_logprobs=request.logprobs,
348
)
349
else: