@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 640052b069
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 23:15 UTC

[Bugfix][Frontend] Cleanup "fix chat logprobs" (#5026)

Cyrus Leung committed Jun 11, 2024 at 13:36 UTC 640052b0698d64d03806c98bc118a425afc53eff
6 files changed +122 -123
tests/async_engine/test_openapi_server_ray.py
+11 -14
@@ -55,9 +55,8 @@ async def test_single_completion(server, client: openai.AsyncOpenAI):
55 temperature=0.0)
56
57 assert completion.id is not None
58 - assert completion.choices is not None and len(completion.choices) == 1
59 - assert completion.choices[0].text is not None and len(
60 - completion.choices[0].text) >= 5
58 + assert len(completion.choices) == 1
59 + assert len(completion.choices[0].text) >= 5
60 assert completion.choices[0].finish_reason == "length"
61 assert completion.usage == openai.types.CompletionUsage(
62 completion_tokens=5, prompt_tokens=6, total_tokens=11)
@@ -69,8 +68,7 @@ async def test_single_completion(server, client: openai.AsyncOpenAI):
68 max_tokens=5,
69 temperature=0.0,
70 )
72 - assert completion.choices[0].text is not None and len(
73 - completion.choices[0].text) >= 5
71 + assert len(completion.choices[0].text) >= 5
72
73
74 @pytest.mark.asyncio
@@ -90,15 +88,14 @@ async def test_single_chat_session(server, client: openai.AsyncOpenAI):
88 logprobs=True,
89 top_logprobs=5)
90 assert chat_completion.id is not None
93 - assert chat_completion.choices is not None and len(
94 - chat_completion.choices) == 1
95 - assert chat_completion.choices[0].message is not None
96 - assert chat_completion.choices[0].logprobs is not None
97 - assert chat_completion.choices[0].logprobs.content[
98 - 0].top_logprobs is not None
99 - assert len(
100 - chat_completion.choices[0].logprobs.content[0].top_logprobs) == 5
101 - message = chat_completion.choices[0].message
91 + assert len(chat_completion.choices) == 1
92 +
93 + choice = chat_completion.choices[0]
94 + assert choice.finish_reason == "length"
95 + assert chat_completion.usage == openai.types.CompletionUsage(
96 + completion_tokens=10, prompt_tokens=13, total_tokens=23)
97 +
98 + message = choice.message
99 assert message.content is not None and len(message.content) >= 10
100 assert message.role == "assistant"
101 messages.append({"role": "assistant", "content": message.content})
tests/entrypoints/test_openai_server.py
+84 -85
@@ -167,9 +167,10 @@ async def test_single_completion(server, client: openai.AsyncOpenAI,
167
168 assert completion.id is not None
169 assert completion.choices is not None and len(completion.choices) == 1
170 - assert completion.choices[0].text is not None and len(
171 - completion.choices[0].text) >= 5
172 - assert completion.choices[0].finish_reason == "length"
170 +
171 + choice = completion.choices[0]
172 + assert len(choice.text) >= 5
173 + assert choice.finish_reason == "length"
174 assert completion.usage == openai.types.CompletionUsage(
175 completion_tokens=5, prompt_tokens=6, total_tokens=11)
176
@@ -180,8 +181,7 @@ async def test_single_completion(server, client: openai.AsyncOpenAI,
181 max_tokens=5,
182 temperature=0.0,
183 )
183 - assert completion.choices[0].text is not None and len(
184 - completion.choices[0].text) >= 5
184 + assert len(completion.choices[0].text) >= 5
185
186
187 @pytest.mark.asyncio
@@ -206,9 +206,9 @@ async def test_no_logprobs(server, client: openai.AsyncOpenAI,
206
207 @pytest.mark.asyncio
208 @pytest.mark.parametrize(
209 - # first test base model, then test loras
209 + # just test 1 lora hereafter
210 "model_name",
211 - [MODEL_NAME, "zephyr-lora", "zephyr-lora2"],
211 + [MODEL_NAME, "zephyr-lora"],
212 )
213 async def test_zero_logprobs(server, client: openai.AsyncOpenAI,
214 model_name: str):
@@ -291,55 +291,7 @@ async def test_too_many_completion_logprobs(server, client: openai.AsyncOpenAI,
291 max_tokens=5,
292 temperature=0.0,
293 )
294 - completion = completion.choices[0].text
295 - assert completion is not None and len(completion) >= 0
296 -
297 -
298 -@pytest.mark.asyncio
299 -@pytest.mark.parametrize(
300 - # just test 1 lora hereafter
301 - "model_name",
302 - [MODEL_NAME, "zephyr-lora"],
303 -)
304 -async def test_single_chat_session(server, client: openai.AsyncOpenAI,
305 - model_name: str):
306 - messages = [{
307 - "role": "system",
308 - "content": "you are a helpful assistant"
309 - }, {
310 - "role": "user",
311 - "content": "what is 1+1?"
312 - }]
313 -
314 - # test single completion
315 - chat_completion = await client.chat.completions.create(model=model_name,
316 - messages=messages,
317 - max_tokens=10,
318 - logprobs=True,
319 - top_logprobs=5)
320 - assert chat_completion.id is not None
321 - assert chat_completion.choices is not None and len(
322 - chat_completion.choices) == 1
323 - assert chat_completion.choices[0].message is not None
324 - assert chat_completion.choices[0].logprobs is not None
325 - assert chat_completion.choices[0].logprobs.content[
326 - 0].top_logprobs is not None
327 - assert len(
328 - chat_completion.choices[0].logprobs.content[0].top_logprobs) == 5
329 - message = chat_completion.choices[0].message
330 - assert message.content is not None and len(message.content) >= 10
331 - assert message.role == "assistant"
332 - messages.append({"role": "assistant", "content": message.content})
333 -
334 - # test multi-turn dialogue
335 - messages.append({"role": "user", "content": "express your result in json"})
336 - chat_completion = await client.chat.completions.create(
337 - model=model_name,
338 - messages=messages,
339 - max_tokens=10,
340 - )
341 - message = chat_completion.choices[0].message
342 - assert message.content is not None and len(message.content) >= 0
294 + assert len(completion.choices[0].text) >= 0
295
296
297 @pytest.mark.asyncio
@@ -394,7 +346,7 @@ async def test_zero_logprobs_chat(server, client: openai.AsyncOpenAI,
346 choice = chat_completion.choices[0]
347 assert choice.logprobs is not None
348 assert choice.logprobs.content is not None
397 - assert len(choice.logprobs.content[0].top_logprobs) <= 1
349 + assert len(choice.logprobs.content[0].top_logprobs) == 0
350
351
352 @pytest.mark.asyncio
@@ -422,11 +374,14 @@ async def test_some_logprobs_chat(server, client: openai.AsyncOpenAI,
374 choice = chat_completion.choices[0]
375 assert choice.logprobs is not None
376 assert choice.logprobs.content is not None
425 - assert len(choice.logprobs.content[0].top_logprobs) <= 6
377 + assert len(choice.logprobs.content[0].top_logprobs) == 5
378
379
380 @pytest.mark.asyncio
429 -@pytest.mark.parametrize("model_name", [MODEL_NAME])
381 +@pytest.mark.parametrize(
382 + "model_name",
383 + [MODEL_NAME, "zephyr-lora"],
384 +)
385 async def test_too_many_chat_logprobs(server, client: openai.AsyncOpenAI,
386 model_name: str):
387 messages = [{
@@ -467,7 +422,51 @@ async def test_too_many_chat_logprobs(server, client: openai.AsyncOpenAI,
422
423 @pytest.mark.asyncio
424 @pytest.mark.parametrize(
470 - # just test 1 lora hereafter
425 + "model_name",
426 + [MODEL_NAME, "zephyr-lora"],
427 +)
428 +async def test_single_chat_session(server, client: openai.AsyncOpenAI,
429 + model_name: str):
430 + messages = [{
431 + "role": "system",
432 + "content": "you are a helpful assistant"
433 + }, {
434 + "role": "user",
435 + "content": "what is 1+1?"
436 + }]
437 +
438 + # test single completion
439 + chat_completion = await client.chat.completions.create(model=model_name,
440 + messages=messages,
441 + max_tokens=10,
442 + logprobs=True,
443 + top_logprobs=5)
444 + assert chat_completion.id is not None
445 + assert len(chat_completion.choices) == 1
446 +
447 + choice = chat_completion.choices[0]
448 + assert choice.finish_reason == "length"
449 + assert chat_completion.usage == openai.types.CompletionUsage(
450 + completion_tokens=10, prompt_tokens=37, total_tokens=47)
451 +
452 + message = choice.message
453 + assert message.content is not None and len(message.content) >= 10
454 + assert message.role == "assistant"
455 + messages.append({"role": "assistant", "content": message.content})
456 +
457 + # test multi-turn dialogue
458 + messages.append({"role": "user", "content": "express your result in json"})
459 + chat_completion = await client.chat.completions.create(
460 + model=model_name,
461 + messages=messages,
462 + max_tokens=10,
463 + )
464 + message = chat_completion.choices[0].message
465 + assert message.content is not None and len(message.content) >= 0
466 +
467 +
468 +@pytest.mark.asyncio
469 +@pytest.mark.parametrize(
470 "model_name",
471 [MODEL_NAME, "zephyr-lora"],
472 )
@@ -753,8 +752,7 @@ async def test_logits_bias(server, client: openai.AsyncOpenAI):
752 logit_bias={str(token_id): 100},
753 seed=42,
754 )
756 - assert completion.choices[0].text is not None and len(
757 - completion.choices[0].text) >= 5
755 + assert len(completion.choices[0].text) >= 5
756 response_tokens = tokenizer(completion.choices[0].text,
757 add_special_tokens=False)["input_ids"]
758 expected_tokens = tokenizer(tokenizer.decode([token_id] * 5),
@@ -801,9 +799,8 @@ async def test_guided_json_completion(server, client: openai.AsyncOpenAI,
799 guided_decoding_backend=guided_decoding_backend))
800
801 assert completion.id is not None
804 - assert completion.choices is not None and len(completion.choices) == 3
802 + assert len(completion.choices) == 3
803 for i in range(3):
806 - assert completion.choices[i].text is not None
804 output_json = json.loads(completion.choices[i].text)
805 jsonschema.validate(instance=output_json, schema=TEST_SCHEMA)
806
@@ -870,9 +867,8 @@ async def test_guided_regex_completion(server, client: openai.AsyncOpenAI,
867 guided_decoding_backend=guided_decoding_backend))
868
869 assert completion.id is not None
873 - assert completion.choices is not None and len(completion.choices) == 3
870 + assert len(completion.choices) == 3
871 for i in range(3):
875 - assert completion.choices[i].text is not None
872 assert re.fullmatch(TEST_REGEX, completion.choices[i].text) is not None
873
874
@@ -929,7 +925,7 @@ async def test_guided_choice_completion(server, client: openai.AsyncOpenAI,
925 guided_decoding_backend=guided_decoding_backend))
926
927 assert completion.id is not None
932 - assert completion.choices is not None and len(completion.choices) == 2
928 + assert len(completion.choices) == 2
929 for i in range(2):
930 assert completion.choices[i].text in TEST_CHOICE
931
@@ -1031,12 +1027,14 @@ async def test_guided_choice_chat_logprobs(server, client: openai.AsyncOpenAI,
1027 top_logprobs=5,
1028 extra_body=dict(guided_choice=TEST_CHOICE,
1029 guided_decoding_backend=guided_decoding_backend))
1030 +
1031 + assert chat_completion.choices[0].logprobs is not None
1032 + assert chat_completion.choices[0].logprobs.content is not None
1033 top_logprobs = chat_completion.choices[0].logprobs.content[0].top_logprobs
1034
1035 # -9999.0 is the minimum logprob returned by OpenAI
1037 - assert all(
1038 - isinstance(token.logprob, float) and token.logprob >= -9999.0
1039 - for token in top_logprobs)
1036 + for item in top_logprobs:
1037 + assert item.logprob >= -9999.0, f"Failed (top_logprobs={top_logprobs})"
1038
1039
1040 @pytest.mark.asyncio
@@ -1238,6 +1236,8 @@ async def test_response_format_json_object(server, client: openai.AsyncOpenAI):
1236 response_format={"type": "json_object"})
1237
1238 content = resp.choices[0].message.content
1239 + assert content is not None
1240 +
1241 loaded = json.loads(content)
1242 assert loaded == {"result": 2}, loaded
1243
@@ -1365,8 +1365,7 @@ async def test_echo_logprob_completion(server, client: openai.AsyncOpenAI,
1365
1366 prompt_text = tokenizer.decode(prompt) if isinstance(prompt,
1367 list) else prompt
1368 - assert (completion.choices[0].text is not None
1369 - and re.search(r"^" + prompt_text, completion.choices[0].text))
1368 + assert re.search(r"^" + prompt_text, completion.choices[0].text)
1369 logprobs = completion.choices[0].logprobs
1370 assert logprobs is not None
1371 assert len(logprobs.text_offset) > 5
@@ -1407,32 +1406,32 @@ async def test_long_seed(server, client: openai.AsyncOpenAI):
1406 )
1407 async def test_single_embedding(embedding_server, client: openai.AsyncOpenAI,
1408 model_name: str):
1410 - input = [
1409 + input_texts = [
1410 "The chef prepared a delicious meal.",
1411 ]
1412
1413 # test single embedding
1414 embeddings = await client.embeddings.create(
1415 model=model_name,
1417 - input=input,
1416 + input=input_texts,
1417 encoding_format="float",
1418 )
1419 assert embeddings.id is not None
1421 - assert embeddings.data is not None and len(embeddings.data) == 1
1420 + assert len(embeddings.data) == 1
1421 assert len(embeddings.data[0].embedding) == 4096
1422 assert embeddings.usage.completion_tokens == 0
1423 assert embeddings.usage.prompt_tokens == 9
1424 assert embeddings.usage.total_tokens == 9
1425
1426 # test using token IDs
1428 - input = [1, 1, 1, 1, 1]
1427 + input_tokens = [1, 1, 1, 1, 1]
1428 embeddings = await client.embeddings.create(
1429 model=model_name,
1431 - input=input,
1430 + input=input_tokens,
1431 encoding_format="float",
1432 )
1433 assert embeddings.id is not None
1435 - assert embeddings.data is not None and len(embeddings.data) == 1
1434 + assert len(embeddings.data) == 1
1435 assert len(embeddings.data[0].embedding) == 4096
1436 assert embeddings.usage.completion_tokens == 0
1437 assert embeddings.usage.prompt_tokens == 5
@@ -1447,29 +1446,29 @@ async def test_single_embedding(embedding_server, client: openai.AsyncOpenAI,
1446 async def test_batch_embedding(embedding_server, client: openai.AsyncOpenAI,
1447 model_name: str):
1448 # test List[str]
1450 - inputs = [
1449 + input_texts = [
1450 "The cat sat on the mat.", "A feline was resting on a rug.",
1451 "Stars twinkle brightly in the night sky."
1452 ]
1453 embeddings = await client.embeddings.create(
1454 model=model_name,
1456 - input=inputs,
1455 + input=input_texts,
1456 encoding_format="float",
1457 )
1458 assert embeddings.id is not None
1460 - assert embeddings.data is not None and len(embeddings.data) == 3
1459 + assert len(embeddings.data) == 3
1460 assert len(embeddings.data[0].embedding) == 4096
1461
1462 # test List[List[int]]
1464 - inputs = [[4, 5, 7, 9, 20], [15, 29, 499], [24, 24, 24, 24, 24],
1465 - [25, 32, 64, 77]]
1463 + input_tokens = [[4, 5, 7, 9, 20], [15, 29, 499], [24, 24, 24, 24, 24],
1464 + [25, 32, 64, 77]]
1465 embeddings = await client.embeddings.create(
1466 model=model_name,
1468 - input=inputs,
1467 + input=input_tokens,
1468 encoding_format="float",
1469 )
1470 assert embeddings.id is not None
1472 - assert embeddings.data is not None and len(embeddings.data) == 4
1471 + assert len(embeddings.data) == 4
1472 assert len(embeddings.data[0].embedding) == 4096
1473 assert embeddings.usage.completion_tokens == 0
1474 assert embeddings.usage.prompt_tokens == 17
tests/tensorizer_loader/test_tensorizer.py
+2 -3
@@ -209,9 +209,8 @@ def test_openai_apiserver_with_tensorizer(vllm_runner, tmp_path):
209 temperature=0.0)
210
211 assert completion.id is not None
212 - assert completion.choices is not None and len(completion.choices) == 1
213 - assert completion.choices[0].text is not None and len(
214 - completion.choices[0].text) >= 5
212 + assert len(completion.choices) == 1
213 + assert len(completion.choices[0].text) >= 5
214 assert completion.choices[0].finish_reason == "length"
215 assert completion.usage == openai.types.CompletionUsage(
216 completion_tokens=5, prompt_tokens=6, total_tokens=11)
vllm/entrypoints/openai/protocol.py
+4 -3
@@ -513,7 +513,8 @@ class CompletionLogProbs(OpenAIBaseModel):
513 text_offset: List[int] = Field(default_factory=list)
514 token_logprobs: List[Optional[float]] = Field(default_factory=list)
515 tokens: List[str] = Field(default_factory=list)
516 - top_logprobs: Optional[List[Optional[Dict[str, float]]]] = None
516 + top_logprobs: List[Optional[Dict[str,
517 + float]]] = Field(default_factory=list)
518
519
520 class CompletionResponseChoice(OpenAIBaseModel):
@@ -612,7 +613,7 @@ class ChatCompletionResponseChoice(OpenAIBaseModel):
613 index: int
614 message: ChatMessage
615 logprobs: Optional[ChatCompletionLogProbs] = None
615 - finish_reason: Optional[Literal["stop", "length", "tool_calls"]] = None
616 + finish_reason: Optional[str] = None
617 stop_reason: Optional[Union[int, str]] = None
618
619
@@ -635,7 +636,7 @@ class ChatCompletionResponseStreamChoice(OpenAIBaseModel):
636 index: int
637 delta: DeltaMessage
638 logprobs: Optional[ChatCompletionLogProbs] = None
638 - finish_reason: Optional[Literal["stop", "length", "tool_calls"]] = None
639 + finish_reason: Optional[str] = None
640 stop_reason: Optional[Union[int, str]] = None
641
642
vllm/entrypoints/openai/serving_chat.py
+9 -6
@@ -373,13 +373,15 @@ class OpenAIServingChat(OpenAIServing):
373 continue
374
375 delta_token_ids = output.token_ids[previous_num_tokens[i]:]
376 - top_logprobs = output.logprobs[
376 + out_logprobs = output.logprobs[
377 previous_num_tokens[i]:] if output.logprobs else None
378
379 - if request.logprobs:
379 + if request.logprobs and request.top_logprobs is not None:
380 + assert out_logprobs is not None, (
381 + "Did not output logprobs")
382 logprobs = self._create_chat_logprobs(
383 token_ids=delta_token_ids,
382 - top_logprobs=top_logprobs,
384 + top_logprobs=out_logprobs,
385 num_output_top_logprobs=request.top_logprobs,
386 )
387 else:
@@ -490,12 +492,13 @@ class OpenAIServingChat(OpenAIServing):
492 role = self.get_chat_request_role(request)
493 for output in final_res.outputs:
494 token_ids = output.token_ids
493 - top_logprobs = output.logprobs
495 + out_logprobs = output.logprobs
496
495 - if request.logprobs:
497 + if request.logprobs and request.top_logprobs is not None:
498 + assert out_logprobs is not None, "Did not output logprobs"
499 logprobs = self._create_chat_logprobs(
500 token_ids=token_ids,
498 - top_logprobs=top_logprobs,
501 + top_logprobs=out_logprobs,
502 num_output_top_logprobs=request.top_logprobs,
503 )
504 else:
vllm/entrypoints/openai/serving_completion.py
+12 -12
@@ -8,6 +8,7 @@ from fastapi import Request
8
9 from vllm.config import ModelConfig
10 from vllm.engine.async_llm_engine import AsyncLLMEngine
11 +# yapf conflicts with isort for this block
12 # yapf: disable
13 from vllm.entrypoints.openai.protocol import (CompletionLogProbs,
14 CompletionRequest,
@@ -16,7 +17,6 @@ from vllm.entrypoints.openai.protocol import (CompletionLogProbs,
17 CompletionResponseStreamChoice,
18 CompletionStreamResponse,
19 UsageInfo)
19 -# yapf: enable
20 from vllm.entrypoints.openai.serving_engine import (LoRAModulePath,
21 OpenAIServing)
22 from vllm.logger import init_logger
@@ -221,7 +221,7 @@ class OpenAIServingCompletion(OpenAIServing):
221 # only return the prompt
222 delta_text = res.prompt
223 delta_token_ids = res.prompt_token_ids
224 - top_logprobs = res.prompt_logprobs
224 + out_logprobs = res.prompt_logprobs
225 has_echoed[i] = True
226 elif (request.echo and request.max_tokens > 0
227 and not has_echoed[i]):
@@ -229,7 +229,7 @@ class OpenAIServingCompletion(OpenAIServing):
229 delta_text = res.prompt + output.text
230 delta_token_ids = (res.prompt_token_ids +
231 output.token_ids)
232 - top_logprobs = res.prompt_logprobs + (output.logprobs
232 + out_logprobs = res.prompt_logprobs + (output.logprobs
233 or [])
234 has_echoed[i] = True
235 else:
@@ -237,13 +237,15 @@ class OpenAIServingCompletion(OpenAIServing):
237 delta_text = output.text[len(previous_texts[i]):]
238 delta_token_ids = output.token_ids[
239 previous_num_tokens[i]:]
240 - top_logprobs = output.logprobs[previous_num_tokens[
240 + out_logprobs = output.logprobs[previous_num_tokens[
241 i]:] if output.logprobs else None
242
243 if request.logprobs is not None:
244 + assert out_logprobs is not None, (
245 + "Did not output logprobs")
246 logprobs = self._create_completion_logprobs(
247 token_ids=delta_token_ids,
246 - top_logprobs=top_logprobs,
248 + top_logprobs=out_logprobs,
249 num_output_top_logprobs=request.logprobs,
250 initial_text_offset=len(previous_texts[i]),
251 )
@@ -325,25 +327,23 @@ class OpenAIServingCompletion(OpenAIServing):
327 assert request.max_tokens is not None
328 if request.echo and request.max_tokens == 0:
329 token_ids = prompt_token_ids
328 - top_logprobs = prompt_logprobs
330 + out_logprobs = prompt_logprobs
331 output_text = prompt_text
332 elif request.echo and request.max_tokens > 0:
333 token_ids = prompt_token_ids + output.token_ids
332 - top_logprobs = (prompt_logprobs + output.logprobs
334 + out_logprobs = (prompt_logprobs + output.logprobs
335 if request.logprobs is not None else None)
336 output_text = prompt_text + output.text
337 else:
338 token_ids = output.token_ids
337 - top_logprobs = output.logprobs
339 + out_logprobs = output.logprobs
340 output_text = output.text
341
342 if request.logprobs is not None:
341 - assert top_logprobs is not None, (
342 - "top_logprobs must be provided when logprobs "
343 - "is requested")
343 + assert out_logprobs is not None, "Did not output logprobs"
344 logprobs = self._create_completion_logprobs(
345 token_ids=token_ids,
346 - top_logprobs=top_logprobs,
346 + top_logprobs=out_logprobs,
347 num_output_top_logprobs=request.logprobs,
348 )
349 else: