@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / ba30942240
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 10:45 UTC

[Bugfix] Fix vLLM UsageInfo and logprobs None AssertionError with empty token_ids (#9034)

Co-authored-by: Nick Hill <nickhill@us.ibm.com>

Chang Su committed Oct 15, 2024 at 15:40 UTC ba30942240d35eb26b503e139eb4b01ccbbeb954
4 files changed +140 -2
tests/entrypoints/openai/test_chunked_prompt.py new
+126
@@ -0,0 +1,126 @@
1 +import openai # use the official client for correctness check
2 +import pytest
3 +import pytest_asyncio
4 +
5 +from ...utils import RemoteOpenAIServer
6 +
7 +# any model with a chat template should work here
8 +MODEL_NAME = "HuggingFaceH4/zephyr-7b-beta"
9 +
10 +
11 +@pytest.fixture(scope="module")
12 +def server():
13 + args = [
14 + # use half precision for speed and memory savings in CI environment
15 + "--dtype",
16 + "bfloat16",
17 + "--max-model-len",
18 + "8192",
19 + "--enforce-eager",
20 + # lora config below
21 + "--max-num-seqs",
22 + "128",
23 + "--enable-chunked-prefill",
24 + "--max-num-batched-tokens",
25 + "1000",
26 + # large prompts create a lot of output
27 + "--disable-log-requests",
28 + ]
29 +
30 + with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
31 + yield remote_server
32 +
33 +
34 +@pytest_asyncio.fixture
35 +async def client(server):
36 + async with server.get_async_client() as async_client:
37 + yield async_client
38 +
39 +
40 +@pytest.mark.asyncio
41 +async def test_completion_stream_options_and_logprobs_with_long_prompts(
42 + client: openai.AsyncOpenAI):
43 + # Test stream with long prompt
44 + prompt = "What is the capital of France?" * 400
45 +
46 + stream = await client.completions.create(
47 + model=MODEL_NAME,
48 + prompt=prompt,
49 + max_tokens=5,
50 + temperature=0.0,
51 + stream=True,
52 + stream_options={
53 + "include_usage": True,
54 + "continuous_usage_stats": True,
55 + },
56 + logprobs=5,
57 + )
58 +
59 + tokens_received = 0
60 + finished = False
61 + async for chunk in stream:
62 + assert chunk.usage.prompt_tokens >= 0
63 + assert chunk.usage.completion_tokens >= 0
64 + assert chunk.usage.total_tokens == (chunk.usage.prompt_tokens +
65 + chunk.usage.completion_tokens)
66 + if not finished:
67 + tokens_received += 1
68 + assert chunk.choices[0].text
69 +
70 + if chunk.choices[0].finish_reason is not None:
71 + finished = True
72 +
73 + if finished:
74 + assert chunk.usage.completion_tokens == tokens_received
75 +
76 +
77 +@pytest.mark.asyncio
78 +async def test_chat_completion_stream_options_and_logprobs_with_long_prompts(
79 + client: openai.AsyncOpenAI):
80 + # Test stream with long prompt
81 + messages = [{
82 + "role": "system",
83 + "content": "You are a helpful assistant."
84 + }, {
85 + "role": "user",
86 + "content": "What is the capital of France?" * 400
87 + }]
88 + stream = await client.chat.completions.create(
89 + model=MODEL_NAME,
90 + messages=messages,
91 + max_tokens=5,
92 + temperature=0.0,
93 + stream=True,
94 + stream_options={
95 + "include_usage": True,
96 + "continuous_usage_stats": True,
97 + },
98 + logprobs=True,
99 + top_logprobs=5,
100 + )
101 +
102 + tokens_received = 0
103 + empty_chunks_received = 0
104 + finished = False
105 + async for chunk in stream:
106 + assert chunk.usage.prompt_tokens >= 0
107 + assert chunk.usage.completion_tokens >= 0
108 + assert chunk.usage.total_tokens == (chunk.usage.prompt_tokens +
109 + chunk.usage.completion_tokens)
110 +
111 + if not finished:
112 + if chunk.choices[0].delta.content == "":
113 + # when there is no tokens generated
114 + assert chunk.usage.completion_tokens == 0
115 + assert chunk.choices[0].logprobs is None
116 + empty_chunks_received += 1
117 + else:
118 + tokens_received += 1
119 +
120 + if chunk.choices[0].finish_reason is not None:
121 + finished = True
122 +
123 + if finished:
124 + assert chunk.usage.completion_tokens == tokens_received
125 +
126 + assert empty_chunks_received <= 1
vllm/entrypoints/openai/serving_chat.py
+6
@@ -435,6 +435,12 @@ class OpenAIServingChat(OpenAIServing):
435 logprobs = None
436
437 delta_text = output.text
438 +
439 + if not delta_text and not output.token_ids and \
440 + not previous_num_tokens[i]:
441 + # Chunked prefill case, don't return empty chunks
442 + continue
443 +
444 delta_message: Optional[DeltaMessage]
445
446 # handle streaming deltas for tools with named tool_choice
vllm/entrypoints/openai/serving_completion.py
+5 -2
@@ -274,8 +274,6 @@ class OpenAIServingCompletion(OpenAIServing):
274
275 for output in res.outputs:
276 i = output.index + prompt_idx * num_choices
277 - # TODO(simon): optimize the performance by avoiding full
278 - # text O(n^2) sending.
277
278 assert request.max_tokens is not None
279 if request.echo and request.max_tokens == 0:
@@ -307,6 +305,11 @@ class OpenAIServingCompletion(OpenAIServing):
305 delta_token_ids = output.token_ids
306 out_logprobs = output.logprobs
307
308 + if not delta_text and not delta_token_ids \
309 + and not previous_num_tokens[i]:
310 + # Chunked prefill case, don't return empty chunks
311 + continue
312 +
313 if request.logprobs is not None:
314 assert out_logprobs is not None, (
315 "Did not output logprobs")
vllm/sequence.py
+3
@@ -532,6 +532,9 @@ class Sequence:
532 # (which is what we have most of the time)
533 return self.data._cached_all_token_ids[-1]
534
535 + if num_new_tokens == 0:
536 + return []
537 +
538 return self.data._cached_all_token_ids[-num_new_tokens:]
539
540 def hash_of_block(self, logical_idx: int) -> int: