A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 10:45 UTC
[Bugfix] Fix vLLM UsageInfo and logprobs None AssertionError with empty token_ids (#9034)
Co-authored-by: Nick Hill <nickhill@us.ibm.com>
Chang Su committed
Oct 15, 2024 at 15:40 UTC
ba30942240d35eb26b503e139eb4b01ccbbeb954
4 files changed
+140
-2
tests/entrypoints/openai/test_chunked_prompt.py
new
+126
@@ -0,0 +1,126 @@
1
+import openai # use the official client for correctness check
2
+import pytest
3
+import pytest_asyncio
4
+
5
+from ...utils import RemoteOpenAIServer
6
+
7
+# any model with a chat template should work here
8
+MODEL_NAME = "HuggingFaceH4/zephyr-7b-beta"
9
+
10
+
11
+@pytest.fixture(scope="module")
12
+def server():
13
+ args = [
14
+ # use half precision for speed and memory savings in CI environment
15
+ "--dtype",
16
+ "bfloat16",
17
+ "--max-model-len",
18
+ "8192",
19
+ "--enforce-eager",
20
+ # lora config below
21
+ "--max-num-seqs",
22
+ "128",
23
+ "--enable-chunked-prefill",
24
+ "--max-num-batched-tokens",
25
+ "1000",
26
+ # large prompts create a lot of output
27
+ "--disable-log-requests",
28
+ ]
29
+
30
+ with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
31
+ yield remote_server
32
+
33
+
34
+@pytest_asyncio.fixture
35
+async def client(server):
36
+ async with server.get_async_client() as async_client:
37
+ yield async_client
38
+
39
+
40
+@pytest.mark.asyncio
41
+async def test_completion_stream_options_and_logprobs_with_long_prompts(
42
+ client: openai.AsyncOpenAI):
43
+ # Test stream with long prompt
44
+ prompt = "What is the capital of France?" * 400
45
+
46
+ stream = await client.completions.create(
47
+ model=MODEL_NAME,
48
+ prompt=prompt,
49
+ max_tokens=5,
50
+ temperature=0.0,
51
+ stream=True,
52
+ stream_options={
53
+ "include_usage": True,
54
+ "continuous_usage_stats": True,
55
+ },
56
+ logprobs=5,
57
+ )
58
+
59
+ tokens_received = 0
60
+ finished = False
61
+ async for chunk in stream:
62
+ assert chunk.usage.prompt_tokens >= 0
63
+ assert chunk.usage.completion_tokens >= 0
64
+ assert chunk.usage.total_tokens == (chunk.usage.prompt_tokens +
65
+ chunk.usage.completion_tokens)
66
+ if not finished:
67
+ tokens_received += 1
68
+ assert chunk.choices[0].text
69
+
70
+ if chunk.choices[0].finish_reason is not None:
71
+ finished = True
72
+
73
+ if finished:
74
+ assert chunk.usage.completion_tokens == tokens_received
75
+
76
+
77
+@pytest.mark.asyncio
78
+async def test_chat_completion_stream_options_and_logprobs_with_long_prompts(
79
+ client: openai.AsyncOpenAI):
80
+ # Test stream with long prompt
81
+ messages = [{
82
+ "role": "system",
83
+ "content": "You are a helpful assistant."
84
+ }, {
85
+ "role": "user",
86
+ "content": "What is the capital of France?" * 400
87
+ }]
88
+ stream = await client.chat.completions.create(
89
+ model=MODEL_NAME,
90
+ messages=messages,
91
+ max_tokens=5,
92
+ temperature=0.0,
93
+ stream=True,
94
+ stream_options={
95
+ "include_usage": True,
96
+ "continuous_usage_stats": True,
97
+ },
98
+ logprobs=True,
99
+ top_logprobs=5,
100
+ )
101
+
102
+ tokens_received = 0
103
+ empty_chunks_received = 0
104
+ finished = False
105
+ async for chunk in stream:
106
+ assert chunk.usage.prompt_tokens >= 0
107
+ assert chunk.usage.completion_tokens >= 0
108
+ assert chunk.usage.total_tokens == (chunk.usage.prompt_tokens +
109
+ chunk.usage.completion_tokens)
110
+
111
+ if not finished:
112
+ if chunk.choices[0].delta.content == "":
113
+ # when there is no tokens generated
114
+ assert chunk.usage.completion_tokens == 0
115
+ assert chunk.choices[0].logprobs is None
116
+ empty_chunks_received += 1
117
+ else:
118
+ tokens_received += 1
119
+
120
+ if chunk.choices[0].finish_reason is not None:
121
+ finished = True
122
+
123
+ if finished:
124
+ assert chunk.usage.completion_tokens == tokens_received
125
+
126
+ assert empty_chunks_received <= 1
vllm/entrypoints/openai/serving_chat.py
+6
@@ -435,6 +435,12 @@ class OpenAIServingChat(OpenAIServing):
435
logprobs = None
436
437
delta_text = output.text
438
+
439
+ if not delta_text and not output.token_ids and \
440
+ not previous_num_tokens[i]:
441
+ # Chunked prefill case, don't return empty chunks
442
+ continue
443
+
444
delta_message: Optional[DeltaMessage]
445
446
# handle streaming deltas for tools with named tool_choice
vllm/entrypoints/openai/serving_completion.py
+5
-2
@@ -274,8 +274,6 @@ class OpenAIServingCompletion(OpenAIServing):
274
275
for output in res.outputs:
276
i = output.index + prompt_idx * num_choices
277
- # TODO(simon): optimize the performance by avoiding full
278
- # text O(n^2) sending.
277
278
assert request.max_tokens is not None
279
if request.echo and request.max_tokens == 0:
@@ -307,6 +305,11 @@ class OpenAIServingCompletion(OpenAIServing):
305
delta_token_ids = output.token_ids
306
out_logprobs = output.logprobs
307
308
+ if not delta_text and not delta_token_ids \
309
+ and not previous_num_tokens[i]:
310
+ # Chunked prefill case, don't return empty chunks
311
+ continue
312
+
313
if request.logprobs is not None:
314
assert out_logprobs is not None, (
315
"Did not output logprobs")
vllm/sequence.py
+3
@@ -532,6 +532,9 @@ class Sequence:
532
# (which is what we have most of the time)
533
return self.data._cached_all_token_ids[-1]
534
535
+ if num_new_tokens == 0:
536
+ return []
537
+
538
return self.data._cached_all_token_ids[-num_new_tokens:]
539
540
def hash_of_block(self, logical_idx: int) -> int: