A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 19:30 UTC
[Bugfix] Set SamplingParams.max_tokens for OpenAI requests if not provided by user (#6954)
zifeitong committed
Jul 31, 2024 at 21:13 UTC
3c10591ef2e78fbb6aa341195c4b24c36ae8b84d
5 files changed
+92
-44
tests/entrypoints/openai/test_serving_chat.py
+39
@@ -1,7 +1,12 @@
1
import asyncio
2
+from contextlib import suppress
3
from dataclasses import dataclass
4
+from unittest.mock import MagicMock
5
6
+from vllm.engine.async_llm_engine import AsyncLLMEngine
7
+from vllm.entrypoints.openai.protocol import ChatCompletionRequest
8
from vllm.entrypoints.openai.serving_chat import OpenAIServingChat
9
+from vllm.transformers_utils.tokenizer import get_tokenizer
10
11
MODEL_NAME = "openai-community/gpt2"
12
CHAT_TEMPLATE = "Dummy chat template for testing {}"
@@ -42,3 +47,37 @@ async def _async_serving_chat_init():
47
def test_async_serving_chat_init():
48
serving_completion = asyncio.run(_async_serving_chat_init())
49
assert serving_completion.chat_template == CHAT_TEMPLATE
50
+
51
+
52
+def test_serving_chat_should_set_correct_max_tokens():
53
+ mock_engine = MagicMock(spec=AsyncLLMEngine)
54
+ mock_engine.get_tokenizer.return_value = get_tokenizer(MODEL_NAME)
55
+
56
+ serving_chat = OpenAIServingChat(mock_engine,
57
+ MockModelConfig(),
58
+ served_model_names=[MODEL_NAME],
59
+ response_role="assistant",
60
+ chat_template=CHAT_TEMPLATE,
61
+ lora_modules=None,
62
+ prompt_adapters=None,
63
+ request_logger=None)
64
+ req = ChatCompletionRequest(
65
+ model=MODEL_NAME,
66
+ messages=[{
67
+ "role": "user",
68
+ "content": "what is 1+1?"
69
+ }],
70
+ guided_decoding_backend="outlines",
71
+ )
72
+
73
+ with suppress(Exception):
74
+ asyncio.run(serving_chat.create_chat_completion(req))
75
+
76
+ # AsyncLLMEngine.generate(inputs, sampling_params, ...)
77
+ assert mock_engine.generate.call_args.args[1].max_tokens == 93
78
+
79
+ req.max_tokens = 10
80
+ with suppress(Exception):
81
+ asyncio.run(serving_chat.create_chat_completion(req))
82
+
83
+ assert mock_engine.generate.call_args.args[1].max_tokens == 10
vllm/entrypoints/openai/protocol.py
+23
-7
@@ -11,7 +11,7 @@ from typing_extensions import Annotated
11
from vllm.entrypoints.chat_utils import ChatCompletionMessageParam
12
from vllm.entrypoints.openai.logits_processors import get_logits_processors
13
from vllm.pooling_params import PoolingParams
14
-from vllm.sampling_params import SamplingParams
14
+from vllm.sampling_params import LogitsProcessor, SamplingParams
15
from vllm.utils import random_uuid
16
17
@@ -215,15 +215,22 @@ class ChatCompletionRequest(OpenAIBaseModel):
215
216
# doc: end-chat-completion-extra-params
217
218
- def to_sampling_params(self,
219
- tokenizer: PreTrainedTokenizer) -> SamplingParams:
220
- # We now allow logprobs being true without top_logrobs.
218
+ def to_sampling_params(
219
+ self, tokenizer: PreTrainedTokenizer,
220
+ guided_decode_logits_processor: Optional[LogitsProcessor],
221
+ default_max_tokens: int) -> SamplingParams:
222
+ max_tokens = self.max_tokens
223
+ if max_tokens is None:
224
+ max_tokens = default_max_tokens
225
226
+ # We now allow logprobs being true without top_logrobs.
227
logits_processors = get_logits_processors(
228
logit_bias=self.logit_bias,
229
allowed_token_ids=None,
230
tokenizer=tokenizer,
231
)
232
+ if guided_decode_logits_processor:
233
+ logits_processors.append(guided_decode_logits_processor)
234
235
return SamplingParams(
236
n=self.n,
@@ -241,7 +248,7 @@ class ChatCompletionRequest(OpenAIBaseModel):
248
logprobs=self.top_logprobs if self.logprobs else None,
249
prompt_logprobs=self.top_logprobs if self.echo else None,
250
ignore_eos=self.ignore_eos,
244
- max_tokens=self.max_tokens,
251
+ max_tokens=max_tokens,
252
min_tokens=self.min_tokens,
253
use_beam_search=self.use_beam_search,
254
early_stopping=self.early_stopping,
@@ -395,7 +402,14 @@ class CompletionRequest(OpenAIBaseModel):
402
403
# doc: end-completion-extra-params
404
398
- def to_sampling_params(self, tokenizer: PreTrainedTokenizer):
405
+ def to_sampling_params(
406
+ self, tokenizer: PreTrainedTokenizer,
407
+ guided_decode_logits_processor: Optional[LogitsProcessor],
408
+ default_max_tokens: int) -> SamplingParams:
409
+ max_tokens = self.max_tokens
410
+ if max_tokens is None:
411
+ max_tokens = default_max_tokens
412
+
413
echo_without_generation = self.echo and self.max_tokens == 0
414
415
logits_processors = get_logits_processors(
@@ -403,6 +417,8 @@ class CompletionRequest(OpenAIBaseModel):
417
allowed_token_ids=self.allowed_token_ids,
418
tokenizer=tokenizer,
419
)
420
+ if guided_decode_logits_processor:
421
+ logits_processors.append(guided_decode_logits_processor)
422
423
return SamplingParams(
424
n=self.n,
@@ -419,7 +435,7 @@ class CompletionRequest(OpenAIBaseModel):
435
stop_token_ids=self.stop_token_ids,
436
logprobs=self.logprobs,
437
ignore_eos=self.ignore_eos,
422
- max_tokens=self.max_tokens if not echo_without_generation else 1,
438
+ max_tokens=max_tokens if not echo_without_generation else 1,
439
min_tokens=self.min_tokens,
440
use_beam_search=self.use_beam_search,
441
early_stopping=self.early_stopping,
vllm/entrypoints/openai/serving_chat.py
+8
-15
@@ -25,8 +25,6 @@ from vllm.entrypoints.openai.serving_engine import (LoRAModulePath,
25
PromptAdapterPath)
26
from vllm.inputs import PromptInputs
27
from vllm.logger import init_logger
28
-from vllm.model_executor.guided_decoding import (
29
- get_guided_decoding_logits_processor)
28
from vllm.multimodal import MultiModalDataDict
29
from vllm.outputs import RequestOutput
30
from vllm.sequence import Logprob
@@ -134,28 +132,23 @@ class OpenAIServingChat(OpenAIServing):
132
133
request_id = f"chat-{random_uuid()}"
134
try:
137
- sampling_params = request.to_sampling_params(tokenizer)
138
- decoding_config = await self.engine.get_decoding_config()
139
- guided_decoding_backend = request.guided_decoding_backend \
140
- or decoding_config.guided_decoding_backend
135
guided_decode_logits_processor = (
142
- await
143
- get_guided_decoding_logits_processor(guided_decoding_backend,
144
- request, tokenizer))
145
- if guided_decode_logits_processor:
146
- if sampling_params.logits_processors is None:
147
- sampling_params.logits_processors = []
148
- sampling_params.logits_processors.append(
149
- guided_decode_logits_processor)
136
+ await self._guided_decode_logits_processor(request, tokenizer))
137
138
prompt_inputs = self._tokenize_prompt_input(
139
request,
140
tokenizer,
141
prompt,
155
- truncate_prompt_tokens=sampling_params.truncate_prompt_tokens,
142
+ truncate_prompt_tokens=request.truncate_prompt_tokens,
143
add_special_tokens=request.add_special_tokens,
144
)
145
146
+ sampling_params = request.to_sampling_params(
147
+ tokenizer,
148
+ guided_decode_logits_processor,
149
+ default_max_tokens=self.max_model_len -
150
+ len(prompt_inputs["prompt_token_ids"]))
151
+
152
self._log_inputs(request_id,
153
prompt_inputs,
154
params=sampling_params,
vllm/entrypoints/openai/serving_completion.py
+9
-18
@@ -24,8 +24,6 @@ from vllm.entrypoints.openai.serving_engine import (LoRAModulePath,
24
OpenAIServing,
25
PromptAdapterPath)
26
from vllm.logger import init_logger
27
-from vllm.model_executor.guided_decoding import (
28
- get_guided_decoding_logits_processor)
27
from vllm.outputs import RequestOutput
28
from vllm.sequence import Logprob
29
from vllm.tracing import (contains_trace_headers, extract_trace_headers,
@@ -95,31 +93,24 @@ class OpenAIServingCompletion(OpenAIServing):
93
94
tokenizer = await self.engine.get_tokenizer(lora_request)
95
98
- sampling_params = request.to_sampling_params(tokenizer)
99
- decoding_config = await self.engine.get_decoding_config()
100
- guided_decoding_backend = request.guided_decoding_backend \
101
- or decoding_config.guided_decoding_backend
102
- guided_decode_logit_processor = (
103
- await
104
- get_guided_decoding_logits_processor(guided_decoding_backend,
105
- request, tokenizer))
106
- if guided_decode_logit_processor is not None:
107
- if sampling_params.logits_processors is None:
108
- sampling_params.logits_processors = []
109
- sampling_params.logits_processors.append(
110
- guided_decode_logit_processor)
111
-
96
+ guided_decode_logits_processor = (
97
+ await self._guided_decode_logits_processor(request, tokenizer))
98
prompts = list(
99
self._tokenize_prompt_input_or_inputs(
100
request,
101
tokenizer,
102
request.prompt,
117
- truncate_prompt_tokens=sampling_params.
118
- truncate_prompt_tokens,
103
+ truncate_prompt_tokens=request.truncate_prompt_tokens,
104
add_special_tokens=request.add_special_tokens,
105
))
106
107
for i, prompt_inputs in enumerate(prompts):
108
+ sampling_params = request.to_sampling_params(
109
+ tokenizer,
110
+ guided_decode_logits_processor,
111
+ default_max_tokens=self.max_model_len -
112
+ len(prompt_inputs["prompt_token_ids"]))
113
+
114
request_id_item = f"{request_id}-{i}"
115
116
self._log_inputs(request_id_item,
vllm/entrypoints/openai/serving_engine.py
+13
-4
@@ -25,9 +25,11 @@ from vllm.entrypoints.openai.protocol import (ChatCompletionRequest,
25
from vllm.inputs import parse_and_batch_prompt
26
from vllm.logger import init_logger
27
from vllm.lora.request import LoRARequest
28
+from vllm.model_executor.guided_decoding import (
29
+ get_guided_decoding_logits_processor)
30
from vllm.pooling_params import PoolingParams
31
from vllm.prompt_adapter.request import PromptAdapterRequest
30
-from vllm.sampling_params import SamplingParams
32
+from vllm.sampling_params import LogitsProcessor, SamplingParams
33
from vllm.sequence import Logprob
34
from vllm.transformers_utils.tokenizer_group import AnyTokenizer
35
@@ -150,6 +152,15 @@ class OpenAIServing:
152
})
153
return json_str
154
155
+ async def _guided_decode_logits_processor(
156
+ self, request: Union[ChatCompletionRequest, CompletionRequest],
157
+ tokenizer: AnyTokenizer) -> Optional[LogitsProcessor]:
158
+ decoding_config = await self.engine.get_decoding_config()
159
+ guided_decoding_backend = request.guided_decoding_backend \
160
+ or decoding_config.guided_decoding_backend
161
+ return await get_guided_decoding_logits_processor(
162
+ guided_decoding_backend, request, tokenizer)
163
+
164
async def _check_model(
165
self,
166
request: AnyRequest,
@@ -254,9 +265,7 @@ class OpenAIServing:
265
f"{self.max_model_len} tokens. However, you requested "
266
f"{token_num} tokens in the messages, "
267
f"Please reduce the length of the messages.")
257
- request.max_tokens = self.max_model_len - token_num
258
-
259
- if token_num + request.max_tokens > self.max_model_len:
268
+ elif token_num + request.max_tokens > self.max_model_len:
269
raise ValueError(
270
f"This model's maximum context length is "
271
f"{self.max_model_len} tokens. However, you requested "