A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 09:30 UTC
[Frontend] re-enable multi-modality input in the new beam search implementation (#9427)
Signed-off-by: Qishuai Ferdinandzhong@gmail.com
Zhong Qishuai committed
Oct 29, 2024 at 19:49 UTC
ef7865b4f9013e1d328058091b12b28c4a078e91
7 files changed
+150
-40
tests/entrypoints/openai/test_vision.py
+71
@@ -107,6 +107,42 @@ async def test_single_chat_session_image(client: openai.AsyncOpenAI,
107
assert message.content is not None and len(message.content) >= 0
108
109
110
+@pytest.mark.asyncio
111
+@pytest.mark.parametrize("model_name", [MODEL_NAME])
112
+@pytest.mark.parametrize("image_url", TEST_IMAGE_URLS)
113
+async def test_single_chat_session_image_beamsearch(client: openai.AsyncOpenAI,
114
+ model_name: str,
115
+ image_url: str):
116
+ messages = [{
117
+ "role":
118
+ "user",
119
+ "content": [
120
+ {
121
+ "type": "image_url",
122
+ "image_url": {
123
+ "url": image_url
124
+ }
125
+ },
126
+ {
127
+ "type": "text",
128
+ "text": "What's in this image?"
129
+ },
130
+ ],
131
+ }]
132
+
133
+ chat_completion = await client.chat.completions.create(
134
+ model=model_name,
135
+ messages=messages,
136
+ n=2,
137
+ max_tokens=10,
138
+ logprobs=True,
139
+ top_logprobs=5,
140
+ extra_body=dict(use_beam_search=True))
141
+ assert len(chat_completion.choices) == 2
142
+ assert chat_completion.choices[
143
+ 0].message.content != chat_completion.choices[1].message.content
144
+
145
+
146
@pytest.mark.asyncio
147
@pytest.mark.parametrize("model_name", [MODEL_NAME])
148
@pytest.mark.parametrize("image_url", TEST_IMAGE_URLS)
@@ -162,6 +198,41 @@ async def test_single_chat_session_image_base64encoded(
198
assert message.content is not None and len(message.content) >= 0
199
200
201
+@pytest.mark.asyncio
202
+@pytest.mark.parametrize("model_name", [MODEL_NAME])
203
+@pytest.mark.parametrize("image_url", TEST_IMAGE_URLS)
204
+async def test_single_chat_session_image_base64encoded_beamsearch(
205
+ client: openai.AsyncOpenAI, model_name: str, image_url: str,
206
+ base64_encoded_image: Dict[str, str]):
207
+
208
+ messages = [{
209
+ "role":
210
+ "user",
211
+ "content": [
212
+ {
213
+ "type": "image_url",
214
+ "image_url": {
215
+ "url":
216
+ f"data:image/jpeg;base64,{base64_encoded_image[image_url]}"
217
+ }
218
+ },
219
+ {
220
+ "type": "text",
221
+ "text": "What's in this image?"
222
+ },
223
+ ],
224
+ }]
225
+ chat_completion = await client.chat.completions.create(
226
+ model=model_name,
227
+ messages=messages,
228
+ n=2,
229
+ max_tokens=10,
230
+ extra_body=dict(use_beam_search=True))
231
+ assert len(chat_completion.choices) == 2
232
+ assert chat_completion.choices[
233
+ 0].message.content != chat_completion.choices[1].message.content
234
+
235
+
236
@pytest.mark.asyncio
237
@pytest.mark.parametrize("model_name", [MODEL_NAME])
238
@pytest.mark.parametrize("image_url", TEST_IMAGE_URLS)
vllm/beam_search.py
+8
-1
@@ -1,8 +1,11 @@
1
from dataclasses import dataclass
2
-from typing import Dict, List, Optional
2
+from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union
3
4
from vllm.sequence import Logprob
5
6
+if TYPE_CHECKING:
7
+ from vllm.multimodal import MultiModalDataDict
8
+
9
10
@dataclass
11
class BeamSearchSequence:
@@ -16,6 +19,10 @@ class BeamSearchSequence:
19
logprobs: List[Dict[int, Logprob]]
20
cum_logprob: float = 0.0
21
text: Optional[str] = None
22
+ finish_reason: Optional[str] = None
23
+ stop_reason: Union[int, str, None] = None
24
+ multi_modal_data: Optional["MultiModalDataDict"] = None
25
+ mm_processor_kwargs: Optional[Dict[str, Any]] = None
26
27
28
@dataclass
vllm/engine/protocol.py
+57
-31
@@ -6,6 +6,7 @@ from vllm.beam_search import BeamSearchSequence, create_sort_beams_key_function
6
from vllm.config import DecodingConfig, ModelConfig
7
from vllm.core.scheduler import SchedulerOutputs
8
from vllm.inputs.data import PromptType, TokensPrompt
9
+from vllm.inputs.preprocess import InputPreprocessor
10
from vllm.logger import init_logger
11
from vllm.lora.request import LoRARequest
12
from vllm.model_executor.layers.sampler import SamplerOutput
@@ -59,7 +60,8 @@ class EngineClient(ABC):
60
61
async def beam_search(
62
self,
62
- prompt: Union[str, List[int]],
63
+ prompt: Union[PromptType, List[int]],
64
+ model_config: ModelConfig,
65
request_id: str,
66
params: BeamSearchParams,
67
) -> AsyncGenerator[RequestOutput, None]:
@@ -69,32 +71,40 @@ class EngineClient(ABC):
71
ignore_eos = params.ignore_eos
72
temperature = params.temperature
73
length_penalty = params.length_penalty
74
+ include_stop_str_in_output = params.include_stop_str_in_output
75
73
- tokenizer = await self.get_tokenizer(lora_request=None)
74
- if isinstance(prompt, str):
75
- tokenized_prompt = tokenizer.encode(prompt)
76
- prompt_text = prompt
77
- else:
78
- tokenized_prompt = prompt
79
- prompt_text = None
80
- tokenized_length = len(tokenized_prompt)
76
+ tokenizer = await self.get_tokenizer()
77
+ input_preprocessor = InputPreprocessor(model_config, tokenizer)
78
+
79
+ (prompt_text, prompt_token_ids, multi_modal_data,
80
+ mm_processor_kwargs) = input_preprocessor._extract_prompt_components(
81
+ prompt,
82
+ request_id=request_id,
83
+ )
84
+ tokenized_length = len(prompt_token_ids)
85
86
sort_beams_key = create_sort_beams_key_function(
87
tokenizer.eos_token_id, length_penalty)
88
85
- beam_search_params = SamplingParams(logprobs=2 * beam_width,
86
- max_tokens=1,
87
- temperature=temperature)
89
+ beam_search_params = SamplingParams(
90
+ logprobs=2 * beam_width,
91
+ max_tokens=1,
92
+ temperature=temperature,
93
+ )
94
all_beams = [
89
- BeamSearchSequence(tokens=tokenized_prompt,
95
+ BeamSearchSequence(tokens=prompt_token_ids,
96
+ cum_logprob=0,
97
logprobs=[],
91
- cum_logprob=0)
98
+ multi_modal_data=multi_modal_data,
99
+ mm_processor_kwargs=mm_processor_kwargs)
100
]
101
completed = []
102
103
for _ in range(max_tokens):
104
prompts_batch = [
97
- TokensPrompt(prompt_token_ids=beam.tokens)
105
+ TokensPrompt(prompt_token_ids=beam.tokens,
106
+ multi_modal_data=beam.multi_modal_data,
107
+ mm_processor_kwargs=beam.mm_processor_kwargs)
108
for beam in all_beams
109
]
110
@@ -120,17 +130,31 @@ class EngineClient(ABC):
130
if result.outputs[0].logprobs is not None:
131
logprobs = result.outputs[0].logprobs[0]
132
for token_id, logprob_obj in logprobs.items():
123
- new_beam = BeamSearchSequence(
124
- tokens=current_beam.tokens + [token_id],
125
- logprobs=current_beam.logprobs + [logprobs],
126
- cum_logprob=current_beam.cum_logprob +
127
- logprob_obj.logprob)
128
-
133
if token_id == tokenizer.eos_token_id and \
134
not ignore_eos:
131
- completed.append(new_beam)
135
+ completed.append(
136
+ BeamSearchSequence(
137
+ tokens=current_beam.tokens +
138
+ [token_id] if include_stop_str_in_output
139
+ else current_beam.tokens,
140
+ logprobs=current_beam.logprobs +
141
+ [logprobs],
142
+ cum_logprob=current_beam.cum_logprob +
143
+ logprob_obj.logprob,
144
+ finish_reason="stop",
145
+ stop_reason=tokenizer.eos_token_id))
146
else:
133
- new_beams.append(new_beam)
147
+ new_beams.append(
148
+ BeamSearchSequence(
149
+ tokens=current_beam.tokens + [token_id],
150
+ logprobs=current_beam.logprobs +
151
+ [logprobs],
152
+ cum_logprob=current_beam.cum_logprob +
153
+ logprob_obj.logprob,
154
+ multi_modal_data=current_beam.
155
+ multi_modal_data,
156
+ mm_processor_kwargs=current_beam.
157
+ mm_processor_kwargs))
158
159
sorted_beams = sorted(new_beams, key=sort_beams_key, reverse=True)
160
all_beams = sorted_beams[:beam_width]
@@ -151,16 +175,18 @@ class EngineClient(ABC):
175
request_id=request_id,
176
prompt=prompt_text,
177
outputs=[
154
- CompletionOutput(
155
- text=beam.text,
156
- cumulative_logprob=beam.cum_logprob,
157
- token_ids=beam.tokens[tokenized_length:],
158
- index=i,
159
- logprobs=beam.logprobs,
160
- ) for (i, beam) in enumerate(best_beams)
178
+ CompletionOutput(text=beam.text,
179
+ cumulative_logprob=beam.cum_logprob,
180
+ token_ids=beam.tokens[tokenized_length:],
181
+ index=i,
182
+ logprobs=beam.logprobs,
183
+ finish_reason=beam.finish_reason if
184
+ beam.finish_reason is not None else "length",
185
+ stop_reason=beam.stop_reason)
186
+ for (i, beam) in enumerate(best_beams)
187
],
188
finished=True,
163
- prompt_token_ids=tokenized_prompt,
189
+ prompt_token_ids=prompt_token_ids,
190
prompt_logprobs=None)
191
192
yield beam_search_output
vllm/entrypoints/openai/protocol.py
+2
-2
@@ -308,7 +308,7 @@ class ChatCompletionRequest(OpenAIBaseModel):
308
ignore_eos=self.ignore_eos,
309
temperature=temperature,
310
length_penalty=self.length_penalty,
311
- )
311
+ include_stop_str_in_output=self.include_stop_str_in_output)
312
313
def to_sampling_params(self, default_max_tokens: int) -> SamplingParams:
314
max_tokens = self.max_tokens
@@ -606,7 +606,7 @@ class CompletionRequest(OpenAIBaseModel):
606
ignore_eos=self.ignore_eos,
607
temperature=temperature,
608
length_penalty=self.length_penalty,
609
- )
609
+ include_stop_str_in_output=self.include_stop_str_in_output)
610
611
def to_sampling_params(self, default_max_tokens: int) -> SamplingParams:
612
max_tokens = self.max_tokens
vllm/entrypoints/openai/serving_chat.py
+4
-3
@@ -236,9 +236,10 @@ class OpenAIServingChat(OpenAIServing):
236
237
if isinstance(sampling_params, BeamSearchParams):
238
result_generator = self.engine_client.beam_search(
239
- engine_inputs['prompt_token_ids'],
240
- request_id,
241
- sampling_params,
239
+ prompt=engine_inputs,
240
+ model_config=self.model_config,
241
+ request_id=request_id,
242
+ params=sampling_params,
243
)
244
else:
245
result_generator = self.engine_client.generate(
vllm/entrypoints/openai/serving_completion.py
+7
-3
@@ -150,9 +150,13 @@ class OpenAIServingCompletion(OpenAIServing):
150
151
if isinstance(sampling_params, BeamSearchParams):
152
generator = self.engine_client.beam_search(
153
- prompt_inputs["prompt_token_ids"],
154
- request_id_item,
155
- sampling_params,
153
+ prompt={
154
+ "prompt_token_ids":
155
+ prompt_inputs["prompt_token_ids"]
156
+ },
157
+ model_config=self.model_config,
158
+ request_id=request_id,
159
+ params=sampling_params,
160
)
161
else:
162
generator = self.engine_client.generate(
vllm/sampling_params.py
+1
@@ -500,3 +500,4 @@ class BeamSearchParams(
500
ignore_eos: bool = False
501
temperature: float = 0.0
502
length_penalty: float = 1.0
503
+ include_stop_str_in_output: bool = False