A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 07:00 UTC
[VLM] Report multi_modal_placeholders in output (#10407)
Signed-off-by: Linkun Chen <lkchen+anyscale@github.com>
lkchen committed
Nov 18, 2024 at 00:06 UTC
c7dec926f6f1beaed759b8689373926e68867358
3 files changed
+115
-10
tests/models/decoder_only/vision_language/test_pixtral.py
+78
-1
@@ -8,13 +8,17 @@ from dataclasses import asdict
8
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple
9
10
import pytest
11
+from mistral_common.multimodal import download_image
12
from mistral_common.protocol.instruct.messages import ImageURLChunk
13
from mistral_common.protocol.instruct.request import ChatCompletionRequest
14
from mistral_common.tokens.tokenizers.mistral import MistralTokenizer
15
from mistral_common.tokens.tokenizers.multimodal import image_from_chunk
16
+from transformers import AutoProcessor
17
16
-from vllm import EngineArgs, LLMEngine, SamplingParams, TokensPrompt
18
+from vllm import (EngineArgs, LLMEngine, RequestOutput, SamplingParams,
19
+ TextPrompt, TokensPrompt)
20
from vllm.multimodal import MultiModalDataBuiltins
21
+from vllm.multimodal.inputs import PlaceholderRange
22
from vllm.sequence import Logprob, SampleLogprobs
23
24
from ....utils import VLLM_PATH, large_gpu_test
@@ -49,6 +53,20 @@ def _create_msg_format(urls: List[str]) -> List[Dict[str, Any]]:
53
}]
54
55
56
+def _create_msg_format_hf(urls: List[str]) -> List[Dict[str, Any]]:
57
+ return [{
58
+ "role":
59
+ "user",
60
+ "content": [{
61
+ "type": "text",
62
+ "content": PROMPT,
63
+ }, *({
64
+ "type": "image",
65
+ "image": download_image(url)
66
+ } for url in urls)],
67
+ }]
68
+
69
+
70
def _create_engine_inputs(urls: List[str]) -> TokensPrompt:
71
msg = _create_msg_format(urls)
72
@@ -70,6 +88,23 @@ def _create_engine_inputs(urls: List[str]) -> TokensPrompt:
88
return engine_inputs
89
90
91
+def _create_engine_inputs_hf(urls: List[str]) -> TextPrompt:
92
+ msg = _create_msg_format_hf(urls)
93
+
94
+ tokenizer = AutoProcessor.from_pretrained("mistral-community/pixtral-12b")
95
+ prompt = tokenizer.apply_chat_template(msg)
96
+
97
+ images = []
98
+ for chunk in msg[0]["content"]:
99
+ if chunk["type"] == "image":
100
+ images.append(chunk["image"])
101
+
102
+ mm_data = MultiModalDataBuiltins(image=images)
103
+ engine_inputs = TextPrompt(prompt=prompt, multi_modal_data=mm_data)
104
+
105
+ return engine_inputs
106
+
107
+
108
MSGS = [
109
_create_msg_format(IMG_URLS[:1]),
110
_create_msg_format(IMG_URLS[:2]),
@@ -191,3 +226,45 @@ def test_model_engine(vllm_runner, model: str, dtype: str) -> None:
226
outputs_1_lst=logprobs,
227
name_0="h100_ref",
228
name_1="output")
229
+
230
+
231
+@large_gpu_test(min_gb=24)
232
+@pytest.mark.parametrize(
233
+ "prompt,expected_ranges",
234
+ [(_create_engine_inputs_hf(IMG_URLS[:1]), [{
235
+ "offset": 10,
236
+ "length": 494
237
+ }]),
238
+ (_create_engine_inputs_hf(IMG_URLS[1:4]), [{
239
+ "offset": 10,
240
+ "length": 266
241
+ }, {
242
+ "offset": 276,
243
+ "length": 1056
244
+ }, {
245
+ "offset": 1332,
246
+ "length": 418
247
+ }])])
248
+def test_multi_modal_placeholders(
249
+ vllm_runner, prompt, expected_ranges: list[PlaceholderRange]) -> None:
250
+ with vllm_runner(
251
+ "mistral-community/pixtral-12b",
252
+ max_model_len=8192,
253
+ limit_mm_per_prompt=LIMIT_MM_PER_PROMPT,
254
+ ) as vllm_model:
255
+ outputs = vllm_model.model.generate(prompt)
256
+
257
+ assert len(outputs) == 1, f"{len(outputs)=}"
258
+ output: RequestOutput = outputs[0]
259
+ assert hasattr(output,
260
+ "multi_modal_placeholders"), f"{output.__dict__=}"
261
+ assert "image" in output.multi_modal_placeholders, \
262
+ f"{output.multi_modal_placeholders.keys()=}"
263
+ image_placeholder_ranges: list[
264
+ PlaceholderRange] = output.multi_modal_placeholders["image"]
265
+ assert len(image_placeholder_ranges) == len(
266
+ expected_ranges), f"{image_placeholder_ranges=}"
267
+ for real_range, expected_range in zip(image_placeholder_ranges,
268
+ expected_ranges):
269
+ assert real_range == expected_range, \
270
+ f"{real_range=} {expected_range=}"
vllm/model_executor/models/pixtral.py
+15
-1
@@ -30,6 +30,7 @@ from vllm.model_executor.model_loader.weight_utils import default_weight_loader
30
from vllm.model_executor.models.utils import merge_multimodal_embeddings
31
from vllm.model_executor.sampling_metadata import SamplingMetadata
32
from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalKwargs
33
+from vllm.multimodal.inputs import PlaceholderRange
34
from vllm.multimodal.utils import (cached_get_tokenizer,
35
consecutive_placeholder_ranges)
36
from vllm.sequence import IntermediateTensors, SequenceData
@@ -773,15 +774,28 @@ def input_processor_for_pixtral_hf(
774
replace_tokens[-1] = image_end_id
775
replace_tokens_list.append(replace_tokens)
776
777
+ reverse_offsets: List[int] = []
778
# Backward iteration for replacement without affecting known indices
779
for placeholder_idx, replace_tokens in zip(reversed(placeholder_indices),
780
reversed(replace_tokens_list)):
781
+ reverse_offsets.append(
782
+ len(new_token_ids) - placeholder_idx + len(replace_tokens))
783
new_token_ids[placeholder_idx:placeholder_idx + 1] = replace_tokens
784
785
+ placeholder_ranges: List[PlaceholderRange] = []
786
+ for reverse_offset, replace_tokens in zip(reversed(reverse_offsets),
787
+ replace_tokens_list):
788
+ placeholder_ranges.append(
789
+ PlaceholderRange(
790
+ offset=len(new_token_ids) - reverse_offset,
791
+ length=len(replace_tokens),
792
+ ))
793
+
794
# NOTE: Create a defensive copy of the original inputs
795
return token_inputs(prompt_token_ids=new_token_ids,
796
prompt=new_prompt,
784
- multi_modal_data=multi_modal_data)
797
+ multi_modal_data=multi_modal_data,
798
+ multi_modal_placeholders={"image": placeholder_ranges})
799
800
801
class PixtralHFMLP(nn.Module):
vllm/outputs.py
+22
-8
@@ -5,6 +5,7 @@ from typing import Sequence as GenericSequence
5
from typing import Union
6
7
from vllm.lora.request import LoRARequest
8
+from vllm.multimodal.inputs import MultiModalPlaceholderDict
9
from vllm.sampling_params import RequestOutputKind
10
from vllm.sequence import (PromptLogprobs, RequestMetrics, SampleLogprobs,
11
SequenceGroup, SequenceGroupBase, SequenceStatus)
@@ -103,10 +104,13 @@ class RequestOutput:
104
encoder_prompt: Optional[str] = None,
105
encoder_prompt_token_ids: Optional[List[int]] = None,
106
num_cached_tokens: Optional[int] = None,
107
+ *,
108
+ multi_modal_placeholders: Optional[MultiModalPlaceholderDict] = None,
109
) -> None:
110
self.request_id = request_id
111
self.prompt = prompt
112
self.prompt_token_ids = prompt_token_ids
113
+ self.multi_modal_placeholders = multi_modal_placeholders or {}
114
self.prompt_logprobs = prompt_logprobs
115
self.outputs = outputs
116
self.finished = finished
@@ -275,17 +279,26 @@ class RequestOutput:
279
finished_time = time.time() if finished else None
280
seq_group.set_finished_time(finished_time)
281
278
- init_args = (seq_group.request_id, prompt, prompt_token_ids,
279
- prompt_logprobs, outputs, finished, seq_group.metrics,
280
- seq_group.lora_request, encoder_prompt,
281
- encoder_prompt_token_ids, num_cached_tokens)
282
+ init_kwargs = {
283
+ "request_id": seq_group.request_id,
284
+ "prompt": prompt,
285
+ "prompt_token_ids": prompt_token_ids,
286
+ "prompt_logprobs": prompt_logprobs,
287
+ "outputs": outputs,
288
+ "finished": finished,
289
+ "metrics": seq_group.metrics,
290
+ "lora_request": seq_group.lora_request,
291
+ "encoder_prompt": encoder_prompt,
292
+ "encoder_prompt_token_ids": encoder_prompt_token_ids,
293
+ "num_cached_tokens": num_cached_tokens,
294
+ "multi_modal_placeholders": seq_group.multi_modal_placeholders
295
+ }
296
297
if use_cache:
298
request_output = seq_group.cached_request_output
285
- request_output.__init__(*init_args) # type: ignore
286
-
299
+ request_output.__init__(**init_kwargs) # type: ignore
300
else:
288
- request_output = cls(*init_args)
301
+ request_output = cls(**init_kwargs) # type: ignore
302
303
return request_output
304
@@ -300,7 +313,8 @@ class RequestOutput:
313
f"finished={self.finished}, "
314
f"metrics={self.metrics}, "
315
f"lora_request={self.lora_request}, "
303
- f"num_cached_tokens={self.num_cached_tokens})")
316
+ f"num_cached_tokens={self.num_cached_tokens}, "
317
+ f"multi_modal_placeholders={self.multi_modal_placeholders})")
318
319
320
class EmbeddingRequestOutput: