@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / c7dec926f6
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 07:00 UTC

[VLM] Report multi_modal_placeholders in output (#10407)

Signed-off-by: Linkun Chen <lkchen+anyscale@github.com>

lkchen committed Nov 18, 2024 at 00:06 UTC c7dec926f6f1beaed759b8689373926e68867358
3 files changed +115 -10
tests/models/decoder_only/vision_language/test_pixtral.py
+78 -1
@@ -8,13 +8,17 @@ from dataclasses import asdict
8 from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple
9
10 import pytest
11 +from mistral_common.multimodal import download_image
12 from mistral_common.protocol.instruct.messages import ImageURLChunk
13 from mistral_common.protocol.instruct.request import ChatCompletionRequest
14 from mistral_common.tokens.tokenizers.mistral import MistralTokenizer
15 from mistral_common.tokens.tokenizers.multimodal import image_from_chunk
16 +from transformers import AutoProcessor
17
16 -from vllm import EngineArgs, LLMEngine, SamplingParams, TokensPrompt
18 +from vllm import (EngineArgs, LLMEngine, RequestOutput, SamplingParams,
19 + TextPrompt, TokensPrompt)
20 from vllm.multimodal import MultiModalDataBuiltins
21 +from vllm.multimodal.inputs import PlaceholderRange
22 from vllm.sequence import Logprob, SampleLogprobs
23
24 from ....utils import VLLM_PATH, large_gpu_test
@@ -49,6 +53,20 @@ def _create_msg_format(urls: List[str]) -> List[Dict[str, Any]]:
53 }]
54
55
56 +def _create_msg_format_hf(urls: List[str]) -> List[Dict[str, Any]]:
57 + return [{
58 + "role":
59 + "user",
60 + "content": [{
61 + "type": "text",
62 + "content": PROMPT,
63 + }, *({
64 + "type": "image",
65 + "image": download_image(url)
66 + } for url in urls)],
67 + }]
68 +
69 +
70 def _create_engine_inputs(urls: List[str]) -> TokensPrompt:
71 msg = _create_msg_format(urls)
72
@@ -70,6 +88,23 @@ def _create_engine_inputs(urls: List[str]) -> TokensPrompt:
88 return engine_inputs
89
90
91 +def _create_engine_inputs_hf(urls: List[str]) -> TextPrompt:
92 + msg = _create_msg_format_hf(urls)
93 +
94 + tokenizer = AutoProcessor.from_pretrained("mistral-community/pixtral-12b")
95 + prompt = tokenizer.apply_chat_template(msg)
96 +
97 + images = []
98 + for chunk in msg[0]["content"]:
99 + if chunk["type"] == "image":
100 + images.append(chunk["image"])
101 +
102 + mm_data = MultiModalDataBuiltins(image=images)
103 + engine_inputs = TextPrompt(prompt=prompt, multi_modal_data=mm_data)
104 +
105 + return engine_inputs
106 +
107 +
108 MSGS = [
109 _create_msg_format(IMG_URLS[:1]),
110 _create_msg_format(IMG_URLS[:2]),
@@ -191,3 +226,45 @@ def test_model_engine(vllm_runner, model: str, dtype: str) -> None:
226 outputs_1_lst=logprobs,
227 name_0="h100_ref",
228 name_1="output")
229 +
230 +
231 +@large_gpu_test(min_gb=24)
232 +@pytest.mark.parametrize(
233 + "prompt,expected_ranges",
234 + [(_create_engine_inputs_hf(IMG_URLS[:1]), [{
235 + "offset": 10,
236 + "length": 494
237 + }]),
238 + (_create_engine_inputs_hf(IMG_URLS[1:4]), [{
239 + "offset": 10,
240 + "length": 266
241 + }, {
242 + "offset": 276,
243 + "length": 1056
244 + }, {
245 + "offset": 1332,
246 + "length": 418
247 + }])])
248 +def test_multi_modal_placeholders(
249 + vllm_runner, prompt, expected_ranges: list[PlaceholderRange]) -> None:
250 + with vllm_runner(
251 + "mistral-community/pixtral-12b",
252 + max_model_len=8192,
253 + limit_mm_per_prompt=LIMIT_MM_PER_PROMPT,
254 + ) as vllm_model:
255 + outputs = vllm_model.model.generate(prompt)
256 +
257 + assert len(outputs) == 1, f"{len(outputs)=}"
258 + output: RequestOutput = outputs[0]
259 + assert hasattr(output,
260 + "multi_modal_placeholders"), f"{output.__dict__=}"
261 + assert "image" in output.multi_modal_placeholders, \
262 + f"{output.multi_modal_placeholders.keys()=}"
263 + image_placeholder_ranges: list[
264 + PlaceholderRange] = output.multi_modal_placeholders["image"]
265 + assert len(image_placeholder_ranges) == len(
266 + expected_ranges), f"{image_placeholder_ranges=}"
267 + for real_range, expected_range in zip(image_placeholder_ranges,
268 + expected_ranges):
269 + assert real_range == expected_range, \
270 + f"{real_range=} {expected_range=}"
vllm/model_executor/models/pixtral.py
+15 -1
@@ -30,6 +30,7 @@ from vllm.model_executor.model_loader.weight_utils import default_weight_loader
30 from vllm.model_executor.models.utils import merge_multimodal_embeddings
31 from vllm.model_executor.sampling_metadata import SamplingMetadata
32 from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalKwargs
33 +from vllm.multimodal.inputs import PlaceholderRange
34 from vllm.multimodal.utils import (cached_get_tokenizer,
35 consecutive_placeholder_ranges)
36 from vllm.sequence import IntermediateTensors, SequenceData
@@ -773,15 +774,28 @@ def input_processor_for_pixtral_hf(
774 replace_tokens[-1] = image_end_id
775 replace_tokens_list.append(replace_tokens)
776
777 + reverse_offsets: List[int] = []
778 # Backward iteration for replacement without affecting known indices
779 for placeholder_idx, replace_tokens in zip(reversed(placeholder_indices),
780 reversed(replace_tokens_list)):
781 + reverse_offsets.append(
782 + len(new_token_ids) - placeholder_idx + len(replace_tokens))
783 new_token_ids[placeholder_idx:placeholder_idx + 1] = replace_tokens
784
785 + placeholder_ranges: List[PlaceholderRange] = []
786 + for reverse_offset, replace_tokens in zip(reversed(reverse_offsets),
787 + replace_tokens_list):
788 + placeholder_ranges.append(
789 + PlaceholderRange(
790 + offset=len(new_token_ids) - reverse_offset,
791 + length=len(replace_tokens),
792 + ))
793 +
794 # NOTE: Create a defensive copy of the original inputs
795 return token_inputs(prompt_token_ids=new_token_ids,
796 prompt=new_prompt,
784 - multi_modal_data=multi_modal_data)
797 + multi_modal_data=multi_modal_data,
798 + multi_modal_placeholders={"image": placeholder_ranges})
799
800
801 class PixtralHFMLP(nn.Module):
vllm/outputs.py
+22 -8
@@ -5,6 +5,7 @@ from typing import Sequence as GenericSequence
5 from typing import Union
6
7 from vllm.lora.request import LoRARequest
8 +from vllm.multimodal.inputs import MultiModalPlaceholderDict
9 from vllm.sampling_params import RequestOutputKind
10 from vllm.sequence import (PromptLogprobs, RequestMetrics, SampleLogprobs,
11 SequenceGroup, SequenceGroupBase, SequenceStatus)
@@ -103,10 +104,13 @@ class RequestOutput:
104 encoder_prompt: Optional[str] = None,
105 encoder_prompt_token_ids: Optional[List[int]] = None,
106 num_cached_tokens: Optional[int] = None,
107 + *,
108 + multi_modal_placeholders: Optional[MultiModalPlaceholderDict] = None,
109 ) -> None:
110 self.request_id = request_id
111 self.prompt = prompt
112 self.prompt_token_ids = prompt_token_ids
113 + self.multi_modal_placeholders = multi_modal_placeholders or {}
114 self.prompt_logprobs = prompt_logprobs
115 self.outputs = outputs
116 self.finished = finished
@@ -275,17 +279,26 @@ class RequestOutput:
279 finished_time = time.time() if finished else None
280 seq_group.set_finished_time(finished_time)
281
278 - init_args = (seq_group.request_id, prompt, prompt_token_ids,
279 - prompt_logprobs, outputs, finished, seq_group.metrics,
280 - seq_group.lora_request, encoder_prompt,
281 - encoder_prompt_token_ids, num_cached_tokens)
282 + init_kwargs = {
283 + "request_id": seq_group.request_id,
284 + "prompt": prompt,
285 + "prompt_token_ids": prompt_token_ids,
286 + "prompt_logprobs": prompt_logprobs,
287 + "outputs": outputs,
288 + "finished": finished,
289 + "metrics": seq_group.metrics,
290 + "lora_request": seq_group.lora_request,
291 + "encoder_prompt": encoder_prompt,
292 + "encoder_prompt_token_ids": encoder_prompt_token_ids,
293 + "num_cached_tokens": num_cached_tokens,
294 + "multi_modal_placeholders": seq_group.multi_modal_placeholders
295 + }
296
297 if use_cache:
298 request_output = seq_group.cached_request_output
285 - request_output.__init__(*init_args) # type: ignore
286 -
299 + request_output.__init__(**init_kwargs) # type: ignore
300 else:
288 - request_output = cls(*init_args)
301 + request_output = cls(**init_kwargs) # type: ignore
302
303 return request_output
304
@@ -300,7 +313,8 @@ class RequestOutput:
313 f"finished={self.finished}, "
314 f"metrics={self.metrics}, "
315 f"lora_request={self.lora_request}, "
303 - f"num_cached_tokens={self.num_cached_tokens})")
316 + f"num_cached_tokens={self.num_cached_tokens}, "
317 + f"multi_modal_placeholders={self.multi_modal_placeholders})")
318
319
320 class EmbeddingRequestOutput: