A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 17:00 UTC
[Hotfix][Pixtral] Fix multiple images bugs (#8415)
Patrick von Platen committed
Sep 13, 2024 at 00:21 UTC
d31174a4e1ff7ac1efbdb5d89a24f0e477f95cc8
5 files changed
+196
-77
tests/conftest.py
+1
-1
@@ -658,8 +658,8 @@ class VllmRunner:
658
outputs.append((req_sample_output_ids, req_sample_output_strs))
659
return outputs
660
661
+ @staticmethod
662
def _final_steps_generate_w_logprobs(
662
- self,
663
req_outputs: List[RequestOutput],
664
) -> List[Tuple[List[int], str, Optional[SampleLogprobs]]]:
665
outputs: List[Tuple[List[int], str, Optional[SampleLogprobs]]] = []
tests/models/fixtures/pixtral_chat.pickle
Binary files /dev/null and b/tests/models/fixtures/pixtral_chat.pickle differ
tests/models/fixtures/pixtral_chat_engine.pickle
Binary files /dev/null and b/tests/models/fixtures/pixtral_chat_engine.pickle differ
tests/models/test_pixtral.py
+146
-42
@@ -2,13 +2,92 @@
2
3
Run `pytest tests/models/test_mistral.py`.
4
"""
5
+import pickle
6
+import uuid
7
+from typing import Any, Dict, List
8
+
9
import pytest
10
+from mistral_common.protocol.instruct.messages import ImageURLChunk
11
+from mistral_common.protocol.instruct.request import ChatCompletionRequest
12
+from mistral_common.tokens.tokenizers.mistral import MistralTokenizer
13
+from mistral_common.tokens.tokenizers.multimodal import image_from_chunk
14
+
15
+from vllm import EngineArgs, LLMEngine, SamplingParams, TokensPrompt
16
+from vllm.multimodal import MultiModalDataBuiltins
17
7
-from vllm.sampling_params import SamplingParams
18
+from .utils import check_logprobs_close
19
20
pytestmark = pytest.mark.vlm
21
22
MODELS = ["mistralai/Pixtral-12B-2409"]
23
+IMG_URLS = [
24
+ "https://picsum.photos/id/237/400/300",
25
+ "https://picsum.photos/id/231/200/300",
26
+ "https://picsum.photos/id/27/500/500",
27
+ "https://picsum.photos/id/17/150/600",
28
+]
29
+PROMPT = "Describe each image in one short sentence."
30
+
31
+
32
+def _create_msg_format(urls: List[str]) -> List[Dict[str, Any]]:
33
+ return [{
34
+ "role":
35
+ "user",
36
+ "content": [{
37
+ "type": "text",
38
+ "text": PROMPT,
39
+ }] + [{
40
+ "type": "image_url",
41
+ "image_url": {
42
+ "url": url
43
+ }
44
+ } for url in urls],
45
+ }]
46
+
47
+
48
+def _create_engine_inputs(urls: List[str]) -> TokensPrompt:
49
+ msg = _create_msg_format(urls)
50
+
51
+ tokenizer = MistralTokenizer.from_model("pixtral")
52
+
53
+ request = ChatCompletionRequest(messages=msg) # type: ignore[type-var]
54
+ tokenized = tokenizer.encode_chat_completion(request)
55
+
56
+ engine_inputs = TokensPrompt(prompt_token_ids=tokenized.tokens)
57
+
58
+ images = []
59
+ for chunk in request.messages[0].content:
60
+ if isinstance(chunk, ImageURLChunk):
61
+ images.append(image_from_chunk(chunk))
62
+
63
+ mm_data = MultiModalDataBuiltins(image=images)
64
+ engine_inputs["multi_modal_data"] = mm_data
65
+
66
+ return engine_inputs
67
+
68
+
69
+MSGS = [
70
+ _create_msg_format(IMG_URLS[:1]),
71
+ _create_msg_format(IMG_URLS[:2]),
72
+ _create_msg_format(IMG_URLS),
73
+]
74
+ENGINE_INPUTS = [
75
+ _create_engine_inputs(IMG_URLS[:1]),
76
+ _create_engine_inputs(IMG_URLS[:2]),
77
+ _create_engine_inputs(IMG_URLS),
78
+]
79
+
80
+SAMPLING_PARAMS = SamplingParams(max_tokens=512, temperature=0.0, logprobs=5)
81
+LIMIT_MM_PER_PROMPT = dict(image=4)
82
+
83
+MAX_MODEL_LEN = [8192, 65536]
84
+FIXTURE_LOGPROBS_CHAT = "tests/models/fixtures/pixtral_chat.pickle"
85
+FIXTURE_LOGPROBS_ENGINE = "tests/models/fixtures/pixtral_chat_engine.pickle"
86
+
87
+
88
+def load_logprobs(filename: str) -> Any:
89
+ with open(filename, 'rb') as f:
90
+ return pickle.load(f)
91
92
93
@pytest.mark.skip(
@@ -16,49 +95,74 @@ MODELS = ["mistralai/Pixtral-12B-2409"]
95
"Model is too big, test passed on A100 locally but will OOM on CI machine."
96
)
97
@pytest.mark.parametrize("model", MODELS)
98
+@pytest.mark.parametrize("max_model_len", MAX_MODEL_LEN)
99
@pytest.mark.parametrize("dtype", ["bfloat16"])
20
-@pytest.mark.parametrize("max_tokens", [64])
21
-@pytest.mark.parametrize("num_logprobs", [5])
22
-def test_models(
100
+def test_chat(
101
vllm_runner,
24
- example_prompts,
102
+ max_model_len: int,
103
model: str,
104
dtype: str,
27
- max_tokens: int,
28
- num_logprobs: int,
105
) -> None:
30
- image_urls = [
31
- "https://picsum.photos/id/237/200/300",
32
- "https://picsum.photos/seed/picsum/200/300"
33
- ]
34
- expected = [
35
- "The image depicts a black dog lying on a wooden surface, looking directly at the camera with a calm expression.", # noqa
36
- "The image depicts a serene landscape with a snow-covered mountain under a pastel-colored sky during sunset." # noqa
37
- ]
38
- prompt = "Describe the image in one short sentence."
39
-
40
- sampling_params = SamplingParams(max_tokens=512, temperature=0.0)
41
-
42
- with vllm_runner(model, dtype=dtype,
43
- tokenizer_mode="mistral") as vllm_model:
44
-
45
- for i, image_url in enumerate(image_urls):
46
- messages = [
47
- {
48
- "role":
49
- "user",
50
- "content": [{
51
- "type": "text",
52
- "text": prompt
53
- }, {
54
- "type": "image_url",
55
- "image_url": {
56
- "url": image_url
57
- }
58
- }]
59
- },
60
- ]
61
-
62
- outputs = vllm_model.model.chat(messages,
63
- sampling_params=sampling_params)
64
- assert outputs[0].outputs[0].text == expected[i]
106
+ EXPECTED_CHAT_LOGPROBS = load_logprobs(FIXTURE_LOGPROBS_CHAT)
107
+ with vllm_runner(
108
+ model,
109
+ dtype=dtype,
110
+ tokenizer_mode="mistral",
111
+ enable_chunked_prefill=False,
112
+ max_model_len=max_model_len,
113
+ limit_mm_per_prompt=LIMIT_MM_PER_PROMPT,
114
+ ) as vllm_model:
115
+ outputs = []
116
+ for msg in MSGS:
117
+ output = vllm_model.model.chat(msg,
118
+ sampling_params=SAMPLING_PARAMS)
119
+
120
+ outputs.extend(output)
121
+
122
+ logprobs = vllm_runner._final_steps_generate_w_logprobs(outputs)
123
+ check_logprobs_close(outputs_0_lst=logprobs,
124
+ outputs_1_lst=EXPECTED_CHAT_LOGPROBS,
125
+ name_0="output",
126
+ name_1="h100_ref")
127
+
128
+
129
+@pytest.mark.skip(
130
+ reason=
131
+ "Model is too big, test passed on A100 locally but will OOM on CI machine."
132
+)
133
+@pytest.mark.parametrize("model", MODELS)
134
+@pytest.mark.parametrize("dtype", ["bfloat16"])
135
+def test_model_engine(vllm_runner, model: str, dtype: str) -> None:
136
+ EXPECTED_ENGINE_LOGPROBS = load_logprobs(FIXTURE_LOGPROBS_ENGINE)
137
+ args = EngineArgs(
138
+ model=model,
139
+ tokenizer_mode="mistral",
140
+ enable_chunked_prefill=False,
141
+ limit_mm_per_prompt=LIMIT_MM_PER_PROMPT,
142
+ dtype=dtype,
143
+ )
144
+ engine = LLMEngine.from_engine_args(args)
145
+
146
+ engine.add_request(uuid.uuid4().hex, ENGINE_INPUTS[0], SAMPLING_PARAMS)
147
+ engine.add_request(uuid.uuid4().hex, ENGINE_INPUTS[1], SAMPLING_PARAMS)
148
+
149
+ outputs = []
150
+ count = 0
151
+ while True:
152
+ out = engine.step()
153
+ count += 1
154
+ for request_output in out:
155
+ if request_output.finished:
156
+ outputs.append(request_output)
157
+
158
+ if count == 2:
159
+ engine.add_request(uuid.uuid4().hex, ENGINE_INPUTS[2],
160
+ SAMPLING_PARAMS)
161
+ if not engine.has_unfinished_requests():
162
+ break
163
+
164
+ logprobs = vllm_runner._final_steps_generate_w_logprobs(outputs)
165
+ check_logprobs_close(outputs_0_lst=logprobs,
166
+ outputs_1_lst=EXPECTED_ENGINE_LOGPROBS,
167
+ name_0="output",
168
+ name_1="h100_ref")
vllm/model_executor/models/pixtral.py
+49
-34
@@ -1,4 +1,3 @@
1
-import math
1
from array import array
2
from dataclasses import dataclass, fields
3
from itertools import tee
@@ -15,11 +14,12 @@ from xformers.ops.fmha.attn_bias import BlockDiagonalMask
14
15
from vllm.attention import AttentionMetadata
16
from vllm.config import CacheConfig, MultiModalConfig
18
-from vllm.inputs import INPUT_REGISTRY, InputContext
17
+from vllm.inputs import INPUT_REGISTRY, InputContext, LLMInputs
18
from vllm.model_executor.layers.layernorm import RMSNorm
19
from vllm.model_executor.layers.quantization import QuantizationConfig
20
from vllm.model_executor.layers.sampler import SamplerOutput
21
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
22
+from vllm.model_executor.models.utils import merge_multimodal_embeddings
23
from vllm.model_executor.sampling_metadata import SamplingMetadata
24
from vllm.multimodal import MULTIMODAL_REGISTRY
25
from vllm.multimodal.base import MultiModalInputs
@@ -48,23 +48,29 @@ def dummy_data_for_pixtral(ctx: InputContext, seq_len: int,
48
tokenizer = cached_get_tokenizer(
49
ctx.model_config.tokenizer,
50
tokenizer_mode=ctx.model_config.tokenizer_mode)
51
- mm_encoder = tokenizer.instruct.mm_encoder
51
53
- mm_config = ctx.model_config.multimodal_config
54
- max_num_images_per_request = mm_config.limit_per_prompt.get("image", 1)
52
+ mm_encoder = tokenizer.mistral.instruct_tokenizer.mm_encoder
53
+ patch_size = mm_encoder.mm_config.image_patch_size
54
+ image_token_id = mm_encoder.special_ids.img
55
56
- # approximate image size
57
- size = int(math.sqrt(seq_len) * mm_encoder.mm_config.image_patch_size)
56
+ mm_config = ctx.model_config.multimodal_config
57
+ num_images = mm_config.limit_per_prompt.get("image", 1)
58
59
+ # dummy size
60
+ size = 256
61
image = Image.new("RGB", (size, size), color=0)
60
- img_chunk = ImageChunk(image=image)
62
62
- tokens = mm_encoder(img_chunk).tokens
63
- token_ids = max_num_images_per_request * array(VLLM_TOKEN_ID_ARRAY_TYPE,
64
- tokens)
63
+ image_feature_size = (size**2) // (patch_size**2)
64
+
65
+ num_image_tokens = image_feature_size * num_images
66
+
67
+ token_ids = array(VLLM_TOKEN_ID_ARRAY_TYPE,
68
+ [image_token_id]) * num_image_tokens
69
+ token_ids += array(VLLM_TOKEN_ID_ARRAY_TYPE,
70
+ [0]) * (seq_len - num_image_tokens)
71
72
seq_data = SequenceData(token_ids)
67
- mm_data = {"image": max_num_images_per_request * [image]}
73
+ mm_data = {"image": num_images * [image]}
74
return seq_data, mm_data
75
76
@@ -99,32 +105,31 @@ def input_mapper_for_pixtral(ctx: InputContext,
105
return MultiModalInputs({"images": images})
106
107
102
-def merge_multimodal_embeddings(input_ids: torch.Tensor,
103
- inputs_embeds: torch.Tensor,
104
- image_features: Optional[List[torch.Tensor]],
105
- image_id: int) -> torch.Tensor:
106
- text_locations = input_ids != image_id
107
- image_locations = input_ids == image_id
108
-
109
- seq_len = input_ids.shape[0]
108
+def input_processor_for_pixtral(ctx: InputContext, llm_inputs: LLMInputs):
109
+ multi_modal_data = llm_inputs.get("multi_modal_data")
110
+ if multi_modal_data is not None and "image" in multi_modal_data:
111
+ tokenizer = cached_get_tokenizer(
112
+ ctx.model_config.tokenizer,
113
+ tokenizer_mode=ctx.model_config.tokenizer_mode)
114
111
- N_txt = text_locations.sum().item()
112
- _, D_txt = inputs_embeds.shape
113
- N_img, D_img = image_features.shape
115
+ mm_encoder = tokenizer.mistral.instruct_tokenizer.mm_encoder
116
+ image_token_id = mm_encoder.special_ids.img
117
115
- assert (D_txt == D_img), (f"Text features dim {D_txt} should be equal "
116
- "to image features dim {D_img}")
117
- assert (seq_len == N_txt +
118
- N_img), (f"seq_len {seq_len} should be equal to N_txt + N_img "
119
- f"{(N_txt, N_img, image_locations.sum().item())}")
118
+ if image_token_id not in llm_inputs['prompt_token_ids']:
119
+ raise ValueError(
120
+ (f"You've passed {llm_inputs=} without {image_token_id=}"
121
+ " Make sure to process your input via mistral_common's"
122
+ " tokenizer or pass a chat completion request. For more"
123
+ " For more info, see: "
124
+ "https://github.com/vllm-project/vllm/issues/8411."))
125
121
- inputs_embeds[image_locations, :] = image_features
122
- return inputs_embeds
126
+ return llm_inputs
127
128
129
@MULTIMODAL_REGISTRY.register_image_input_mapper(input_mapper_for_pixtral)
130
@MULTIMODAL_REGISTRY.register_max_image_tokens(get_max_pixtral_image_tokens)
131
@INPUT_REGISTRY.register_dummy_data(dummy_data_for_pixtral)
132
+@INPUT_REGISTRY.register_input_processor(input_processor_for_pixtral)
133
class PixtralForConditionalGeneration(nn.Module, SupportsMultiModal):
134
135
def __init__(self,
@@ -201,11 +206,21 @@ class PixtralForConditionalGeneration(nn.Module, SupportsMultiModal):
206
return None
207
208
if isinstance(images, torch.Tensor):
204
- # always take last images
205
- images = [images[-1][i] for i in range(images.size(1))]
209
+ # if passed as batch take all images
210
+ N, B, C, W, H = images.shape
211
+ images = images.reshape(N * B, C, W, H)
212
+ images = [images[i] for i in range(images.size(0))]
213
elif isinstance(images, list):
207
- # always take last images
208
- images = [images[-1][i] for i in range(len(images[0]))]
214
+ # if passed as list flatten lists of tensors
215
+ flatten_images = []
216
+ for imgs_per_req in images:
217
+ imgs_per_req = [
218
+ imgs_per_req[i] for i in range(imgs_per_req.size(0))
219
+ ] if isinstance(imgs_per_req, torch.Tensor) else imgs_per_req
220
+
221
+ flatten_images.extend(imgs_per_req)
222
+
223
+ images = flatten_images
224
225
return images
226