A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 15:45 UTC
[Bugfix][VLM] Fix Fuyu batching inference with `max_num_seqs>1` (#8892)
Isotr0py committed
Sep 27, 2024 at 16:15 UTC
6d792d2f31b2cfb335d1a4a7c45fe4ce143c203a
2 files changed
+37
-20
tests/models/decoder_only/vision_language/test_fuyu.py
+2
-4
@@ -65,8 +65,8 @@ def run_test(
65
66
# max_model_len should be greater than image_feature_size
67
with vllm_runner(model,
68
- max_model_len=2560,
69
- max_num_seqs=1,
68
+ max_model_len=2048,
69
+ max_num_seqs=2,
70
dtype=dtype,
71
tensor_parallel_size=tensor_parallel_size,
72
distributed_executor_backend=distributed_executor_backend,
@@ -80,8 +80,6 @@ def run_test(
80
]
81
82
with hf_runner(model, dtype=dtype) as hf_model:
83
- hf_model.model.get_output_embeddings = lambda: \
84
- hf_model.model.language_model.get_output_embeddings()
83
eos_token_id = hf_model.processor.tokenizer.eos_token_id
84
hf_outputs_per_image = [
85
hf_model.generate_greedy_logprobs_limit(prompts,
vllm/model_executor/models/fuyu.py
+35
-16
@@ -42,7 +42,7 @@ from vllm.sequence import (VLLM_TOKEN_ID_ARRAY_TYPE, IntermediateTensors,
42
SequenceData)
43
44
from .interfaces import SupportsMultiModal
45
-from .utils import merge_multimodal_embeddings
45
+from .utils import flatten_bn, merge_multimodal_embeddings
46
47
# Cannot find the following 2 numbers from hf config.
48
_IMAGE_TOKEN_ID = 71011
@@ -165,7 +165,7 @@ def input_processor_for_fuyu(ctx: InputContext, llm_inputs: LLMInputs):
165
model_config.model)
166
167
model_image_input = _fuyu_image_preprocess(image_processor, image_data)
168
- image_patches = torch.stack([
168
+ image_patches = torch.cat([
169
image_patch[0]
170
for image_patch in model_image_input["image_patches"]
171
])
@@ -210,7 +210,7 @@ def input_mapper_for_fuyu(ctx: InputContext, data: object):
210
])
211
212
# image has been processed with prompt in input processor
213
- return MultiModalInputs({"image_patches": data})
213
+ return MultiModalInputs({"pixel_values": data})
214
215
216
@MULTIMODAL_REGISTRY.register_image_input_mapper(input_mapper_for_fuyu)
@@ -242,23 +242,42 @@ class FuyuForCausalLM(nn.Module, SupportsMultiModal):
242
cache_config=cache_config,
243
quant_config=quant_config)
244
245
+ def _validate_pixel_values(self, data: torch.Tensor) -> torch.Tensor:
246
+
247
+ h = w = self.config.patch_size
248
+ num_channels = self.config.num_channels
249
+ expected_dims = num_channels * h * w
250
+
251
+ def _validate_shape(d: torch.Tensor):
252
+ actual_dims = d.size(-1)
253
+
254
+ if actual_dims != expected_dims:
255
+ expected_expr = str(expected_dims)
256
+ raise ValueError(
257
+ "The expected shape of pixel values per image per batch "
258
+ f" per patch is {expected_expr}. "
259
+ f"You supplied {tuple(d.shape)}.")
260
+
261
+ for d in data:
262
+ _validate_shape(d)
263
+
264
+ return data.to(self.vision_embed_tokens.weight.dtype)
265
+
266
def _parse_and_validate_image_input(
267
self, **kwargs: object) -> Optional[FuyuImagePixelInputs]:
247
- image_patches = kwargs.pop("image_patches", None)
268
+ pixel_values = kwargs.pop("pixel_values", None)
269
249
- if isinstance(image_patches, torch.Tensor):
250
- # Remove the N dimension until multiple images are supported.
251
- image_patches = image_patches.squeeze(1)
270
+ if pixel_values is not None:
271
+ if not isinstance(pixel_values, (torch.Tensor, list)):
272
+ raise ValueError("Incorrect type of image patches. "
273
+ f"Got type: {type(pixel_values)}")
274
+
275
+ return FuyuImagePixelInputs(
276
+ type="pixel_values",
277
+ data=self._validate_pixel_values(
278
+ flatten_bn(pixel_values, concat=True)),
279
+ )
280
253
- expected_feature_size = self.image_feature_size
254
- if image_patches.size(-1) != expected_feature_size:
255
- raise ValueError(
256
- f"Expected image patches to have the last dimension of "
257
- f"{expected_feature_size}, got {image_patches.size(-1)}")
258
- image_patches = image_patches.to(
259
- self.vision_embed_tokens.weight.dtype)
260
- return FuyuImagePixelInputs(type="pixel_values",
261
- data=image_patches)
281
return None
282
283
def _process_image_input(