A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 08:30 UTC
[Model][Gemma3] Cast image pixel values already on CPU (#18732)
Signed-off-by: Lukas Geiger <lukas.geiger94@gmail.com>
Lukas Geiger committed
May 27, 2025 at 06:42 UTC
b50602d5f04677e75158c0d2e0e8b51793a5d545
1 file changed
+6
-3
vllm/model_executor/models/gemma3_mm.py
+6
-3
@@ -263,6 +263,11 @@ class Gemma3MultiModalProcessor(BaseMultiModalProcessor[Gemma3ProcessingInfo]):
263
mm_data,
264
mm_kwargs,
265
)
266
+ if "pixel_values" in processed_outputs:
267
+ # Cast pixel values to model dtype already here,
268
+ # so we need to transfer less data to the GPU
269
+ processed_outputs["pixel_values"] = processed_outputs[
270
+ "pixel_values"].to(self.info.ctx.model_config.dtype)
271
272
# HF processor pops the `num_crops` kwarg, which is needed by vLLM
273
if (images := mm_data.get("images")) is not None:
@@ -543,9 +548,7 @@ class Gemma3ForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP,
548
vision_tower: SiglipVisionModel,
549
pixel_values: torch.Tensor,
550
) -> torch.Tensor:
546
- target_dtype = vision_tower.get_input_embeddings().weight.dtype
547
- image_features = vision_tower(pixel_values.to(dtype=target_dtype))
548
- return image_features
551
+ return vision_tower(pixel_values)
552
553
def _process_image_input(
554
self,