A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 5, 01:00 UTC
[Models][Gemma4] Prevent GPU/CPU sync in `embed_input_ids` (#39234)
Signed-off-by: Lukas Geiger <lukas.geiger94@gmail.com>
Lukas Geiger committed
Apr 17, 2026 at 13:37 UTC
b1dc87a0989fd98a614e4e7e6b318a19e8c5c6f9
1 file changed
+3
-2
vllm/model_executor/models/gemma4_mm.py
+3
-2
@@ -1254,9 +1254,10 @@ class Gemma4ForConditionalGeneration(
1254
# computation (using token_type_ids == 0 as text_mask).
1255
# Replicate this: map image token positions to token 0.
1256
if is_multimodal is not None:
1257
- is_multimodal = is_multimodal.to(input_ids.device)
1257
ple_input_ids = torch.where(
1259
- is_multimodal, torch.zeros_like(input_ids), input_ids
1258
+ is_multimodal.to(input_ids.device, non_blocking=True),
1259
+ torch.zeros_like(input_ids),
1260
+ input_ids,
1261
)
1262
else:
1263
ple_input_ids = input_ids