A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 1, 23:45 UTC
[Bugfix] fix EVS for qwen3-vl (#44205)
Signed-off-by: Rui "Garry" Gao <garrygaogg@gmail.com> Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>
Rui "Garry" Gao committed
Jun 4, 2026 at 19:06 UTC
4b87b3e845fcf2ce54f74fa3178a714d755c3a6a
1 file changed
+4
-4
vllm/model_executor/models/qwen3_vl.py
+4
-4
@@ -2269,6 +2269,8 @@ class Qwen3VLForConditionalGeneration(
2269
input_embeds for the LLM.
2270
"""
2271
2272
+ device = video_embeddings.device
2273
+
2274
# Generate video replacement token IDs using get_video_repl
2275
# This tokenizes each frame separator independently, then uses pre-tokenized
2276
# special tokens to ensure consistent tokenization regardless of
@@ -2283,10 +2285,8 @@ class Qwen3VLForConditionalGeneration(
2285
select_token_id=self.is_multimodal_pruning_enabled,
2286
)
2287
2286
- repl_token_ids = torch.tensor(video_repl.full)
2287
- embed_token_id = _cached_tensor(
2288
- self.config.video_token_id, repl_token_ids.device
2289
- )
2288
+ repl_token_ids = torch.tensor(video_repl.full, device=device)
2289
+ embed_token_id = _cached_tensor(self.config.video_token_id, device=device)
2290
is_video_embed = torch.isin(repl_token_ids, embed_token_id)
2291
2292
# Get text embeddings for indicator tokens (has only `visual_dim``).