A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 11, 12:30 UTC
[Bugfix] Revert Qwen2-VL part of change in #28271 (#30542)
Signed-off-by: Zifei Tong <zifeitong@gmail.com>
zifeitong committed
Dec 14, 2025 at 05:20 UTC
48b8456ff9927f619ab9463106735b83d3035113
1 file changed
+12
-1
vllm/model_executor/models/qwen2_vl.py
+12
-1
@@ -50,7 +50,7 @@ from vllm.attention.layer import (
50
)
51
from vllm.config import VllmConfig
52
from vllm.config.multimodal import BaseDummyOptions
53
-from vllm.distributed import parallel_state
53
+from vllm.distributed import parallel_state, tensor_model_parallel_all_gather
54
from vllm.distributed import utils as dist_utils
55
from vllm.logger import init_logger
56
from vllm.model_executor.layers.activation import QuickGELU
@@ -360,10 +360,21 @@ class Qwen2VisionAttention(nn.Module):
360
def split_qkv(self, qkv: torch.Tensor) -> tuple[torch.Tensor, ...]:
361
# [s, b, 3 * head * head_dim]
362
seq_len, bs, _ = qkv.shape
363
+ if self.tp_size > 1:
364
+ qkv = tensor_model_parallel_all_gather(qkv)
365
366
# [s, b, 3 * head * head_dim] -> 3 * [s, b, head * head_dim]
367
q, k, v = qkv.chunk(3, dim=2)
368
369
+ # 3 * [s, b, head * head_dim]
370
+ if self.tp_size > 1:
371
+ splitter = partial(
372
+ dist_utils.split_tensor_along_last_dim, num_partitions=self.tp_size
373
+ )
374
+ q = splitter(q)[self.tp_rank]
375
+ k = splitter(k)[self.tp_rank]
376
+ v = splitter(v)[self.tp_rank]
377
+
378
# 3 * [s, b, head * head_dim] -> 3 * [s, b, head, head_dim]
379
new_shape = (
380
seq_len,