A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 23:30 UTC
[Misc] Add qwen2.5-vl BNB support (#12944)
Isotr0py committed
Feb 8, 2025 at 20:24 UTC
4c8dd12ef3474e43c614a229dabe85cc47432cf8
1 file changed
+29
-30
vllm/model_executor/models/qwen2_5_vl.py
+29
-30
@@ -40,7 +40,7 @@ from transformers.models.qwen2_5_vl.configuration_qwen2_5_vl import (
40
41
from vllm.attention import AttentionMetadata
42
from vllm.config import VllmConfig
43
-from vllm.distributed import parallel_state
43
+from vllm.distributed import parallel_state, tensor_model_parallel_all_gather
44
from vllm.distributed import utils as dist_utils
45
from vllm.logger import init_logger
46
from vllm.model_executor import SamplingMetadata
@@ -207,11 +207,12 @@ class Qwen2_5_VisionAttention(nn.Module):
207
) -> None:
208
super().__init__()
209
# Per attention head and per partition values.
210
- world_size = parallel_state.get_tensor_model_parallel_world_size()
210
+ self.tp_size = parallel_state.get_tensor_model_parallel_world_size()
211
+ self.tp_rank = parallel_state.get_tensor_model_parallel_rank()
212
self.hidden_size_per_attention_head = dist_utils.divide(
213
projection_size, num_heads)
214
self.num_attention_heads_per_partition = dist_utils.divide(
214
- num_heads, world_size)
215
+ num_heads, self.tp_size)
216
217
self.qkv = ColumnParallelLinear(input_size=embed_dim,
218
output_size=3 * projection_size,
@@ -231,6 +232,29 @@ class Qwen2_5_VisionAttention(nn.Module):
232
f"Qwen2.5-VL does not support {self.attn_backend} backend now."
233
)
234
235
+ def split_qkv(self, qkv: torch.Tensor) -> tuple[torch.Tensor, ...]:
236
+ # [s, b, 3 * head * head_dim]
237
+ seq_len, bs, _ = qkv.shape
238
+ if self.tp_size > 1:
239
+ qkv = tensor_model_parallel_all_gather(qkv)
240
+
241
+ # [s, b, 3 * head * head_dim] -> 3 * [s, b, head * head_dim]
242
+ q, k, v = qkv.chunk(3, dim=2)
243
+
244
+ # 3 * [s, b, head * head_dim]
245
+ if self.tp_size > 1:
246
+ splitter = partial(dist_utils.split_tensor_along_last_dim,
247
+ num_partitions=self.tp_size)
248
+ q = splitter(q)[self.tp_rank]
249
+ k = splitter(k)[self.tp_rank]
250
+ v = splitter(v)[self.tp_rank]
251
+
252
+ # 3 * [s, b, head * head_dim] -> 3 * [s, b, head, head_dim]
253
+ new_shape = (seq_len, bs, self.num_attention_heads_per_partition,
254
+ self.hidden_size_per_attention_head)
255
+ q, k, v = (x.view(*new_shape) for x in (q, k, v))
256
+ return q, k, v
257
+
258
def forward(
259
self,
260
x: torch.Tensor,
@@ -240,15 +264,8 @@ class Qwen2_5_VisionAttention(nn.Module):
264
# [s, b, c] --> [s, b, head * 3 * head_dim]
265
x, _ = self.qkv(x)
266
243
- # [s, b, head * 3 * head_dim] --> [s, b, head, 3 * head_dim]
244
- new_x_shape = x.size()[:-1] + (
245
- self.num_attention_heads_per_partition,
246
- 3 * self.hidden_size_per_attention_head,
247
- )
248
- x = x.view(*new_x_shape)
249
-
250
- # [s, b, head, 3 * head_dim] --> 3 [s, b, head, head_dim]
251
- q, k, v = dist_utils.split_tensor_along_last_dim(x, 3)
267
+ # [s, b, 3 * head * head_dim] -> 3 * [s, b, head, head_dim]
268
+ q, k, v = self.split_qkv(x)
269
batch_size = q.shape[1]
270
271
q, k, v = (rearrange(x, "s b ... -> b s ...").contiguous()
@@ -665,24 +682,6 @@ class Qwen2_5_VisionTransformer(nn.Module):
682
weight_loader(param, loaded_weight, shard_id)
683
break
684
else:
668
- if name.endswith("qkv.weight"):
669
- visual_num_heads = self.num_heads
670
- visual_embed_dim = self.hidden_size
671
- head_size = visual_embed_dim // visual_num_heads
672
- loaded_weight = loaded_weight.view(3, visual_num_heads,
673
- head_size,
674
- visual_embed_dim)
675
- loaded_weight = loaded_weight.transpose(0, 1)
676
- loaded_weight = loaded_weight.reshape(-1, visual_embed_dim)
677
- elif name.endswith("qkv.bias"):
678
- visual_num_heads = self.num_heads
679
- visual_embed_dim = self.hidden_size
680
- head_size = visual_embed_dim // visual_num_heads
681
- loaded_weight = loaded_weight.view(3, visual_num_heads,
682
- head_size)
683
- loaded_weight = loaded_weight.transpose(0, 1)
684
- loaded_weight = loaded_weight.reshape(-1)
685
-
685
param = params_dict[name]
686
weight_loader = getattr(param, "weight_loader",
687
default_weight_loader)