@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 4c8dd12ef3
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 15, 23:30 UTC

[Misc] Add qwen2.5-vl BNB support (#12944)

Isotr0py committed Feb 8, 2025 at 20:24 UTC 4c8dd12ef3474e43c614a229dabe85cc47432cf8
1 file changed +29 -30
vllm/model_executor/models/qwen2_5_vl.py
+29 -30
@@ -40,7 +40,7 @@ from transformers.models.qwen2_5_vl.configuration_qwen2_5_vl import (
40
41 from vllm.attention import AttentionMetadata
42 from vllm.config import VllmConfig
43 -from vllm.distributed import parallel_state
43 +from vllm.distributed import parallel_state, tensor_model_parallel_all_gather
44 from vllm.distributed import utils as dist_utils
45 from vllm.logger import init_logger
46 from vllm.model_executor import SamplingMetadata
@@ -207,11 +207,12 @@ class Qwen2_5_VisionAttention(nn.Module):
207 ) -> None:
208 super().__init__()
209 # Per attention head and per partition values.
210 - world_size = parallel_state.get_tensor_model_parallel_world_size()
210 + self.tp_size = parallel_state.get_tensor_model_parallel_world_size()
211 + self.tp_rank = parallel_state.get_tensor_model_parallel_rank()
212 self.hidden_size_per_attention_head = dist_utils.divide(
213 projection_size, num_heads)
214 self.num_attention_heads_per_partition = dist_utils.divide(
214 - num_heads, world_size)
215 + num_heads, self.tp_size)
216
217 self.qkv = ColumnParallelLinear(input_size=embed_dim,
218 output_size=3 * projection_size,
@@ -231,6 +232,29 @@ class Qwen2_5_VisionAttention(nn.Module):
232 f"Qwen2.5-VL does not support {self.attn_backend} backend now."
233 )
234
235 + def split_qkv(self, qkv: torch.Tensor) -> tuple[torch.Tensor, ...]:
236 + # [s, b, 3 * head * head_dim]
237 + seq_len, bs, _ = qkv.shape
238 + if self.tp_size > 1:
239 + qkv = tensor_model_parallel_all_gather(qkv)
240 +
241 + # [s, b, 3 * head * head_dim] -> 3 * [s, b, head * head_dim]
242 + q, k, v = qkv.chunk(3, dim=2)
243 +
244 + # 3 * [s, b, head * head_dim]
245 + if self.tp_size > 1:
246 + splitter = partial(dist_utils.split_tensor_along_last_dim,
247 + num_partitions=self.tp_size)
248 + q = splitter(q)[self.tp_rank]
249 + k = splitter(k)[self.tp_rank]
250 + v = splitter(v)[self.tp_rank]
251 +
252 + # 3 * [s, b, head * head_dim] -> 3 * [s, b, head, head_dim]
253 + new_shape = (seq_len, bs, self.num_attention_heads_per_partition,
254 + self.hidden_size_per_attention_head)
255 + q, k, v = (x.view(*new_shape) for x in (q, k, v))
256 + return q, k, v
257 +
258 def forward(
259 self,
260 x: torch.Tensor,
@@ -240,15 +264,8 @@ class Qwen2_5_VisionAttention(nn.Module):
264 # [s, b, c] --> [s, b, head * 3 * head_dim]
265 x, _ = self.qkv(x)
266
243 - # [s, b, head * 3 * head_dim] --> [s, b, head, 3 * head_dim]
244 - new_x_shape = x.size()[:-1] + (
245 - self.num_attention_heads_per_partition,
246 - 3 * self.hidden_size_per_attention_head,
247 - )
248 - x = x.view(*new_x_shape)
249 -
250 - # [s, b, head, 3 * head_dim] --> 3 [s, b, head, head_dim]
251 - q, k, v = dist_utils.split_tensor_along_last_dim(x, 3)
267 + # [s, b, 3 * head * head_dim] -> 3 * [s, b, head, head_dim]
268 + q, k, v = self.split_qkv(x)
269 batch_size = q.shape[1]
270
271 q, k, v = (rearrange(x, "s b ... -> b s ...").contiguous()
@@ -665,24 +682,6 @@ class Qwen2_5_VisionTransformer(nn.Module):
682 weight_loader(param, loaded_weight, shard_id)
683 break
684 else:
668 - if name.endswith("qkv.weight"):
669 - visual_num_heads = self.num_heads
670 - visual_embed_dim = self.hidden_size
671 - head_size = visual_embed_dim // visual_num_heads
672 - loaded_weight = loaded_weight.view(3, visual_num_heads,
673 - head_size,
674 - visual_embed_dim)
675 - loaded_weight = loaded_weight.transpose(0, 1)
676 - loaded_weight = loaded_weight.reshape(-1, visual_embed_dim)
677 - elif name.endswith("qkv.bias"):
678 - visual_num_heads = self.num_heads
679 - visual_embed_dim = self.hidden_size
680 - head_size = visual_embed_dim // visual_num_heads
681 - loaded_weight = loaded_weight.view(3, visual_num_heads,
682 - head_size)
683 - loaded_weight = loaded_weight.transpose(0, 1)
684 - loaded_weight = loaded_weight.reshape(-1)
685 -
685 param = params_dict[name]
686 weight_loader = getattr(param, "weight_loader",
687 default_weight_loader)