@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / a655eb3025
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 03:15 UTC

[Misc]Add BNB quantization for Qwen2VL (#11719)

Signed-off-by: Jee Jee Li <pandaleefree@gmail.com> Signed-off-by: Isotr0py <2037008807@qq.com> Co-authored-by: Isotr0py <2037008807@qq.com>

Jee Jee Li committed Jan 4, 2025 at 06:19 UTC a655eb30252fe266ce16fde2aa9f8f9554ccd46e
1 file changed +40 -29
vllm/model_executor/models/qwen2_vl.py
+40 -29
@@ -38,7 +38,7 @@ from transformers.models.qwen2_vl.image_processing_qwen2_vl import smart_resize
38
39 from vllm.attention import AttentionMetadata
40 from vllm.config import VllmConfig
41 -from vllm.distributed import parallel_state
41 +from vllm.distributed import parallel_state, tensor_model_parallel_all_gather
42 from vllm.distributed import utils as dist_utils
43 from vllm.logger import init_logger
44 from vllm.model_executor import SamplingMetadata
@@ -239,6 +239,8 @@ class Qwen2VisionAttention(nn.Module):
239 super().__init__()
240 # Per attention head and per partition values.
241 world_size = parallel_state.get_tensor_model_parallel_world_size()
242 + self.tp_size = world_size
243 + self.tp_rank = parallel_state.get_tensor_model_parallel_rank()
244 self.hidden_size_per_attention_head = dist_utils.divide(
245 projection_size, num_heads)
246 self.num_attention_heads_per_partition = dist_utils.divide(
@@ -261,24 +263,41 @@ class Qwen2VisionAttention(nn.Module):
263 raise RuntimeError(
264 f"Qwen2-VL does not support {self.attn_backend} backend now.")
265
266 + def split_qkv(self, qkv: torch.Tensor) -> tuple[torch.Tensor, ...]:
267 + # [s, b, 3 * head * head_dim]
268 + seq_len, bs, _ = qkv.shape
269 + if self.tp_size > 1:
270 + qkv = tensor_model_parallel_all_gather(qkv)
271 +
272 + # [s, b, 3 * head * head_dim] -> 3 * [s, b, head * head_dim]
273 + q, k, v = qkv.chunk(3, dim=2)
274 +
275 + # 3 * [s, b, head * head_dim]
276 + if self.tp_size > 1:
277 + splitter = partial(dist_utils.split_tensor_along_last_dim,
278 + num_partitions=self.tp_size)
279 + q = splitter(q)[self.tp_rank]
280 + k = splitter(k)[self.tp_rank]
281 + v = splitter(v)[self.tp_rank]
282 +
283 + # 3 * [s, b, head * head_dim] -> 3 * [s, b, head, head_dim]
284 + new_shape = (seq_len, bs, self.num_attention_heads_per_partition,
285 + self.hidden_size_per_attention_head)
286 + q, k, v = (x.view(*new_shape) for x in (q, k, v))
287 + return q, k, v
288 +
289 def forward(
290 self,
291 x: torch.Tensor,
292 cu_seqlens: torch.Tensor,
293 rotary_pos_emb: torch.Tensor,
294 ) -> torch.Tensor:
270 - # [s, b, c] --> [s, b, head * 3 * head_dim]
271 - x, _ = self.qkv(x)
295
273 - # [s, b, head * 3 * head_dim] --> [s, b, head, 3 * head_dim]
274 - new_x_shape = x.size()[:-1] + (
275 - self.num_attention_heads_per_partition,
276 - 3 * self.hidden_size_per_attention_head,
277 - )
278 - x = x.view(*new_x_shape)
296 + # [s, b, c] --> [s, b, 3 * head * head_dim]
297 + x, _ = self.qkv(x)
298
280 - # [s, b, head, 3 * head_dim] --> 3 [s, b, head, head_dim]
281 - q, k, v = dist_utils.split_tensor_along_last_dim(x, 3)
299 + # [s, b, 3 * head * head_dim] -> 3 * [s, b, head, head_dim]
300 + q, k, v = self.split_qkv(x)
301 batch_size = q.shape[1]
302
303 q, k, v = (rearrange(x, "s b ... -> b s ...").contiguous()
@@ -614,24 +633,6 @@ class Qwen2VisionTransformer(nn.Module):
633 weight_loader(param, loaded_weight, shard_id)
634 break
635 else:
617 - if name.endswith("qkv.weight"):
618 - visual_num_heads = self.num_heads
619 - visual_embed_dim = self.embed_dim
620 - head_size = visual_embed_dim // visual_num_heads
621 - loaded_weight = loaded_weight.view(3, visual_num_heads,
622 - head_size,
623 - visual_embed_dim)
624 - loaded_weight = loaded_weight.transpose(0, 1)
625 - loaded_weight = loaded_weight.reshape(-1, visual_embed_dim)
626 - elif name.endswith("qkv.bias"):
627 - visual_num_heads = self.num_heads
628 - visual_embed_dim = self.embed_dim
629 - head_size = visual_embed_dim // visual_num_heads
630 - loaded_weight = loaded_weight.view(3, visual_num_heads,
631 - head_size)
632 - loaded_weight = loaded_weight.transpose(0, 1)
633 - loaded_weight = loaded_weight.reshape(-1)
634 -
636 param = params_dict[name]
637 weight_loader = getattr(param, "weight_loader",
638 default_weight_loader)
@@ -935,6 +936,16 @@ class Qwen2VLForConditionalGeneration(nn.Module, SupportsMultiModal,
936 embedding_modules = {}
937 embedding_padding_modules = []
938
939 + # BitandBytes specific attributes
940 + bitsandbytes_stacked_params_mapping = {
941 + # shard_name, weight_name, index
942 + "q_proj": ("qkv_proj", 0),
943 + "k_proj": ("qkv_proj", 1),
944 + "v_proj": ("qkv_proj", 2),
945 + "gate_proj": ("gate_up_proj", 0),
946 + "up_proj": ("gate_up_proj", 1),
947 + }
948 +
949 # To ensure correct weight loading and mapping.
950 hf_to_vllm_mapper = WeightsMapper(orig_to_new_prefix={
951 "lm_head.": "language_model.lm_head.",