@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 281cc4b3cd
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 07:00 UTC

[Model][Bugfix] Support TP for PixtralHF ViT (#10405)

Signed-off-by: mgoin <michael@neuralmagic.com>

Michael Goin committed Nov 18, 2024 at 13:04 UTC 281cc4b3cd2f6c84c2cd8272ef83d97edd1c323a
1 file changed +6 -2
vllm/model_executor/models/pixtral.py
+6 -2
@@ -17,6 +17,7 @@ from transformers.models.pixtral.modeling_pixtral import (
17
18 from vllm.attention import AttentionMetadata
19 from vllm.config import ModelConfig, VllmConfig
20 +from vllm.distributed import divide, get_tensor_model_parallel_world_size
21 from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, DummyData,
22 InputContext, token_inputs)
23 from vllm.model_executor.layers.activation import get_act_and_mul_fn
@@ -843,17 +844,20 @@ class PixtralHFAttention(nn.Module):
844
845 self.config = config
846 assert not config.hidden_size % config.num_attention_heads
846 - self.n_heads = config.num_attention_heads
847 + self.total_num_heads = config.num_attention_heads
848 + tp_size = get_tensor_model_parallel_world_size()
849 + self.n_heads = divide(config.num_attention_heads, tp_size)
850 self.head_dim = config.hidden_size // config.num_attention_heads
851
852 self.qkv_proj = QKVParallelLinear(
853 hidden_size=config.hidden_size,
854 head_size=self.head_dim,
852 - total_num_heads=self.n_heads,
855 + total_num_heads=self.total_num_heads,
856 bias=False,
857 quant_config=quant_config,
858 prefix=f"{prefix}.qkv_proj",
859 )
860 + assert self.total_num_heads * self.head_dim == config.hidden_size
861 self.o_proj = RowParallelLinear(
862 input_size=config.hidden_size,
863 output_size=config.hidden_size,