A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 07:00 UTC
[Model][Bugfix] Support TP for PixtralHF ViT (#10405)
Signed-off-by: mgoin <michael@neuralmagic.com>
Michael Goin committed
Nov 18, 2024 at 13:04 UTC
281cc4b3cd2f6c84c2cd8272ef83d97edd1c323a
1 file changed
+6
-2
vllm/model_executor/models/pixtral.py
+6
-2
@@ -17,6 +17,7 @@ from transformers.models.pixtral.modeling_pixtral import (
17
18
from vllm.attention import AttentionMetadata
19
from vllm.config import ModelConfig, VllmConfig
20
+from vllm.distributed import divide, get_tensor_model_parallel_world_size
21
from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, DummyData,
22
InputContext, token_inputs)
23
from vllm.model_executor.layers.activation import get_act_and_mul_fn
@@ -843,17 +844,20 @@ class PixtralHFAttention(nn.Module):
844
845
self.config = config
846
assert not config.hidden_size % config.num_attention_heads
846
- self.n_heads = config.num_attention_heads
847
+ self.total_num_heads = config.num_attention_heads
848
+ tp_size = get_tensor_model_parallel_world_size()
849
+ self.n_heads = divide(config.num_attention_heads, tp_size)
850
self.head_dim = config.hidden_size // config.num_attention_heads
851
852
self.qkv_proj = QKVParallelLinear(
853
hidden_size=config.hidden_size,
854
head_size=self.head_dim,
852
- total_num_heads=self.n_heads,
855
+ total_num_heads=self.total_num_heads,
856
bias=False,
857
quant_config=quant_config,
858
prefix=f"{prefix}.qkv_proj",
859
)
860
+ assert self.total_num_heads * self.head_dim == config.hidden_size
861
self.o_proj = RowParallelLinear(
862
input_size=config.hidden_size,
863
output_size=config.hidden_size,