@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 15c6a079b1
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 20:45 UTC

[Model] Support Mistral-Nemo (#6548)

Michael Goin committed Jul 18, 2024 at 16:31 UTC 15c6a079b1ccea03587031c9d709fb9dce0a1d93
1 file changed +5 -1
vllm/model_executor/models/llama.py
+5 -1
@@ -89,6 +89,7 @@ class LlamaAttention(nn.Module):
89
90 def __init__(
91 self,
92 + config: LlamaConfig,
93 hidden_size: int,
94 num_heads: int,
95 num_kv_heads: int,
@@ -115,7 +116,9 @@ class LlamaAttention(nn.Module):
116 # the KV heads across multiple tensor parallel GPUs.
117 assert tp_size % self.total_num_kv_heads == 0
118 self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size)
118 - self.head_dim = hidden_size // self.total_num_heads
119 + # MistralConfig has an optional head_dim introduced by Mistral-Nemo
120 + self.head_dim = getattr(config, "head_dim",
121 + self.hidden_size // self.total_num_heads)
122 self.q_size = self.num_heads * self.head_dim
123 self.kv_size = self.num_kv_heads * self.head_dim
124 self.scaling = self.head_dim**-0.5
@@ -189,6 +192,7 @@ class LlamaDecoderLayer(nn.Module):
192 attention_bias = getattr(config, "attention_bias", False) or getattr(
193 config, "bias", False)
194 self.self_attn = LlamaAttention(
195 + config=config,
196 hidden_size=self.hidden_size,
197 num_heads=config.num_attention_heads,
198 num_kv_heads=getattr(config, "num_key_value_heads",