A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 09:45 UTC
[Bugfix] config.head_dim is now explicitly set to None (#18432)
Signed-off-by: Gregory Shtrasberg <Gregory.Shtrasberg@amd.com>
Gregory Shtrasberg committed
May 21, 2025 at 00:04 UTC
0c15c2e4868173642cec766c9819a210aef5e518
8 files changed
+27
-18
vllm/distributed/kv_transfer/kv_connector/utils.py
+3
-2
@@ -44,8 +44,9 @@ class model_aware_kv_ops_helper:
44
head_size = model_config.qk_nope_head_dim + \
45
model_config.qk_rope_head_dim
46
else:
47
- head_size = getattr(model_config, "head_dim",
48
- int(hidden_size // num_attention_heads))
47
+ head_size = getattr(model_config, "head_dim", None)
48
+ if head_size is None:
49
+ head_size = int(hidden_size // num_attention_heads)
50
51
return num_heads, head_size
52
vllm/model_executor/models/exaone.py
+3
-2
@@ -127,8 +127,9 @@ class ExaoneAttention(nn.Module):
127
assert tp_size % self.total_num_kv_heads == 0
128
self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size)
129
# MistralConfig has an optional head_dim introduced by Mistral-Nemo
130
- self.head_dim = getattr(config, "head_dim",
131
- self.hidden_size // self.total_num_heads)
130
+ self.head_dim = getattr(config, "head_dim", None)
131
+ if self.head_dim is None:
132
+ self.head_dim = self.hidden_size // self.total_num_heads
133
self.q_size = self.num_heads * self.head_dim
134
self.kv_size = self.num_kv_heads * self.head_dim
135
self.scaling = self.head_dim**-0.5
vllm/model_executor/models/granite.py
+3
-2
@@ -122,8 +122,9 @@ class GraniteAttention(nn.Module):
122
assert tp_size % self.total_num_kv_heads == 0
123
self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size)
124
# MistralConfig has an optional head_dim introduced by Mistral-Nemo
125
- self.head_dim = getattr(config, "head_dim",
126
- self.hidden_size // self.total_num_heads)
125
+ self.head_dim = getattr(config, "head_dim", None)
126
+ if self.head_dim is None:
127
+ self.head_dim = self.hidden_size // self.total_num_heads
128
self.q_size = self.num_heads * self.head_dim
129
self.kv_size = self.num_kv_heads * self.head_dim
130
self.scaling = config.attention_multiplier
vllm/model_executor/models/minimax_text_01.py
+6
-4
@@ -604,8 +604,9 @@ class MiniMaxText01DecoderLayer(nn.Module):
604
605
rope_theta = getattr(config, "rope_theta", 10000)
606
607
- head_dim = getattr(config, "head_dim",
608
- config.hidden_size // config.num_attention_heads)
607
+ head_dim = getattr(config, "head_dim", None)
608
+ if head_dim is None:
609
+ head_dim = config.hidden_size // config.num_attention_heads
610
if hasattr(config, "max_model_len") and isinstance(
611
config.max_model_len, int):
612
max_position_embeddings = min(config.max_position_embeddings,
@@ -861,8 +862,9 @@ class MiniMaxText01Model(nn.Module):
862
cache_shape=self.cache_shape)
863
864
rope_theta = getattr(config, "rope_theta", 10000)
864
- head_dim = getattr(config, "head_dim",
865
- config.hidden_size // config.num_attention_heads)
865
+ head_dim = getattr(config, "head_dim", None)
866
+ if head_dim is None:
867
+ head_dim = config.hidden_size // config.num_attention_heads
868
if hasattr(config, "max_model_len") and isinstance(
869
config.max_model_len, int):
870
max_position_embeddings = min(config.max_position_embeddings,
vllm/model_executor/models/mixtral.py
+3
-2
@@ -138,8 +138,9 @@ class MixtralAttention(nn.Module):
138
assert tp_size % self.total_num_kv_heads == 0
139
self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size)
140
# MixtralConfig has an optional head_dim argument
141
- self.head_dim = getattr(config, "head_dim",
142
- self.hidden_size // self.total_num_heads)
141
+ self.head_dim = getattr(config, "head_dim", None)
142
+ if self.head_dim is None:
143
+ self.head_dim = self.hidden_size // self.total_num_heads
144
self.q_size = self.num_heads * self.head_dim
145
self.kv_size = self.num_kv_heads * self.head_dim
146
self.scaling = self.head_dim**-0.5
vllm/model_executor/models/mixtral_quant.py
+3
-2
@@ -193,8 +193,9 @@ class MixtralAttention(nn.Module):
193
assert tp_size % self.total_num_kv_heads == 0
194
self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size)
195
# MixtralConfig has an optional head_dim argument
196
- self.head_dim = getattr(config, "head_dim",
197
- self.hidden_size // self.total_num_heads)
196
+ self.head_dim = getattr(config, "head_dim", None)
197
+ if self.head_dim is None:
198
+ self.head_dim = self.hidden_size // self.total_num_heads
199
self.q_size = self.num_heads * self.head_dim
200
self.kv_size = self.num_kv_heads * self.head_dim
201
self.scaling = self.head_dim**-0.5
vllm/model_executor/models/nemotron.py
+3
-2
@@ -158,8 +158,9 @@ class NemotronAttention(nn.Module):
158
assert tp_size % self.total_num_kv_heads == 0
159
self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size)
160
# MistralConfig has an optional head_dim introduced by Mistral-Nemo
161
- self.head_dim = getattr(config, "head_dim",
162
- self.hidden_size // self.total_num_heads)
161
+ self.head_dim = getattr(config, "head_dim", None)
162
+ if self.head_dim is None:
163
+ self.head_dim = self.hidden_size // self.total_num_heads
164
self.q_size = self.num_heads * self.head_dim
165
self.kv_size = self.num_kv_heads * self.head_dim
166
self.scaling = self.head_dim**-0.5
vllm/model_executor/models/solar.py
+3
-2
@@ -126,8 +126,9 @@ class SolarAttention(nn.Module):
126
assert tp_size % self.total_num_kv_heads == 0
127
self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size)
128
# MistralConfig has an optional head_dim introduced by Mistral-Nemo
129
- self.head_dim = getattr(config, "head_dim",
130
- self.hidden_size // self.total_num_heads)
129
+ self.head_dim = getattr(config, "head_dim", None)
130
+ if self.head_dim is None:
131
+ self.head_dim = self.hidden_size // self.total_num_heads
132
self.q_size = self.num_heads * self.head_dim
133
self.kv_size = self.num_kv_heads * self.head_dim
134
self.scaling = self.head_dim**-0.5