A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 10, 19:00 UTC
[CI Failure] Fix NomicBert max_model_len validation (#31662)
Signed-off-by: wang.yuqi <yuqi.wang@daocloud.io>
wang.yuqi committed
Jan 5, 2026 at 11:06 UTC
8be6432bdaf6275664d857b1e5e9bf8ed1ce299e
2 files changed
+33
-26
vllm/config/vllm.py
-6
@@ -1264,12 +1264,6 @@ class VllmConfig:
1264
computed_compile_ranges_split_points
1265
)
1266
1267
- def recalculate_max_model_len(self, max_model_len: int):
1268
- # Can only be called in try_verify_and_update_config
1269
- model_config = self.model_config
1270
- max_model_len = model_config.get_and_verify_max_len(max_model_len)
1271
- self.model_config.max_model_len = max_model_len
1272
-
1267
def try_verify_and_update_config(self):
1268
if self.model_config is None:
1269
return
vllm/model_executor/models/config.py
+33
-20
@@ -113,8 +113,8 @@ class LlamaBidirectionalConfig(VerifyAndUpdateConfig):
113
114
class NomicBertModelConfig(VerifyAndUpdateConfig):
115
@staticmethod
116
- def verify_and_update_config(vllm_config: "VllmConfig") -> None:
117
- config = vllm_config.model_config.hf_config
116
+ def verify_and_update_model_config(model_config: "ModelConfig") -> None:
117
+ config = model_config.hf_config
118
119
assert config.__class__.__name__ == "NomicBertConfig"
120
assert config.activation_function in ["swiglu", "gelu"]
@@ -137,6 +137,10 @@ class NomicBertModelConfig(VerifyAndUpdateConfig):
137
config.intermediate_size = config.n_inner
138
config.hidden_size = config.n_embd
139
config.num_hidden_layers = config.n_layer
140
+ model_config.model_arch_config.hidden_size = config.hidden_size
141
+ model_config.model_arch_config.total_num_hidden_layers = (
142
+ config.num_hidden_layers
143
+ )
144
145
head_dim = config.hidden_size // config.num_attention_heads
146
max_trained_positions = getattr(config, "max_trained_positions", 2048)
@@ -153,42 +157,43 @@ class NomicBertModelConfig(VerifyAndUpdateConfig):
157
# The context extension uses vllm style rope_theta and rope_parameters.
158
# See #17785 #18755
159
if (
156
- not vllm_config.model_config.hf_overrides
157
- and vllm_config.model_config.original_max_model_len is None
160
+ not model_config.hf_overrides
161
+ and model_config.original_max_model_len is None
162
):
163
# Default
164
# Reset max_model_len to max_trained_positions.
165
# nomic-embed-text-v2-moe the length is set to 512
166
# by sentence_bert_config.json.
163
- max_model_len_before = vllm_config.model_config.max_model_len
164
- max_model_len = min(
165
- vllm_config.model_config.max_model_len, max_trained_positions
166
- )
167
+ max_model_len_before = model_config.max_model_len
168
+ max_model_len = min(model_config.max_model_len, max_trained_positions)
169
168
- vllm_config.recalculate_max_model_len(max_model_len)
169
- logger.warning(
170
- "Nomic context extension is disabled. "
171
- "Changing max_model_len from %s to %s. "
172
- "To enable context extension, see: "
173
- "https://github.com/vllm-project/vllm/tree/main/examples/offline_inference/context_extension.html",
174
- max_model_len_before,
175
- vllm_config.model_config.max_model_len,
170
+ model_config.max_model_len = model_config.get_and_verify_max_len(
171
+ max_model_len
172
)
173
+
174
+ if model_config.max_model_len != max_model_len_before:
175
+ logger.warning(
176
+ "Nomic context extension is disabled. "
177
+ "Changing max_model_len from %s to %s. "
178
+ "To enable context extension, see: "
179
+ "https://github.com/vllm-project/vllm/tree/main/examples/offline_inference/context_extension.html",
180
+ max_model_len_before,
181
+ model_config.max_model_len,
182
+ )
183
else:
184
# We need to re-verify max_model_len to avoid lengths
185
# greater than position_embedding.
180
- model_config = vllm_config.model_config
186
hf_text_config = model_config.hf_text_config
187
188
if isinstance(model_config.hf_overrides, dict):
189
# hf_overrides_kw
190
max_model_len = model_config.hf_overrides.get(
186
- "max_model_len", vllm_config.model_config.max_model_len
191
+ "max_model_len", model_config.max_model_len
192
)
193
else:
194
# hf_overrides_fn
195
# This might be overridden by sentence_bert_config.json.
191
- max_model_len = vllm_config.model_config.max_model_len
196
+ max_model_len = model_config.max_model_len
197
198
# reset hf_text_config for recalculate_max_model_len.
199
if hasattr(hf_text_config, "max_model_len"):
@@ -196,13 +201,21 @@ class NomicBertModelConfig(VerifyAndUpdateConfig):
201
hf_text_config.max_position_embeddings = max_trained_positions
202
hf_text_config.rope_parameters = config.rotary_kwargs["rope_parameters"]
203
204
+ # Update the cached derived_max_model_len to enforce the limit
205
+ model_config.model_arch_config.derived_max_model_len_and_key = (
206
+ float(max_trained_positions),
207
+ "max_position_embeddings",
208
+ )
209
+
210
# The priority of sentence_bert_config.json is higher
211
# than max_position_embeddings
212
encoder_config = deepcopy(model_config.encoder_config)
213
encoder_config.pop("max_seq_length", None)
214
model_config.encoder_config = encoder_config
215
205
- vllm_config.recalculate_max_model_len(max_model_len)
216
+ model_config.max_model_len = model_config.get_and_verify_max_len(
217
+ max_model_len
218
+ )
219
220
221
class Qwen2ForProcessRewardModelConfig(VerifyAndUpdateConfig):