A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 01:00 UTC
[Model] Add ModelConfig class for GraniteMoeHybrid to override default max_seq_len_to_capture (#20923)
Signed-off-by: Thomas Parnell <tpa@zurich.ibm.com>
Thomas Parnell committed
Jul 16, 2025 at 04:19 UTC
6cbc4d4bea2f0169846accbecead8f0fccebd761
1 file changed
+14
vllm/model_executor/models/config.py
+14
@@ -205,6 +205,19 @@ class SnowflakeGteNewModelConfig(VerifyAndUpdateConfig):
205
}
206
207
208
+class GraniteMoeHybridModelConfig(VerifyAndUpdateConfig):
209
+
210
+ @staticmethod
211
+ def verify_and_update_config(vllm_config: "VllmConfig") -> None:
212
+ config = vllm_config.model_config
213
+ config.max_seq_len_to_capture = config.max_model_len
214
+ logger.info(
215
+ "Setting max_seq_len_to_capture to %d "
216
+ "to ensure that CUDA graph capture "
217
+ "covers sequences of length up to max_model_len.",
218
+ config.max_model_len)
219
+
220
+
221
class HybridAttentionMambaModelConfig(VerifyAndUpdateConfig):
222
223
@classmethod
@@ -297,4 +310,5 @@ MODELS_CONFIG_MAP: dict[str, type[VerifyAndUpdateConfig]] = {
310
"Qwen3ForSequenceClassification": Qwen3ForSequenceClassificationConfig,
311
"XLMRobertaModel": JinaRobertaModelConfig,
312
"JinaVLForRanking": JinaVLForSequenceClassificationConfig,
313
+ "GraniteMoeHybridForCausalLM": GraniteMoeHybridModelConfig,
314
}