A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 7, 16:30 UTC
Nemotron: use per-layer config in NemotronHMLPDecoderLayer for heterogeneous models (#35396)
Signed-off-by: dafrimi <dafrimi@nvidia.com>
danielafrimi committed
Feb 26, 2026 at 23:55 UTC
832a780f3aed332287203217d0d946b8b03299b4
1 file changed
+5
vllm/model_executor/models/nemotron_h.py
+5
@@ -298,6 +298,11 @@ class NemotronHMLPDecoderLayer(nn.Module):
298
299
hybrid_override_pattern = config.hybrid_override_pattern
300
mlp_index = hybrid_override_pattern[: layer_idx + 1].count("-") - 1
301
+ # Get per-layer config for heterogeneous models if exist
302
+ get_layer_config = getattr(config, "get_nemotron_h_config_for_layer", None)
303
+ layer_config = get_layer_config(layer_idx) if get_layer_config else config
304
+ config = layer_config
305
+
306
if isinstance(config.intermediate_size, list):
307
if len(config.intermediate_size) == 1:
308
intermediate_size = config.intermediate_size[0]