A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
Enable modelopt gemma3 nvfp4/fp8, make workflow more robust (#22771)
Signed-off-by: Zhiyu Cheng <zhiyuc@nvidia.com> Signed-off-by: Michael Goin <mgoin64@gmail.com> Co-authored-by: Michael Goin <mgoin64@gmail.com>
Zhiyu committed
Sep 19, 2025 at 15:40 UTC
431535b522c62f1422848e22dfd83bec2d04111a
7 files changed
+82
-22
tests/kernels/moe/test_modular_kernel_combinations.py
+2
-1
@@ -11,7 +11,8 @@ import pytest
11
import torch
12
13
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
14
-from vllm.config import VllmConfig, current_platform, set_current_vllm_config
14
+from vllm.config import VllmConfig, set_current_vllm_config
15
+from vllm.platforms import current_platform
16
from vllm.utils import has_deep_ep, has_deep_gemm, has_pplx
17
from vllm.utils.flashinfer import has_flashinfer_cutlass_fused_moe
18
vllm/compilation/backends.py
+5
-2
@@ -31,8 +31,11 @@ logger = init_logger(__name__)
31
32
def make_compiler(compilation_config: CompilationConfig) -> CompilerInterface:
33
if compilation_config.use_inductor:
34
- if envs.VLLM_USE_STANDALONE_COMPILE and is_torch_equal_or_newer(
35
- "2.8.0.dev"):
34
+ # Use standalone compile only if requested, version is new enough,
35
+ # and the symbol actually exists in this PyTorch build.
36
+ if (envs.VLLM_USE_STANDALONE_COMPILE
37
+ and is_torch_equal_or_newer("2.8.0.dev")
38
+ and hasattr(torch._inductor, "standalone_compile")):
39
logger.debug("Using InductorStandaloneAdaptor")
40
return InductorStandaloneAdaptor()
41
else:
vllm/config/model.py
+3
@@ -964,6 +964,9 @@ class ModelConfig:
964
"modelopt",
965
"modelopt_fp4",
966
"petit_nvfp4",
967
+ # Ensure heavy backends are probed last to avoid unnecessary
968
+ # imports during override detection (e.g., MXFP4 imports Triton)
969
+ "mxfp4",
970
]
971
quantization_methods = [
972
q for q in supported_quantization if q not in overrides
vllm/model_executor/layers/fused_moe/gpt_oss_triton_kernels_moe.py
+2
-2
@@ -20,10 +20,10 @@ if has_triton_kernels():
20
from triton_kernels.matmul_ogs import (FnSpecs, FusedActivation,
21
matmul_ogs)
22
from triton_kernels.routing import routing
23
- except ModuleNotFoundError:
23
+ except (ModuleNotFoundError, AttributeError) as e:
24
logger.error(
25
"Failed to import Triton kernels. Please make sure your triton "
26
- "version is compatible.")
26
+ "version is compatible. Error: %s", e)
27
28
29
def triton_kernel_moe_forward(
vllm/model_executor/layers/quantization/modelopt.py
+37
-16
@@ -160,6 +160,7 @@ class ModelOptFp8Config(QuantizationConfig):
160
def is_layer_excluded(self, prefix: str) -> bool:
161
"""
162
Check if a layer should be excluded from quantization.
163
+ Handles both exact matching (for fused layers) and substring matching.
164
165
This method handles both regular models and multimodal models that use
166
the language_model prefix. For multimodal models, it checks if the
@@ -168,11 +169,18 @@ class ModelOptFp8Config(QuantizationConfig):
169
if self.exclude_modules is None:
170
return False
171
171
- # Check if any excluded module matches the prefix
172
+ # First check exact matching with fused layer support
173
+ if is_layer_skipped(prefix, self.exclude_modules,
174
+ self.packed_modules_mapping):
175
+ return True
176
+
177
+ # Then check substring matching for patterns not caught by exact match
178
for module in self.exclude_modules:
173
- if (module in prefix
174
- or (prefix.startswith("language_model.")
175
- and module in prefix.removeprefix("language_model."))):
179
+ # Skip exact matches already handled above
180
+ if (module != prefix and
181
+ (module in prefix or
182
+ (prefix.startswith("language_model.")
183
+ and module in prefix.removeprefix("language_model.")))):
184
return True
185
return False
186
@@ -180,9 +188,10 @@ class ModelOptFp8Config(QuantizationConfig):
188
prefix: str) -> Optional["QuantizeMethodBase"]:
189
from vllm.attention.layer import Attention # Avoid circular import
190
if isinstance(layer, LinearBase):
183
- if (is_layer_skipped(prefix, self.exclude_modules,
184
- self.packed_modules_mapping)
185
- or self.is_layer_excluded(prefix)):
191
+ if self.is_layer_excluded(prefix):
192
+ return UnquantizedLinearMethod()
193
+ # Check if this is a vision model layer that should not be quantized
194
+ if ("vision_tower" in prefix or "vision_model" in prefix):
195
return UnquantizedLinearMethod()
196
return ModelOptFp8LinearMethod(self)
197
elif isinstance(layer, Attention):
@@ -778,22 +787,34 @@ class ModelOptNvFp4Config(QuantizationConfig):
787
return cls(is_checkpoint_nvfp4_serialized, kv_cache_quant_algo,
788
exclude_modules, group_size)
789
781
- def is_layer_excluded(self, prefix: str,
782
- exclude_modules: list[str]) -> bool:
790
+ def is_layer_excluded(self, prefix: str) -> bool:
791
+ """
792
+ Check if a layer should be excluded from quantization.
793
+ Handles both exact matching (for fused layers) and pattern matching.
794
+ """
795
+ # First check exact matching with fused layer support
796
+ if is_layer_skipped(prefix, self.exclude_modules,
797
+ self.packed_modules_mapping):
798
+ return True
799
+
800
+ # Check regex pattern matching for patterns not caught by exact match
801
import regex as re
784
- for pattern in exclude_modules:
785
- regex_str = pattern.replace('.', r'\.').replace('*', r'.*')
786
- if re.fullmatch(regex_str, prefix):
787
- return True
802
+ for pattern in self.exclude_modules:
803
+ # Skip patterns that would be caught by exact matching
804
+ if '*' in pattern or '.' in pattern:
805
+ regex_str = pattern.replace('.', r'\.').replace('*', r'.*')
806
+ if re.fullmatch(regex_str, prefix):
807
+ return True
808
return False
809
810
def get_quant_method(self, layer: torch.nn.Module,
811
prefix: str) -> Optional["QuantizeMethodBase"]:
812
from vllm.attention.layer import Attention # Avoid circular import
813
if isinstance(layer, LinearBase):
794
- if (is_layer_skipped(prefix, self.exclude_modules,
795
- self.packed_modules_mapping)
796
- or self.is_layer_excluded(prefix, self.exclude_modules)):
814
+ if self.is_layer_excluded(prefix):
815
+ return UnquantizedLinearMethod()
816
+ # Check if this is a vision model layer that should not be quantized
817
+ if ("vision_tower" in prefix or "vision_model" in prefix):
818
return UnquantizedLinearMethod()
819
return ModelOptNvFp4LinearMethod(self)
820
elif isinstance(layer, Attention):
vllm/model_executor/models/gemma3.py
+16
@@ -446,6 +446,22 @@ class Gemma3Model(nn.Module):
446
weight_loader(param, loaded_weight)
447
loaded_params.add(scale_name)
448
continue
449
+
450
+ # Check if this is a scale parameter that needs remapping first
451
+ if name.endswith(
452
+ (".k_scale", ".v_scale", ".q_scale", ".prob_scale")):
453
+ # Try to remap the scale name first
454
+ remapped_name = maybe_remap_kv_scale_name(name, params_dict)
455
+ if remapped_name is not None and remapped_name in params_dict:
456
+ # Successfully remapped, use the remapped name
457
+ param = params_dict[remapped_name]
458
+ weight_loader = getattr(param, "weight_loader",
459
+ default_weight_loader)
460
+ weight_loader(param, loaded_weight)
461
+ loaded_params.add(remapped_name)
462
+ continue
463
+ # If remapping failed, continue with normal processing
464
+
465
for (param_name, shard_name, shard_id) in stacked_params_mapping:
466
if shard_name not in name:
467
continue
vllm/model_executor/models/siglip.py
+17
-1
@@ -20,7 +20,8 @@ from vllm.model_executor.layers.linear import (ColumnParallelLinear,
20
from vllm.model_executor.layers.quantization import QuantizationConfig
21
from vllm.model_executor.layers.vocab_parallel_embedding import (
22
VocabParallelEmbedding)
23
-from vllm.model_executor.model_loader.weight_utils import default_weight_loader
23
+from vllm.model_executor.model_loader.weight_utils import (
24
+ default_weight_loader, maybe_remap_kv_scale_name)
25
26
from .vision import VisionEncoderInfo, resolve_visual_encoder_outputs
27
@@ -506,6 +507,21 @@ class SiglipVisionModel(nn.Module):
507
if layer_idx >= layer_count:
508
continue
509
510
+ # Check if this is a scale parameter that needs remapping first
511
+ if name.endswith(
512
+ (".k_scale", ".v_scale", ".q_scale", ".prob_scale")):
513
+ # Try to remap the scale name first
514
+ remapped_name = maybe_remap_kv_scale_name(name, params_dict)
515
+ if remapped_name is not None and remapped_name in params_dict:
516
+ # Successfully remapped, use the remapped name
517
+ param = params_dict[remapped_name]
518
+ weight_loader = getattr(param, "weight_loader",
519
+ default_weight_loader)
520
+ weight_loader(param, loaded_weight)
521
+ loaded_params.add(remapped_name)
522
+ continue
523
+ # If remapping failed, continue with normal processing
524
+
525
for (param_name, weight_name, shard_id) in stacked_params_mapping:
526
if weight_name not in name:
527
continue