A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 04:45 UTC
[Quantization] Add compressed-tensors emulations support for NVFP4 (#19879)
Signed-off-by: Dipika Sikka <dipikasikka1@gmail.com> Signed-off-by: Dipika <dipikasikka1@gmail.com>
Dipika Sikka committed
Jun 25, 2025 at 14:28 UTC
02c97d9a92edca6f8675923afe0f1cf657305e38
3 files changed
+26
-1
vllm/envs.py
+7
@@ -133,6 +133,7 @@ if TYPE_CHECKING:
133
VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: int = 300
134
VLLM_KV_CACHE_LAYOUT: Optional[str] = None
135
VLLM_COMPUTE_NANS_IN_LOGITS: bool = False
136
+ VLLM_USE_NVFP4_CT_EMULATIONS: bool = False
137
138
139
def get_default_cache_root():
@@ -918,6 +919,12 @@ environment_variables: dict[str, Callable[[], Any]] = {
919
# or bad hardware but it may add compute overhead.
920
"VLLM_COMPUTE_NANS_IN_LOGITS":
921
lambda: bool(int(os.getenv("VLLM_COMPUTE_NANS_IN_LOGITS", "0"))),
922
+
923
+ # Controls whether or not emulations are used for NVFP4
924
+ # generations on machines < 100 for compressed-tensors
925
+ # models
926
+ "VLLM_USE_NVFP4_CT_EMULATIONS":
927
+ lambda: bool(int(os.getenv("VLLM_USE_NVFP4_CT_EMULATIONS", "0")))
928
}
929
930
# --8<-- [end:env-vars-definition]
vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors.py
+3
-1
@@ -13,6 +13,7 @@ from compressed_tensors.quantization import (QuantizationArgs,
13
QuantizationType)
14
from pydantic import BaseModel
15
16
+import vllm.envs as envs
17
from vllm.logger import init_logger
18
from vllm.model_executor.layers.fused_moe import FusedMoE
19
from vllm.model_executor.layers.linear import (LinearBase, LinearMethodBase,
@@ -374,7 +375,8 @@ class CompressedTensorsConfig(QuantizationConfig):
375
376
if is_activation_quantization_format(self.quant_format):
377
if self._is_fp4a4_nvfp4(weight_quant, input_quant):
377
- if CompressedTensorsW4A4Fp4.cutlass_fp4_supported():
378
+ if CompressedTensorsW4A4Fp4.cutlass_fp4_supported(
379
+ ) or envs.VLLM_USE_NVFP4_CT_EMULATIONS:
380
return CompressedTensorsW4A4Fp4()
381
else:
382
logger.warning_once(
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w4a4_nvfp4.py
+16
@@ -4,11 +4,14 @@ from typing import Callable, Optional
4
import torch
5
from torch.nn.parameter import Parameter
6
7
+import vllm.envs as envs
8
from vllm._custom_ops import (cutlass_scaled_fp4_mm,
9
cutlass_scaled_mm_supports_fp4, scaled_fp4_quant)
10
from vllm.logger import init_logger
11
from vllm.model_executor.layers.quantization.compressed_tensors.schemes import (
12
CompressedTensorsScheme)
13
+from vllm.model_executor.layers.quantization.utils.nvfp4_emulation_utils import ( # noqa: E501
14
+ run_nvfp4_emulations)
15
from vllm.model_executor.parameter import (GroupQuantScaleParameter,
16
ModelWeightParameter,
17
PerTensorScaleParameter)
@@ -26,6 +29,8 @@ class CompressedTensorsW4A4Fp4(CompressedTensorsScheme):
29
30
@classmethod
31
def get_min_capability(cls) -> int:
32
+ if envs.VLLM_USE_NVFP4_CT_EMULATIONS:
33
+ return 80
34
return 100
35
36
@classmethod
@@ -129,6 +134,17 @@ class CompressedTensorsW4A4Fp4(CompressedTensorsScheme):
134
x: torch.Tensor,
135
bias: Optional[torch.Tensor] = None) -> torch.Tensor:
136
137
+ if envs.VLLM_USE_NVFP4_CT_EMULATIONS:
138
+ out = run_nvfp4_emulations(
139
+ x=x,
140
+ input_global_scale=layer.input_global_scale,
141
+ weight=layer.weight,
142
+ weight_scale_swizzled=layer.weight_scale_swizzled,
143
+ weight_global_scale=layer.weight_global_scale)
144
+ if bias is not None:
145
+ out = out + bias
146
+ return out
147
+
148
output_dtype = x.dtype
149
output_shape = [x.shape[0], layer.weight.shape[0]]
150