A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 9, 13:33 UTC
Support compress-tensors with nvfp4 or fp8 weights and modelopt with nvfp4 weights on Turing (#33076)
Signed-off-by: IriKa Qiu <qiujie.jq@gmail.com>
IriKa committed
Jan 28, 2026 at 00:04 UTC
66e601ef795c27b43025ec7f4f864cac561c322e
3 files changed
+5
-4
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w4a16_nvfp4.py
+1
-1
@@ -29,7 +29,7 @@ class CompressedTensorsW4A16Fp4(CompressedTensorsScheme):
29
@classmethod
30
def get_min_capability(cls) -> int:
31
# don't restrict as emulations
32
- return 80
32
+ return 75
33
34
def create_weights(
35
self,
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a16_fp8.py
+2
-2
@@ -34,8 +34,8 @@ class CompressedTensorsW8A16Fp8(CompressedTensorsScheme):
34
35
@classmethod
36
def get_min_capability(cls) -> int:
37
- # ampere and up
38
- return 80
37
+ # turing and up
38
+ return 75
39
40
# W8A8-Fp8 kernels support only per-tensor and per-channel cases.
41
# So if we have a fused module (QKV, MLP) with per tensor scales,
vllm/model_executor/layers/quantization/modelopt.py
+2
-1
@@ -96,6 +96,7 @@ from vllm.model_executor.parameter import (
96
PerTensorScaleParameter,
97
)
98
from vllm.model_executor.utils import replace_parameter
99
+from vllm.platforms import current_platform
100
from vllm.utils.flashinfer import (
101
flashinfer_scaled_fp4_mm,
102
has_flashinfer,
@@ -1110,7 +1111,7 @@ class ModelOptNvFp4LinearMethod(LinearMethodBase):
1111
1112
self.backend = "none"
1113
if envs.VLLM_NVFP4_GEMM_BACKEND is None:
1113
- if has_flashinfer():
1114
+ if current_platform.has_device_capability(100) and has_flashinfer():
1115
self.backend = "flashinfer-cutlass"
1116
elif cutlass_fp4_supported():
1117
self.backend = "cutlass"