A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 00:45 UTC
[Bugfix] Better FP8 supported defaults
Lucas Wilkinson committed
Feb 5, 2025 at 22:22 UTC
76abd0c88143419826bfc13d2cd29669d0fdfa1b
2 files changed
+22
-12
vllm/model_executor/layers/quantization/utils/fp8_utils.py
+17
-11
@@ -15,7 +15,7 @@ from vllm.logger import init_logger
15
from vllm.model_executor.layers.quantization.utils.quant_utils import (
16
_normalize_quant_group_shape, scaled_dequantize)
17
from vllm.model_executor.layers.quantization.utils.w8a8_utils import (
18
- apply_fp8_linear)
18
+ CUTLASS_BLOCK_FP8_SUPPORTED, CUTLASS_FP8_SUPPORTED, apply_fp8_linear)
19
from vllm.platforms import current_platform
20
21
logger = init_logger(__name__)
@@ -38,7 +38,7 @@ def apply_w8a8_block_fp8_linear(
38
weight_scale: torch.Tensor,
39
input_scale: Optional[torch.Tensor] = None,
40
bias: Optional[torch.Tensor] = None,
41
- cutlass_block_fp8_supported: bool = True,
41
+ cutlass_block_fp8_supported: bool = CUTLASS_BLOCK_FP8_SUPPORTED,
42
) -> torch.Tensor:
43
assert input_scale is None
44
# View input as 2D matrix for fp8 methods
@@ -85,12 +85,14 @@ def apply_w8a8_block_fp8_linear(
85
# `apply_fp8_linear`
86
# NOTE(lucas): this is quite messy, we should think through this more formally
87
def apply_fp8_linear_generic(
88
- input: torch.Tensor,
89
- weight: torch.Tensor,
90
- weight_scale: torch.Tensor,
91
- input_group_shape: Tuple[int, int],
92
- weight_group_shape: Tuple[int, int],
93
- input_scale: Optional[torch.Tensor] = None, # static scale if one
88
+ input: torch.Tensor,
89
+ weight: torch.Tensor,
90
+ weight_scale: torch.Tensor,
91
+ input_group_shape: Tuple[int, int],
92
+ weight_group_shape: Tuple[int, int],
93
+ input_scale: Optional[torch.Tensor] = None, # static scale if one
94
+ cutlass_fp8_supported: bool = CUTLASS_FP8_SUPPORTED,
95
+ cutlass_block_fp8_supported: bool = CUTLASS_BLOCK_FP8_SUPPORTED,
96
) -> torch.Tensor:
97
# View input as 2D matrix for fp8 methods
98
input = input.view(-1, input.shape[-1])
@@ -105,14 +107,18 @@ def apply_fp8_linear_generic(
107
if is_dim_blocked(0, weight.shape, weight_group_shape[0])\
108
and is_dim_blocked(1, weight.shape, weight_group_shape[1]) and\
109
input_group_shape == (1, weight_group_shape[1]):
108
- return apply_w8a8_block_fp8_linear(input, weight,
109
- list(weight_group_shape),
110
- weight_scale)
110
+ return apply_w8a8_block_fp8_linear(
111
+ input,
112
+ weight,
113
+ list(weight_group_shape),
114
+ weight_scale,
115
+ cutlass_block_fp8_supported=cutlass_block_fp8_supported)
116
else:
117
# Despite having linear in the it doesn't conform to
118
# `torch.nn.functional.linear` which is defined as `input @ weight.T`
119
# so we explicitly transpose the weight matrix here
120
return apply_fp8_linear(input, weight.T, weight_scale.T,
121
+ cutlass_fp8_supported=cutlass_fp8_supported,
122
use_per_token_if_dynamic=\
123
(input_group_shape == (1, input.shape[1])))
124
vllm/model_executor/layers/quantization/utils/w8a8_utils.py
+5
-1
@@ -42,6 +42,10 @@ def cutlass_block_fp8_supported() -> bool:
42
return ops.cutlass_scaled_mm_supports_block_fp8(capability)
43
44
45
+CUTLASS_FP8_SUPPORTED = cutlass_fp8_supported()
46
+CUTLASS_BLOCK_FP8_SUPPORTED = cutlass_block_fp8_supported()
47
+
48
+
49
def per_tensor_dequantize(
50
tensor: torch.Tensor, inv_scale: Union[float,
51
torch.Tensor]) -> torch.Tensor:
@@ -109,7 +113,7 @@ def apply_fp8_linear(
113
input_scale: Optional[torch.Tensor] = None,
114
input_scale_ub: Optional[torch.Tensor] = None,
115
bias: Optional[torch.Tensor] = None,
112
- cutlass_fp8_supported: bool = True,
116
+ cutlass_fp8_supported: bool = CUTLASS_FP8_SUPPORTED,
117
use_per_token_if_dynamic: bool = False,
118
) -> torch.Tensor:
119
# ops.scaled_fp8_quant supports both dynamic and static quant.