@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 76abd0c881
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 00:45 UTC

[Bugfix] Better FP8 supported defaults

Lucas Wilkinson committed Feb 5, 2025 at 22:22 UTC 76abd0c88143419826bfc13d2cd29669d0fdfa1b
2 files changed +22 -12
vllm/model_executor/layers/quantization/utils/fp8_utils.py
+17 -11
@@ -15,7 +15,7 @@ from vllm.logger import init_logger
15 from vllm.model_executor.layers.quantization.utils.quant_utils import (
16 _normalize_quant_group_shape, scaled_dequantize)
17 from vllm.model_executor.layers.quantization.utils.w8a8_utils import (
18 - apply_fp8_linear)
18 + CUTLASS_BLOCK_FP8_SUPPORTED, CUTLASS_FP8_SUPPORTED, apply_fp8_linear)
19 from vllm.platforms import current_platform
20
21 logger = init_logger(__name__)
@@ -38,7 +38,7 @@ def apply_w8a8_block_fp8_linear(
38 weight_scale: torch.Tensor,
39 input_scale: Optional[torch.Tensor] = None,
40 bias: Optional[torch.Tensor] = None,
41 - cutlass_block_fp8_supported: bool = True,
41 + cutlass_block_fp8_supported: bool = CUTLASS_BLOCK_FP8_SUPPORTED,
42 ) -> torch.Tensor:
43 assert input_scale is None
44 # View input as 2D matrix for fp8 methods
@@ -85,12 +85,14 @@ def apply_w8a8_block_fp8_linear(
85 # `apply_fp8_linear`
86 # NOTE(lucas): this is quite messy, we should think through this more formally
87 def apply_fp8_linear_generic(
88 - input: torch.Tensor,
89 - weight: torch.Tensor,
90 - weight_scale: torch.Tensor,
91 - input_group_shape: Tuple[int, int],
92 - weight_group_shape: Tuple[int, int],
93 - input_scale: Optional[torch.Tensor] = None, # static scale if one
88 + input: torch.Tensor,
89 + weight: torch.Tensor,
90 + weight_scale: torch.Tensor,
91 + input_group_shape: Tuple[int, int],
92 + weight_group_shape: Tuple[int, int],
93 + input_scale: Optional[torch.Tensor] = None, # static scale if one
94 + cutlass_fp8_supported: bool = CUTLASS_FP8_SUPPORTED,
95 + cutlass_block_fp8_supported: bool = CUTLASS_BLOCK_FP8_SUPPORTED,
96 ) -> torch.Tensor:
97 # View input as 2D matrix for fp8 methods
98 input = input.view(-1, input.shape[-1])
@@ -105,14 +107,18 @@ def apply_fp8_linear_generic(
107 if is_dim_blocked(0, weight.shape, weight_group_shape[0])\
108 and is_dim_blocked(1, weight.shape, weight_group_shape[1]) and\
109 input_group_shape == (1, weight_group_shape[1]):
108 - return apply_w8a8_block_fp8_linear(input, weight,
109 - list(weight_group_shape),
110 - weight_scale)
110 + return apply_w8a8_block_fp8_linear(
111 + input,
112 + weight,
113 + list(weight_group_shape),
114 + weight_scale,
115 + cutlass_block_fp8_supported=cutlass_block_fp8_supported)
116 else:
117 # Despite having linear in the it doesn't conform to
118 # `torch.nn.functional.linear` which is defined as `input @ weight.T`
119 # so we explicitly transpose the weight matrix here
120 return apply_fp8_linear(input, weight.T, weight_scale.T,
121 + cutlass_fp8_supported=cutlass_fp8_supported,
122 use_per_token_if_dynamic=\
123 (input_group_shape == (1, input.shape[1])))
124
vllm/model_executor/layers/quantization/utils/w8a8_utils.py
+5 -1
@@ -42,6 +42,10 @@ def cutlass_block_fp8_supported() -> bool:
42 return ops.cutlass_scaled_mm_supports_block_fp8(capability)
43
44
45 +CUTLASS_FP8_SUPPORTED = cutlass_fp8_supported()
46 +CUTLASS_BLOCK_FP8_SUPPORTED = cutlass_block_fp8_supported()
47 +
48 +
49 def per_tensor_dequantize(
50 tensor: torch.Tensor, inv_scale: Union[float,
51 torch.Tensor]) -> torch.Tensor:
@@ -109,7 +113,7 @@ def apply_fp8_linear(
113 input_scale: Optional[torch.Tensor] = None,
114 input_scale_ub: Optional[torch.Tensor] = None,
115 bias: Optional[torch.Tensor] = None,
112 - cutlass_fp8_supported: bool = True,
116 + cutlass_fp8_supported: bool = CUTLASS_FP8_SUPPORTED,
117 use_per_token_if_dynamic: bool = False,
118 ) -> torch.Tensor:
119 # ops.scaled_fp8_quant supports both dynamic and static quant.