A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 19:30 UTC
[Misc] Pass cutlass_fp8_supported correctly in fbgemm_fp8 (#6871)
Elsa Granger committed
Jul 28, 2024 at 23:13 UTC
3eeb148f467e3619e8890b1a5ebe86a173f91bc9
1 file changed
+11
-8
vllm/model_executor/layers/quantization/fbgemm_fp8.py
+11
-8
@@ -9,6 +9,7 @@ from vllm.model_executor.layers.linear import (LinearBase, LinearMethodBase,
9
UnquantizedLinearMethod)
10
from vllm.model_executor.layers.quantization.base_config import (
11
QuantizationConfig, QuantizeMethodBase)
12
+from vllm.model_executor.layers.quantization.fp8 import cutlass_fp8_supported
13
from vllm.model_executor.layers.quantization.utils.marlin_utils_fp8 import (
14
apply_fp8_marlin_linear, prepare_fp8_layer_for_marlin)
15
from vllm.model_executor.layers.quantization.utils.quant_utils import (
@@ -72,6 +73,7 @@ class FBGEMMFp8LinearMethod(LinearMethodBase):
73
74
def __init__(self, quant_config: FBGEMMFp8Config):
75
self.quant_config = quant_config
76
+ self.cutlass_fp8_supported = cutlass_fp8_supported()
77
78
def create_weights(
79
self,
@@ -139,11 +141,12 @@ class FBGEMMFp8LinearMethod(LinearMethodBase):
141
size_k=layer.input_size_per_partition,
142
bias=bias)
143
142
- return apply_fp8_linear(input=x,
143
- weight=layer.weight,
144
- weight_scale=layer.weight_scale,
145
- input_scale=None,
146
- input_scale_ub=layer.input_scale_ub,
147
- bias=bias,
148
- cutlass_fp8_supported=True,
149
- use_per_token_if_dynamic=True)
144
+ return apply_fp8_linear(
145
+ input=x,
146
+ weight=layer.weight,
147
+ weight_scale=layer.weight_scale,
148
+ input_scale=None,
149
+ input_scale_ub=layer.input_scale_ub,
150
+ bias=bias,
151
+ cutlass_fp8_supported=self.cutlass_fp8_supported,
152
+ use_per_token_if_dynamic=True)