@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 3eeb148f46
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 19:30 UTC

[Misc] Pass cutlass_fp8_supported correctly in fbgemm_fp8 (#6871)

Elsa Granger committed Jul 28, 2024 at 23:13 UTC 3eeb148f467e3619e8890b1a5ebe86a173f91bc9
1 file changed +11 -8
vllm/model_executor/layers/quantization/fbgemm_fp8.py
+11 -8
@@ -9,6 +9,7 @@ from vllm.model_executor.layers.linear import (LinearBase, LinearMethodBase,
9 UnquantizedLinearMethod)
10 from vllm.model_executor.layers.quantization.base_config import (
11 QuantizationConfig, QuantizeMethodBase)
12 +from vllm.model_executor.layers.quantization.fp8 import cutlass_fp8_supported
13 from vllm.model_executor.layers.quantization.utils.marlin_utils_fp8 import (
14 apply_fp8_marlin_linear, prepare_fp8_layer_for_marlin)
15 from vllm.model_executor.layers.quantization.utils.quant_utils import (
@@ -72,6 +73,7 @@ class FBGEMMFp8LinearMethod(LinearMethodBase):
73
74 def __init__(self, quant_config: FBGEMMFp8Config):
75 self.quant_config = quant_config
76 + self.cutlass_fp8_supported = cutlass_fp8_supported()
77
78 def create_weights(
79 self,
@@ -139,11 +141,12 @@ class FBGEMMFp8LinearMethod(LinearMethodBase):
141 size_k=layer.input_size_per_partition,
142 bias=bias)
143
142 - return apply_fp8_linear(input=x,
143 - weight=layer.weight,
144 - weight_scale=layer.weight_scale,
145 - input_scale=None,
146 - input_scale_ub=layer.input_scale_ub,
147 - bias=bias,
148 - cutlass_fp8_supported=True,
149 - use_per_token_if_dynamic=True)
144 + return apply_fp8_linear(
145 + input=x,
146 + weight=layer.weight,
147 + weight_scale=layer.weight_scale,
148 + input_scale=None,
149 + input_scale_ub=layer.input_scale_ub,
150 + bias=bias,
151 + cutlass_fp8_supported=self.cutlass_fp8_supported,
152 + use_per_token_if_dynamic=True)