A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 02:00 UTC
[Core] Default to using per_token quantization for fp8 when cutlass is supported. (#8651)
Signed-off-by: mgoin <michael@neuralmagic.com> Co-authored-by: Michael Goin <mgoin@redhat.com> Co-authored-by: mgoin <michael@neuralmagic.com>
Elfie Guo committed
Jan 15, 2025 at 20:31 UTC
fa0050db08660535368ec5ea41d313bdeb69909d
1 file changed
+2
-1
vllm/model_executor/layers/quantization/fp8.py
+2
-1
@@ -355,7 +355,8 @@ class Fp8LinearMethod(LinearMethodBase):
355
input_scale=layer.input_scale,
356
bias=bias,
357
cutlass_fp8_supported=self.cutlass_fp8_supported,
358
- use_per_token_if_dynamic=False)
358
+ # Default to using per_token quantization if cutlass is supported
359
+ use_per_token_if_dynamic=self.cutlass_fp8_supported)
360
361
362
class Fp8MoEMethod(FusedMoEMethodBase):