@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / fa0050db08
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 02:00 UTC

[Core] Default to using per_token quantization for fp8 when cutlass is supported. (#8651)

Signed-off-by: mgoin <michael@neuralmagic.com> Co-authored-by: Michael Goin <mgoin@redhat.com> Co-authored-by: mgoin <michael@neuralmagic.com>

Elfie Guo committed Jan 15, 2025 at 20:31 UTC fa0050db08660535368ec5ea41d313bdeb69909d
1 file changed +2 -1
vllm/model_executor/layers/quantization/fp8.py
+2 -1
@@ -355,7 +355,8 @@ class Fp8LinearMethod(LinearMethodBase):
355 input_scale=layer.input_scale,
356 bias=bias,
357 cutlass_fp8_supported=self.cutlass_fp8_supported,
358 - use_per_token_if_dynamic=False)
358 + # Default to using per_token quantization if cutlass is supported
359 + use_per_token_if_dynamic=self.cutlass_fp8_supported)
360
361
362 class Fp8MoEMethod(FusedMoEMethodBase):