@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / e38042d4af
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 23:15 UTC

[Kernel] Disable CUTLASS kernels for fp8 (#5505)

Tyler Michael Smith committed Jun 13, 2024 at 16:38 UTC e38042d4af1ddb390c3dd9340250de25bee37c62
1 file changed +3 -1
vllm/model_executor/layers/quantization/fp8.py
+3 -1
@@ -257,7 +257,9 @@ class Fp8LinearMethod(LinearMethodBase):
257 # If dynamic, layer.input_scale is None and x_scale computed from x.
258 # If static, layer.input_scale is scalar and x_scale is input_scale.
259
260 - if bias is None and self.cutlass_fp8_supported:
260 + # Temporarily disable CUTLASS kernels due to an illegal memory access
261 + #if bias is None and self.cutlass_fp8_supported:
262 + if False:
263 qinput, x_scale = ops.scaled_fp8_quant(x, layer.input_scale)
264
265 # Fused GEMM_DQ