A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 23:15 UTC
[Kernel] Disable CUTLASS kernels for fp8 (#5505)
Tyler Michael Smith committed
Jun 13, 2024 at 16:38 UTC
e38042d4af1ddb390c3dd9340250de25bee37c62
1 file changed
+3
-1
vllm/model_executor/layers/quantization/fp8.py
+3
-1
@@ -257,7 +257,9 @@ class Fp8LinearMethod(LinearMethodBase):
257
# If dynamic, layer.input_scale is None and x_scale computed from x.
258
# If static, layer.input_scale is scalar and x_scale is input_scale.
259
260
- if bias is None and self.cutlass_fp8_supported:
260
+ # Temporarily disable CUTLASS kernels due to an illegal memory access
261
+ #if bias is None and self.cutlass_fp8_supported:
262
+ if False:
263
qinput, x_scale = ops.scaled_fp8_quant(x, layer.input_scale)
264
265
# Fused GEMM_DQ