A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 23:15 UTC
[Kernel] Fix CUTLASS 3.x custom broadcast load epilogue (#5516)
Tyler Michael Smith committed
Jun 14, 2024 at 12:30 UTC
703475f6c2771600acc27eba76f6a750f54aae50
2 files changed
+2
-4
csrc/quantization/cutlass_w8a8/broadcast_load_epilogue_c3x.hpp
+1
-1
@@ -153,7 +153,7 @@ struct Sm90RowOrScalarBroadcast {
153
154
CUTLASS_DEVICE void
155
begin(uint64_t* full_mbarrier_ptr, int load_iteration, bool issue_tma_load) {
156
- if (params.ptr_row == nullptr) {
156
+ if (!params.row_broadcast) {
157
return;
158
}
159
vllm/model_executor/layers/quantization/fp8.py
+1
-3
@@ -257,9 +257,7 @@ class Fp8LinearMethod(LinearMethodBase):
257
# If dynamic, layer.input_scale is None and x_scale computed from x.
258
# If static, layer.input_scale is scalar and x_scale is input_scale.
259
260
- # Temporarily disable CUTLASS kernels due to an illegal memory access
261
- #if bias is None and self.cutlass_fp8_supported:
262
- if False:
260
+ if bias is None and self.cutlass_fp8_supported:
261
qinput, x_scale = ops.scaled_fp8_quant(x, layer.input_scale)
262
263
# Fused GEMM_DQ