@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 703475f6c2
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 23:15 UTC

[Kernel] Fix CUTLASS 3.x custom broadcast load epilogue (#5516)

Tyler Michael Smith committed Jun 14, 2024 at 12:30 UTC 703475f6c2771600acc27eba76f6a750f54aae50
2 files changed +2 -4
csrc/quantization/cutlass_w8a8/broadcast_load_epilogue_c3x.hpp
+1 -1
@@ -153,7 +153,7 @@ struct Sm90RowOrScalarBroadcast {
153
154 CUTLASS_DEVICE void
155 begin(uint64_t* full_mbarrier_ptr, int load_iteration, bool issue_tma_load) {
156 - if (params.ptr_row == nullptr) {
156 + if (!params.row_broadcast) {
157 return;
158 }
159
vllm/model_executor/layers/quantization/fp8.py
+1 -3
@@ -257,9 +257,7 @@ class Fp8LinearMethod(LinearMethodBase):
257 # If dynamic, layer.input_scale is None and x_scale computed from x.
258 # If static, layer.input_scale is scalar and x_scale is input_scale.
259
260 - # Temporarily disable CUTLASS kernels due to an illegal memory access
261 - #if bias is None and self.cutlass_fp8_supported:
262 - if False:
260 + if bias is None and self.cutlass_fp8_supported:
261 qinput, x_scale = ops.scaled_fp8_quant(x, layer.input_scale)
262
263 # Fused GEMM_DQ