A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 12:15 UTC
[Bugfix] Fix `numel()` downcast in fused_layernorm_dynamic_per_token_quant.cu (#17316)
Richard Barnes committed
Apr 28, 2025 at 19:23 UTC
d6da8a8ff22e555ce516ca8ce4d005b1bd1d9fe2
1 file changed
+1
-1
csrc/quantization/fused_kernels/fused_layernorm_dynamic_per_token_quant.cu
+1
-1
@@ -96,7 +96,7 @@ void rms_norm_dynamic_per_token_quant_dispatch(
96
std::optional<at::Tensor> const& scale_ub,
97
std::optional<at::Tensor>& residual) {
98
int32_t hidden_size = input.size(-1);
99
- int32_t num_tokens = input.numel() / hidden_size;
99
+ auto num_tokens = input.numel() / hidden_size;
100
101
dim3 grid(num_tokens);
102
dim3 block(std::min(hidden_size, 1024));