A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 06:00 UTC
[Bugfix] Enforce contiguous input for dynamic_per_token FP8/INT8 quant (#19452)
Signed-off-by: mgoin <mgoin64@gmail.com>
Michael Goin committed
Jun 12, 2025 at 15:39 UTC
a3319f4f04fbea7defe883e516df727711e516cd
1 file changed
+3
-3
vllm/_custom_ops.py
+3
-3
@@ -1270,7 +1270,7 @@ def scaled_fp8_quant(
1270
device=input.device,
1271
dtype=torch.float32)
1272
torch.ops._C.dynamic_per_token_scaled_fp8_quant(
1273
- output, input, scale, scale_ub)
1273
+ output, input.contiguous(), scale, scale_ub)
1274
else:
1275
scale = torch.zeros(1, device=input.device, dtype=torch.float32)
1276
torch.ops._C.dynamic_scaled_fp8_quant(output, input, scale)
@@ -1379,8 +1379,8 @@ def scaled_int8_quant(
1379
dtype=torch.float32)
1380
input_azp = None if symmetric else torch.empty_like(input_scales,
1381
dtype=torch.int32)
1382
- torch.ops._C.dynamic_scaled_int8_quant(output, input, input_scales,
1383
- input_azp)
1382
+ torch.ops._C.dynamic_scaled_int8_quant(output, input.contiguous(),
1383
+ input_scales, input_azp)
1384
return output, input_scales, input_azp
1385
1386