A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 23:15 UTC
[Bugfix] Fix w8a8 benchmarks for int8 case (#5643)
Tyler Michael Smith committed
Jun 18, 2024 at 20:33 UTC
6820724e51079120251c8522afd385ca64abc948
1 file changed
+2
-3
benchmarks/cutlass_benchmarks/w8a8_benchmarks.py
+2
-3
@@ -120,9 +120,8 @@ def bench_int8(dtype: torch.dtype, m: int, k: int, n: int, label: str,
120
121
# cutlass impl
122
timers.append(
123
- bench_fn(a, b, scale_a.to(device="cpu"), scale_b.to(device="cpu"),
124
- torch.bfloat16, label, sub_label, cutlass_impl,
125
- "cutlass_i8_i8_bf16_scaled_mm"))
123
+ bench_fn(a, b, scale_a, scale_b, torch.bfloat16, label, sub_label,
124
+ cutlass_impl, "cutlass_i8_i8_bf16_scaled_mm"))
125
126
return timers
127