@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 6820724e51
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 23:15 UTC

[Bugfix] Fix w8a8 benchmarks for int8 case (#5643)

Tyler Michael Smith committed Jun 18, 2024 at 20:33 UTC 6820724e51079120251c8522afd385ca64abc948
1 file changed +2 -3
benchmarks/cutlass_benchmarks/w8a8_benchmarks.py
+2 -3
@@ -120,9 +120,8 @@ def bench_int8(dtype: torch.dtype, m: int, k: int, n: int, label: str,
120
121 # cutlass impl
122 timers.append(
123 - bench_fn(a, b, scale_a.to(device="cpu"), scale_b.to(device="cpu"),
124 - torch.bfloat16, label, sub_label, cutlass_impl,
125 - "cutlass_i8_i8_bf16_scaled_mm"))
123 + bench_fn(a, b, scale_a, scale_b, torch.bfloat16, label, sub_label,
124 + cutlass_impl, "cutlass_i8_i8_bf16_scaled_mm"))
125
126 return timers
127