@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / e2b85cf86a
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 23:15 UTC

Fix w8a8 benchmark and add Llama-3-8B (#5562)

Cody Yu committed Jun 16, 2024 at 23:48 UTC e2b85cf86a522e734a38b1d0314cfe9625003ef9
2 files changed +19 -8
benchmarks/cutlass_benchmarks/w8a8_benchmarks.py
+13 -8
@@ -46,7 +46,7 @@ def make_rand_tensors(dtype: torch.dtype, m: int, n: int,
46 # impl
47
48
49 -def pytorch_i8_impl(a: torch.tensor, b: torch.tensor, scale_a: torch.tensor,
49 +def pytorch_mm_impl(a: torch.tensor, b: torch.tensor, scale_a: torch.tensor,
50 scale_b: torch.tensor,
51 out_dtype: torch.dtype) -> torch.tensor:
52 return torch.mm(a, b)
@@ -115,7 +115,7 @@ def bench_int8(dtype: torch.dtype, m: int, k: int, n: int, label: str,
115 timers.append(
116 bench_fn(a.to(dtype=torch.bfloat16, device="cuda"),
117 b.to(dtype=torch.bfloat16, device="cuda"), scale_a, scale_b,
118 - torch.bfloat16, label, sub_label, pytorch_i8_impl,
118 + torch.bfloat16, label, sub_label, pytorch_mm_impl,
119 "pytorch_bf16_bf16_bf16_matmul-no-scales"))
120
121 # cutlass impl
@@ -136,6 +136,13 @@ def bench_fp8(dtype: torch.dtype, m: int, k: int, n: int, label: str,
136
137 timers = []
138
139 + # pytorch impl w. bf16
140 + timers.append(
141 + bench_fn(a.to(dtype=torch.bfloat16, device="cuda"),
142 + b.to(dtype=torch.bfloat16, device="cuda"), scale_a, scale_b,
143 + torch.bfloat16, label, sub_label, pytorch_mm_impl,
144 + "pytorch_bf16_bf16_bf16_matmul-no-scales"))
145 +
146 # pytorch impl: bf16 output, without fp8 fast accum
147 timers.append(
148 bench_fn(a, b, scale_a, scale_b, torch.bfloat16, label, sub_label,
@@ -160,14 +167,12 @@ def bench_fp8(dtype: torch.dtype, m: int, k: int, n: int, label: str,
167
168 # cutlass impl: bf16 output
169 timers.append(
163 - bench_fn(a, b, scale_a.to(device="cpu"), scale_b.to(device="cpu"),
164 - torch.bfloat16, label, sub_label, cutlass_impl,
165 - "cutlass_fp8_fp8_bf16_scaled_mm"))
170 + bench_fn(a, b, scale_a, scale_b, torch.bfloat16, label, sub_label,
171 + cutlass_impl, "cutlass_fp8_fp8_bf16_scaled_mm"))
172 # cutlass impl: fp16 output
173 timers.append(
168 - bench_fn(a, b, scale_a.to(device="cpu"), scale_b.to(device="cpu"),
169 - torch.float16, label, sub_label, cutlass_impl,
170 - "cutlass_fp8_fp8_fp16_scaled_mm"))
174 + bench_fn(a, b, scale_a, scale_b, torch.float16, label, sub_label,
175 + cutlass_impl, "cutlass_fp8_fp8_fp16_scaled_mm"))
176 return timers
177
178
benchmarks/cutlass_benchmarks/weight_shapes.py
+6
@@ -22,6 +22,12 @@ WEIGHT_SHAPES = {
22 ([4096, 22016], 1),
23 ([11008, 4096], 0),
24 ],
25 + "meta-llama/Llama-3-8b": [
26 + ([4096, 6144], 1),
27 + ([4096, 4096], 0),
28 + ([4096, 28672], 1),
29 + ([14336, 4096], 0),
30 + ],
31 "meta-llama/Llama-2-13b-hf": [
32 ([5120, 15360], 1),
33 ([5120, 5120], 0),