A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 23:15 UTC
Fix w8a8 benchmark and add Llama-3-8B (#5562)
Cody Yu committed
Jun 16, 2024 at 23:48 UTC
e2b85cf86a522e734a38b1d0314cfe9625003ef9
2 files changed
+19
-8
benchmarks/cutlass_benchmarks/w8a8_benchmarks.py
+13
-8
@@ -46,7 +46,7 @@ def make_rand_tensors(dtype: torch.dtype, m: int, n: int,
46
# impl
47
48
49
-def pytorch_i8_impl(a: torch.tensor, b: torch.tensor, scale_a: torch.tensor,
49
+def pytorch_mm_impl(a: torch.tensor, b: torch.tensor, scale_a: torch.tensor,
50
scale_b: torch.tensor,
51
out_dtype: torch.dtype) -> torch.tensor:
52
return torch.mm(a, b)
@@ -115,7 +115,7 @@ def bench_int8(dtype: torch.dtype, m: int, k: int, n: int, label: str,
115
timers.append(
116
bench_fn(a.to(dtype=torch.bfloat16, device="cuda"),
117
b.to(dtype=torch.bfloat16, device="cuda"), scale_a, scale_b,
118
- torch.bfloat16, label, sub_label, pytorch_i8_impl,
118
+ torch.bfloat16, label, sub_label, pytorch_mm_impl,
119
"pytorch_bf16_bf16_bf16_matmul-no-scales"))
120
121
# cutlass impl
@@ -136,6 +136,13 @@ def bench_fp8(dtype: torch.dtype, m: int, k: int, n: int, label: str,
136
137
timers = []
138
139
+ # pytorch impl w. bf16
140
+ timers.append(
141
+ bench_fn(a.to(dtype=torch.bfloat16, device="cuda"),
142
+ b.to(dtype=torch.bfloat16, device="cuda"), scale_a, scale_b,
143
+ torch.bfloat16, label, sub_label, pytorch_mm_impl,
144
+ "pytorch_bf16_bf16_bf16_matmul-no-scales"))
145
+
146
# pytorch impl: bf16 output, without fp8 fast accum
147
timers.append(
148
bench_fn(a, b, scale_a, scale_b, torch.bfloat16, label, sub_label,
@@ -160,14 +167,12 @@ def bench_fp8(dtype: torch.dtype, m: int, k: int, n: int, label: str,
167
168
# cutlass impl: bf16 output
169
timers.append(
163
- bench_fn(a, b, scale_a.to(device="cpu"), scale_b.to(device="cpu"),
164
- torch.bfloat16, label, sub_label, cutlass_impl,
165
- "cutlass_fp8_fp8_bf16_scaled_mm"))
170
+ bench_fn(a, b, scale_a, scale_b, torch.bfloat16, label, sub_label,
171
+ cutlass_impl, "cutlass_fp8_fp8_bf16_scaled_mm"))
172
# cutlass impl: fp16 output
173
timers.append(
168
- bench_fn(a, b, scale_a.to(device="cpu"), scale_b.to(device="cpu"),
169
- torch.float16, label, sub_label, cutlass_impl,
170
- "cutlass_fp8_fp8_fp16_scaled_mm"))
174
+ bench_fn(a, b, scale_a, scale_b, torch.float16, label, sub_label,
175
+ cutlass_impl, "cutlass_fp8_fp8_fp16_scaled_mm"))
176
return timers
177
178
benchmarks/cutlass_benchmarks/weight_shapes.py
+6
@@ -22,6 +22,12 @@ WEIGHT_SHAPES = {
22
([4096, 22016], 1),
23
([11008, 4096], 0),
24
],
25
+ "meta-llama/Llama-3-8b": [
26
+ ([4096, 6144], 1),
27
+ ([4096, 4096], 0),
28
+ ([4096, 28672], 1),
29
+ ([14336, 4096], 0),
30
+ ],
31
"meta-llama/Llama-2-13b-hf": [
32
([5120, 15360], 1),
33
([5120, 5120], 0),