A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 21:00 UTC
[Bugfix] Correctly call `cudaProfilerStop` in benchmarks script (#14183)
Signed-off-by: Brayden Zhong <b8zhong@uwaterloo.ca>
Brayden Zhong committed
Mar 6, 2025 at 19:42 UTC
c34eeec58d3a94437c5311e256f8ba21d1912a39
6 files changed
+5
-6
benchmarks/kernels/benchmark_layernorm.py
+1
-1
@@ -40,7 +40,7 @@ def main(num_tokens: int,
40
41
end_time = time.perf_counter()
42
if profile:
43
- torch.cuda.cudart().cudaProfilerStart()
43
+ torch.cuda.cudart().cudaProfilerStop()
44
return (end_time - start_time) / num_iters
45
46
# Warmup.
benchmarks/kernels/benchmark_lora.py
-1
@@ -153,7 +153,6 @@ def ref_group_gemm(ref_out: torch.Tensor, input: torch.Tensor,
153
result = torch.nn.functional.linear(x, w)
154
result *= scaling
155
out_list.append(result)
156
- torch.cat(out_list, dim=0)
156
157
cat_result = torch.cat(out_list, dim=0)
158
benchmarks/kernels/benchmark_machete.py
+1
-2
@@ -45,7 +45,6 @@ def terse_type_name(dt):
45
torch.float16: "fp16",
46
torch.int8: "int8",
47
torch.float8_e4m3fn: "fp8",
48
- torch.bfloat16: "bf16",
48
torch.float: "float",
49
torch.int: "int",
50
}[dt]
@@ -259,7 +258,7 @@ def machete_create_bench_fn(bt: BenchmarkTensors,
258
259
return lambda: ops.machete_mm(
260
a=bt.a,
262
- b_q=bt.w_q,
261
+ b_q=w_q,
262
b_type=bt.wtype,
263
b_group_scales=bt.w_g_s,
264
b_group_zeros=w_g_zp,
benchmarks/kernels/benchmark_moe.py
+1
@@ -1,6 +1,7 @@
1
# SPDX-License-Identifier: Apache-2.0
2
3
import argparse
4
+import json
5
import time
6
from contextlib import nullcontext
7
from datetime import datetime
benchmarks/kernels/benchmark_paged_attention.py
+1
-1
@@ -176,7 +176,7 @@ def main(
176
177
end_time = time.perf_counter()
178
if profile:
179
- torch.cuda.cudart().cudaProfilerStart()
179
+ torch.cuda.cudart().cudaProfilerStop()
180
return (end_time - start_time) / num_iters
181
182
# Warmup.
benchmarks/kernels/benchmark_quant.py
+1
-1
@@ -40,7 +40,7 @@ def main(num_tokens: int,
40
41
end_time = time.perf_counter()
42
if profile:
43
- torch.cuda.cudart().cudaProfilerStart()
43
+ torch.cuda.cudart().cudaProfilerStop()
44
return (end_time - start_time) / num_iters
45
46
# Warmup.