@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 5fd24ec02e
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 02:00 UTC

[misc] Add LoRA kernel micro benchmarks (#11579)

Varun Sundar Rabindranath committed Jan 16, 2025 at 21:21 UTC 5fd24ec02e0365f96301ac73a31ef06976c256e8
2 files changed +1357
benchmarks/kernels/benchmark_lora.py new
+1147
@@ -0,0 +1,1147 @@
1 +import argparse
2 +import copy
3 +import json
4 +import pickle
5 +import time
6 +from dataclasses import dataclass
7 +from enum import Enum, auto
8 +from itertools import product
9 +from pathlib import Path
10 +from typing import Any, Callable, Dict, List, Optional, Tuple
11 +
12 +import torch
13 +import torch.utils.benchmark as TBenchmark
14 +from torch.utils.benchmark import Measurement as TMeasurement
15 +from utils import ArgPool, Bench, CudaGraphBenchParams
16 +from weight_shapes import WEIGHT_SHAPES
17 +
18 +from vllm.lora.ops.triton_ops.bgmv_expand import bgmv_expand
19 +from vllm.lora.ops.triton_ops.bgmv_expand_slice import bgmv_expand_slice
20 +from vllm.lora.ops.triton_ops.bgmv_shrink import bgmv_shrink
21 +from vllm.lora.ops.triton_ops.sgmv_expand import sgmv_expand
22 +from vllm.lora.ops.triton_ops.sgmv_shrink import sgmv_shrink
23 +from vllm.lora.ops.triton_ops.utils import _LORA_A_PTR_DICT, _LORA_B_PTR_DICT
24 +from vllm.utils import FlexibleArgumentParser
25 +
26 +DEFAULT_MODELS = list(WEIGHT_SHAPES.keys())
27 +DEFAULT_TP_SIZES = [1]
28 +DEFAULT_BATCH_SIZES = [
29 + 1, 16, 32, 64, 128, 192, 256, 320, 384, 448, 512, 640, 768, 896, 1024,
30 + 2048, 3072, 4096, 5120, 6144, 7168, 8192
31 +]
32 +DEFAULT_HIDDEN_SIZES = [1024, 2048, 4096, 8192, 16384]
33 +DEFAULT_LORA_RANKS = [16]
34 +DEFAULT_NUM_LORAS = [1, 2, 3, 4]
35 +DEFAULT_SORT_BY_LORA_IDS = [False, True]
36 +DEFAULT_SEQ_LENGTHS = [1]
37 +DEFAULT_EXPAND_FN_ADD_INPUTS = [True, False]
38 +
39 +
40 +# Utilities
41 +def dtype_to_str(dtype: torch.dtype):
42 + if dtype == torch.float16:
43 + return "f16"
44 + if dtype == torch.bfloat16:
45 + return "bf16"
46 + if dtype == torch.float32:
47 + return "f32"
48 + raise ValueError(f"Unsupported dtype {dtype}")
49 +
50 +
51 +def make_rand_lora_weight_tensor(k: int,
52 + n: int,
53 + num_loras: int,
54 + dtype: torch.dtype,
55 + device: str = "cuda") -> torch.Tensor:
56 +
57 + # LoRA weights column major
58 + return torch.rand((num_loras, n, k), dtype=dtype).to(device)
59 +
60 +
61 +def make_rand_tensors(
62 + a_shape: Tuple[int],
63 + b_shape: Tuple[int],
64 + c_shape: Tuple[int],
65 + a_dtype: torch.dtype,
66 + b_dtype: torch.dtype,
67 + c_dtype: torch.dtype,
68 + num_slices: int,
69 + device: str = "cuda",
70 +) -> Tuple[torch.Tensor, List[torch.Tensor], torch.Tensor]:
71 + """
72 + Make LoRA input/output matrices.
73 + """
74 + A = torch.rand(a_shape, dtype=a_dtype).to(device)
75 +
76 + # LoRA weights column major
77 + Bs = [
78 + torch.rand(b_shape, dtype=b_dtype).to(device)
79 + for _ in range(num_slices)
80 + ]
81 +
82 + C = torch.zeros(c_shape, dtype=c_dtype).to(device)
83 + return A, Bs, C
84 +
85 +
86 +def make_prompt_lora_mapping(num_prompts: int, num_active_loras: int,
87 + sort_by_lora_id: bool,
88 + device: str) -> torch.Tensor:
89 + """
90 + All prompts are mapped to a Lora ID in range [0, num_active_loras).
91 + where 0 refers to first lora, 1 refers to second lora and so on.
92 + """
93 + assert num_active_loras > 0
94 +
95 + if not sort_by_lora_id:
96 + return torch.randint(0,
97 + num_active_loras, (num_prompts, ),
98 + dtype=torch.long)
99 +
100 + # Divide LoRAs equally and in order.
101 + part_size = num_prompts // num_active_loras
102 + part_size = max(part_size, 1)
103 +
104 + lora_id = 0
105 + prompt_lora_mapping = []
106 + while len(prompt_lora_mapping) < num_prompts:
107 + prompt_lora_mapping.extend([lora_id] * part_size)
108 + lora_id = lora_id + 1 if lora_id + 1 < num_active_loras else lora_id
109 + return torch.tensor(prompt_lora_mapping[:num_prompts],
110 + dtype=torch.long,
111 + device=device)
112 +
113 +
114 +def make_token_lora_mapping(num_tokens: int, num_prompts: int,
115 + prompt_lora_mapping: torch.Tensor,
116 + seq_len_tensor: torch.Tensor, device: str):
117 + """
118 + Make token_lora_mapping from prompt_lora_mapping and seq_lens_tensor
119 + """
120 + assert prompt_lora_mapping.shape[0] == num_prompts
121 +
122 + # token to lora index mapping
123 + token_lora_mapping = [0] * num_tokens
124 + current_offset = 0
125 + for b_id in range(num_prompts):
126 + lora_index = prompt_lora_mapping[b_id].item()
127 + s = current_offset
128 + e = s + seq_len_tensor[b_id].item()
129 + token_lora_mapping[s:e] = [lora_index] * (e - s)
130 + current_offset += seq_len_tensor[b_id].item()
131 +
132 + return torch.tensor(token_lora_mapping, dtype=torch.long, device=device)
133 +
134 +
135 +def ref_group_gemm(ref_out: torch.Tensor, input: torch.Tensor,
136 + lora_weights: List[torch.Tensor],
137 + seq_lens_cpu: torch.Tensor,
138 + prompt_lora_mapping_cpu: torch.Tensor, scaling: float,
139 + add_inputs: Optional[bool]):
140 + """
141 + Torch group gemm reference implementation to test correctness of
142 + benchmarking operations.
143 + """
144 + batches = seq_lens_cpu.size(0)
145 + out_list = []
146 + current_offset = 0
147 + for lora_index, b_length in zip(range(batches), seq_lens_cpu):
148 + x = input[current_offset:b_length + current_offset, :]
149 + current_offset += b_length
150 + w = lora_weights[prompt_lora_mapping_cpu[lora_index]]
151 + result = torch.nn.functional.linear(x, w)
152 + result *= scaling
153 + out_list.append(result)
154 + torch.cat(out_list, dim=0)
155 +
156 + cat_result = torch.cat(out_list, dim=0)
157 +
158 + if add_inputs:
159 + ref_out += cat_result
160 + else:
161 + ref_out.copy_(cat_result)
162 +
163 +
164 +class OpType(Enum):
165 + """
166 + LoRA Ops to benchmark and its properties.
167 + """
168 + SGMV_SHRINK = auto()
169 + BGMV_SHRINK = auto()
170 + SGMV_EXPAND = auto()
171 + BGMV_EXPAND = auto()
172 + BGMV_EXPAND_SLICE = auto()
173 +
174 + @staticmethod
175 + def from_str(s: str) -> "OpType":
176 + if s.lower() == 'sgmv_shrink':
177 + return OpType.SGMV_SHRINK
178 + if s.lower() == 'sgmv_expand':
179 + return OpType.SGMV_EXPAND
180 + if s.lower() == 'bgmv_shrink':
181 + return OpType.BGMV_SHRINK
182 + if s.lower() == 'bgmv_expand':
183 + return OpType.BGMV_EXPAND
184 + if s.lower() == "bgmv_expand_slice":
185 + return OpType.BGMV_EXPAND_SLICE
186 + raise ValueError(f"Unrecognized str {s} to convert to OpType")
187 +
188 + def is_shrink_fn(self) -> bool:
189 + return self in [OpType.SGMV_SHRINK, OpType.BGMV_SHRINK]
190 +
191 + def is_expand_fn(self) -> bool:
192 + return self in [OpType.SGMV_EXPAND, OpType.BGMV_EXPAND]
193 +
194 + def is_prefill_op(self) -> bool:
195 + return self in [OpType.SGMV_SHRINK, OpType.SGMV_EXPAND]
196 +
197 + def is_decode_op(self) -> bool:
198 + return self in [
199 + OpType.BGMV_SHRINK, OpType.BGMV_EXPAND, OpType.BGMV_EXPAND_SLICE
200 + ]
201 +
202 + def is_expand_slice_fn(self) -> bool:
203 + return self in [OpType.BGMV_EXPAND_SLICE]
204 +
205 + def num_slices(self) -> List[int]:
206 + if self in [OpType.SGMV_EXPAND, OpType.SGMV_SHRINK]:
207 + # SGMV kernels supports slices
208 + return [1, 2, 3]
209 + if self in [OpType.BGMV_SHRINK, OpType.BGMV_EXPAND]:
210 + return [1]
211 + if self in [OpType.BGMV_EXPAND_SLICE]:
212 + return [2, 3]
213 + raise ValueError(f"Unrecognized OpType {self}")
214 +
215 + def mkn(self, batch_size: int, seq_length: int, hidden_size: int,
216 + lora_rank: int) -> Tuple[int, int, int]:
217 + num_tokens = batch_size * seq_length
218 + if self.is_shrink_fn():
219 + m = num_tokens
220 + k = hidden_size
221 + n = lora_rank
222 + else:
223 + assert self.is_expand_fn() or self.is_expand_slice_fn()
224 + m = num_tokens
225 + k = lora_rank
226 + n = hidden_size
227 + return m, k, n
228 +
229 + def matmul_dtypes(
230 + self, op_dtype: torch.dtype
231 + ) -> Tuple[torch.dtype, torch.dtype, torch.dtype]:
232 + """
233 + return a type, b type and c type for A x B = C
234 + """
235 + if self.is_shrink_fn():
236 + return op_dtype, op_dtype, torch.float32
237 + else:
238 + assert self.is_expand_fn() or self.is_expand_slice_fn()
239 + return torch.float32, op_dtype, op_dtype
240 +
241 + def matmul_shapes(
242 + self, batch_size: int, seq_length: int, hidden_size: int,
243 + lora_rank: int, num_loras: int,
244 + num_slices: int) -> Tuple[Tuple[int], Tuple[int], Tuple[int]]:
245 + """
246 + Given num_slices, return the shapes of the A, B, and C matrices
247 + in A x B = C, for the op_type
248 + """
249 + m, k, n = self.mkn(batch_size, seq_length, hidden_size, lora_rank)
250 +
251 + b_shape = (num_loras, n, k) # col-major
252 + if self == OpType.SGMV_SHRINK:
253 + # SGMV shrink supports num_slices inherently in the kernel
254 + return ((m, k), b_shape, (num_slices, m, n))
255 + if self == OpType.SGMV_EXPAND:
256 + # SGMV expand supports num_slices inherently in the kernel
257 + return ((num_slices, m, k), b_shape, (m, n * num_slices))
258 + if self == OpType.BGMV_SHRINK:
259 + return ((m, k), b_shape, (m, n))
260 + if self == OpType.BGMV_EXPAND:
261 + return ((m, k), b_shape, (m, n))
262 + if self == OpType.BGMV_EXPAND_SLICE:
263 + return ((num_slices, m, k), b_shape, (m, n * num_slices))
264 +
265 + raise ValueError(f"Unrecognized op_type {self}")
266 +
267 + def bench_fn(self) -> Callable:
268 +
269 + def emulate_bgmv_expand_slice(kwargs_list: List[Dict[str, Any]]):
270 + for x in kwargs_list:
271 + bgmv_expand_slice(**x)
272 +
273 + if self == OpType.SGMV_SHRINK:
274 + return sgmv_shrink
275 + if self == OpType.SGMV_EXPAND:
276 + return sgmv_expand
277 + if self == OpType.BGMV_SHRINK:
278 + return bgmv_shrink
279 + if self == OpType.BGMV_EXPAND:
280 + return bgmv_expand
281 + if self == OpType.BGMV_EXPAND_SLICE:
282 + return emulate_bgmv_expand_slice
283 + raise ValueError(f"Unrecognized optype {self}")
284 +
285 + def run_ref_group_gemm(self, output: torch.Tensor, input: torch.Tensor,
286 + lora_weights: List[torch.Tensor],
287 + **kwargs) -> Callable:
288 + """Each benchmark operation expected the input, lora_weights and outputs
289 + in a slightly different format. Refer to self.matmul_shapes().
290 + run_ref_group_gemm accounts for those differences in executing a
291 + reference group gemm for correctness testing.
292 + """
293 + w_dtype = lora_weights[0].dtype
294 + num_slices = len(lora_weights)
295 + if self == OpType.SGMV_SHRINK:
296 + for slice_idx in range(num_slices):
297 + ref_group_gemm(ref_out=output[slice_idx, :],
298 + input=input,
299 + lora_weights=lora_weights[slice_idx],
300 + **kwargs)
301 + if self == OpType.SGMV_EXPAND:
302 + hidden_size = lora_weights[0].shape[1]
303 + for slice_idx in range(num_slices):
304 + slice_offset = slice_idx * hidden_size
305 + ref_group_gemm(
306 + ref_out=output[:, slice_offset:slice_offset + hidden_size],
307 + input=input[slice_idx].clone().to(dtype=w_dtype),
308 + lora_weights=lora_weights[slice_idx],
309 + **kwargs)
310 + if self == OpType.BGMV_SHRINK:
311 + assert num_slices == 1
312 + ref_group_gemm(ref_out=output,
313 + input=input,
314 + lora_weights=lora_weights[0],
315 + **kwargs)
316 + if self == OpType.BGMV_EXPAND:
317 + assert num_slices == 1
318 + ref_group_gemm(ref_out=output,
319 + input=input.clone().to(dtype=w_dtype),
320 + lora_weights=lora_weights[0],
321 + **kwargs)
322 + if self == OpType.BGMV_EXPAND_SLICE:
323 + hidden_size = lora_weights[0].shape[1]
324 + for slice_idx in range(num_slices):
325 + slice_offset = slice_idx * hidden_size
326 + ref_group_gemm(
327 + ref_out=output[:, slice_offset:slice_offset + hidden_size],
328 + input=input[slice_idx].clone().to(dtype=w_dtype),
329 + lora_weights=lora_weights[slice_idx],
330 + **kwargs)
331 + raise ValueError(f"Unrecognized optype {self}")
332 +
333 +
334 +@dataclass
335 +class BenchmarkContext:
336 + """
337 + LoRA benchmark context
338 + """
339 + batch_size: int
340 + hidden_size: int
341 + num_loras: int
342 + num_active_loras: int
343 + lora_rank: int
344 + sort_by_lora_id: bool
345 + dtype: torch.dtype
346 + seq_length: Optional[int] = None
347 + num_slices: Optional[int] = None # num_slices for slice based ops
348 +
349 + def with_seq_length(self, seq_length: int) -> "BenchmarkContext":
350 + ctx = copy.copy(self)
351 + ctx.seq_length = seq_length
352 + return ctx
353 +
354 + def with_num_slices(self, num_slices: int) -> "BenchmarkContext":
355 + ctx = copy.copy(self)
356 + ctx.num_slices = num_slices
357 + return ctx
358 +
359 + def bench_label(self) -> str:
360 + return f"lora-{self.dtype}"
361 +
362 + def bench_sublabel(self, op_type: OpType) -> str:
363 + m, k, n = op_type.mkn(self.batch_size, self.seq_length,
364 + self.hidden_size, self.lora_rank)
365 + desc = {
366 + 'bs': self.batch_size,
367 + 'sl': self.seq_length,
368 + 'm': m,
369 + 'k': k,
370 + 'n': n,
371 + 'num_loras': self.num_loras,
372 + 'sort_by_lora': self.sort_by_lora_id,
373 + 'num_slices': self.num_slices,
374 + }
375 + return json.dumps(desc)
376 +
377 +
378 +@dataclass
379 +class BenchmarkTensors:
380 + """
381 + Input/Output tensors used for benchmarks
382 + """
383 + # matmul tensors
384 + input: torch.Tensor
385 + lora_weights_lst: List[torch.Tensor]
386 + output: torch.Tensor
387 + # metadata tensors
388 + seq_lens: torch.Tensor
389 + seq_start_loc: torch.Tensor
390 + prompt_lora_mapping: torch.Tensor
391 + token_lora_mapping: torch.Tensor
392 +
393 + def io_types(self) -> str:
394 + return (f"{dtype_to_str(self.input.dtype)}x"
395 + f"{dtype_to_str(self.lora_weights_lst[0].dtype)}=>"
396 + f"{dtype_to_str(self.output.dtype)}")
397 +
398 + @staticmethod
399 + def make(ctx: BenchmarkContext,
400 + op_type: OpType,
401 + device: str = "cuda") -> "BenchmarkTensors":
402 +
403 + # Make input / output matmul tensors.
404 + a_shape, b_shape, c_shape = op_type.matmul_shapes(
405 + ctx.batch_size, ctx.seq_length, ctx.hidden_size, ctx.lora_rank,
406 + ctx.num_loras, ctx.num_slices)
407 + a_type, b_type, c_type = op_type.matmul_dtypes(ctx.dtype)
408 + input_tensor, lora_weights, output_tensor = \
409 + make_rand_tensors(a_shape, b_shape, c_shape, a_type, b_type, c_type,
410 + num_slices = ctx.num_slices)
411 +
412 + # Make metadata tensors.
413 + # Keep the metadata tensors in the CPU for further processing if needed.
414 + # The tensors get moved to the GPU before benchmarking.
415 + assert ctx.num_active_loras <= ctx.num_loras
416 + total_tokens = ctx.batch_size * ctx.seq_length
417 +
418 + # Prepare seq lens tensor
419 + seq_len_tensor = torch.randint(ctx.seq_length, ctx.seq_length + 1,
420 + (ctx.batch_size, ))
421 + # Prepare seq_start_loc tensor
422 + seq_start_loc_tensor = torch.cumsum(torch.tensor(
423 + [0] + seq_len_tensor[:-1].tolist(), dtype=torch.long),
424 + dim=0)
425 + assert total_tokens == seq_len_tensor.sum()
426 + # Prepare prompt lora indices tensor
427 + prompt_lora_indices_tensor = make_prompt_lora_mapping(
428 + ctx.batch_size, ctx.num_active_loras, ctx.sort_by_lora_id, "cpu")
429 + # Prepare token lora indices tensor
430 + token_lora_indices_tensor = make_token_lora_mapping(
431 + total_tokens, ctx.batch_size, prompt_lora_indices_tensor,
432 + seq_len_tensor, "cpu")
433 +
434 + return BenchmarkTensors(input_tensor, lora_weights, output_tensor,
435 + seq_len_tensor, seq_start_loc_tensor,
436 + prompt_lora_indices_tensor,
437 + token_lora_indices_tensor)
438 +
439 + def sanity_check(self) -> None:
440 + """
441 + Fails asserts when non-conformality is detected.
442 + """
443 + num_tokens = self.input.shape[-2]
444 + # check metadata tensors
445 + assert torch.sum(self.seq_lens) == num_tokens
446 + num_seqs = self.seq_lens.shape[0]
447 + assert self.seq_start_loc.shape[0] == num_seqs
448 + assert self.prompt_lora_mapping.shape[0] == num_seqs
449 + assert self.token_lora_mapping.shape[0] == num_tokens
450 +
451 + def to_device(self, device: str):
452 + """
453 + Transfer tensors to device if the tensors aren't already on the device
454 + """
455 +
456 + def to_device(tensor: torch.Tensor):
457 + if tensor.device != device:
458 + tensor = tensor.to(device=device)
459 + return tensor
460 +
461 + self.input = to_device(self.input)
462 + self.output = to_device(self.output)
463 + self.seq_lens = to_device(self.seq_lens)
464 + self.seq_start_loc = to_device(self.seq_start_loc)
465 + self.prompt_lora_mapping = to_device(self.prompt_lora_mapping)
466 + self.token_lora_mapping = to_device(self.token_lora_mapping)
467 + for i in range(len(self.lora_weights_lst)):
468 + self.lora_weights_lst[i] = to_device(self.lora_weights_lst[i])
469 +
470 + def metadata(self) -> Tuple[int, int, int]:
471 + """
472 + Return num_seqs, num_tokens and max_seq_len
473 + """
474 + num_seqs = self.seq_lens.shape[0]
475 + num_tokens = self.token_lora_mapping.shape[0]
476 + max_seq_len = torch.max(self.seq_lens).item()
477 + num_slices = len(self.lora_weights_lst)
478 + return num_seqs, num_tokens, max_seq_len, num_slices
479 +
480 + def convert_to_sgmv_benchmark_tensors(self):
481 + """
482 + For sgmv punica kernels, when consecutive sequences have the
483 + same LoRA ID, we just merge them together.
484 + This happens in punica.py::compute_metadata
485 + """
486 +
487 + # Collapse seq_lens and seq_start_loc
488 + _, seq_lens = torch.unique_consecutive(self.token_lora_mapping,
489 + return_counts=True)
490 + cum_result = torch.cumsum(seq_lens, dim=0)
491 + seq_start_loc = torch.zeros_like(seq_lens)
492 + seq_start_loc[1:].copy_(cum_result[:-1])
493 +
494 + # Collapse prompt mapping
495 + prompt_lora_mapping = torch.unique_consecutive(
496 + self.prompt_lora_mapping)
497 +
498 + assert torch.sum(seq_lens) == torch.sum(self.seq_lens), \
499 + f"dont match - new {torch.sum(seq_lens)} vs {torch.sum(self.seq_lens)}"
500 +
501 + self.prompt_lora_mapping = prompt_lora_mapping.to(
502 + dtype=self.prompt_lora_mapping.dtype)
503 + self.seq_lens = seq_lens.to(dtype=self.seq_lens.dtype)
504 + self.seq_start_loc = seq_start_loc.to(dtype=self.seq_start_loc.dtype)
505 +
506 + def as_sgmv_shrink_kwargs(self) -> Dict[str, Any]:
507 + self.convert_to_sgmv_benchmark_tensors()
508 + self.sanity_check()
509 + self.to_device(self.input.device)
510 +
511 + num_seqs, num_tokens, max_seq_len, num_slices = self.metadata()
512 +
513 + # Sanity check matrix shapes.
514 + i_shape, lw_shape, o_shape = self.input.shape, self.lora_weights_lst[
515 + 0].shape, self.output.shape
516 + # Expected input shape [num_tokens, hidden_size]
517 + assert len(i_shape) == 2
518 + assert i_shape[0] == num_tokens
519 + hidden_size = i_shape[1]
520 + # Expected lora weight shape [num_loras, lora_rank, hidden_size]
521 + assert len(lw_shape) == 3
522 + assert lw_shape[2] == hidden_size
523 + lora_rank = lw_shape[1]
524 + # Expected output shape [num_slices, num_tokens, lora_rank]
525 + assert len(o_shape) == 3
526 + assert o_shape == (num_slices, num_tokens, lora_rank)
527 +
528 + return {
529 + 'inputs': self.input,
530 + 'lora_a_weights': self.lora_weights_lst,
531 + 'output_tensor': self.output,
532 + 'b_seq_start_loc': self.seq_start_loc,
533 + 'seq_len_tensor': self.seq_lens,
534 + 'lora_indices_tensor': self.prompt_lora_mapping,
535 + 'batches': num_seqs,
536 + 'max_seq_length': max_seq_len,
537 + 'token_nums': num_tokens,
538 + 'scaling': 1.0,
539 + }
540 +
541 + def as_sgmv_expand_kwargs(self, add_inputs: bool) -> Dict[str, Any]:
542 +
543 + self.convert_to_sgmv_benchmark_tensors()
544 + self.sanity_check()
545 + self.to_device(self.input.device)
546 +
547 + num_seqs, num_tokens, max_seq_len, num_slices = self.metadata()
548 +
549 + # Sanity check matrix shapes.
550 + i_shape, lw_shape, o_shape = self.input.shape, self.lora_weights_lst[
551 + 0].shape, self.output.shape
552 + # Expected input shape : [num_slices, num_tokens, lora_rank]
553 + assert len(i_shape) == 3
554 + assert i_shape[0] == num_slices
555 + assert i_shape[1] == num_tokens
556 + lora_rank = i_shape[2]
557 + # Expected lora weight shape : [num_lora, hidden_size, lora_rank]
558 + assert len(lw_shape) == 3
559 + assert lw_shape[2] == lora_rank
560 + hidden_size = lw_shape[1]
561 + # Expected output shape : [num_tokens, hidden_size * num_slices]
562 + assert len(o_shape) == 2
563 + assert o_shape == (num_tokens, hidden_size * num_slices)
564 +
565 + return {
566 + 'inputs': self.input,
567 + 'lora_b_weights': self.lora_weights_lst,
568 + 'output_tensor': self.output,
569 + 'b_seq_start_loc': self.seq_start_loc,
570 + 'seq_len_tensor': self.seq_lens,
571 + 'lora_indices_tensor': self.prompt_lora_mapping,
572 + 'batches': num_seqs,
573 + 'max_seq_length': max_seq_len,
574 + 'token_nums': num_tokens,
575 + 'offset_start': 0,
576 + 'add_inputs': add_inputs,
577 + }
578 +
579 + def as_bgmv_shrink_kwargs(self) -> Dict[str, Any]:
580 + assert len(self.lora_weights_lst) == 1
581 + self.to_device(self.input.device)
582 +
583 + _, num_tokens, _, _ = self.metadata()
584 + # Sanity check shapes
585 + i_shape, lw_shape, o_shape = self.input.shape, self.lora_weights_lst[
586 + 0].shape, self.output.shape
587 + # Expected input shape [num_tokens, hidden_size]
588 + assert len(i_shape) == 2
589 + assert i_shape[0] == num_tokens
590 + hidden_size = i_shape[1]
591 + # Expected lora weight shape [num_loras, lora_rank, hidden_size]
592 + assert len(lw_shape) == 3
593 + assert lw_shape[2] == hidden_size
594 + lora_rank = lw_shape[1]
595 + # Expected output shape [num_tokens, lora_rank]
596 + assert len(o_shape) == 2
597 + assert o_shape == (num_tokens, lora_rank)
598 +
599 + return {
600 + 'inputs': self.input,
601 + 'lora_a_weights': self.lora_weights_lst[0],
602 + 'output_tensor': self.output,
603 + 'lora_indices_tensor': self.token_lora_mapping,
604 + 'scaling': 1.0
605 + }
606 +
607 + def as_bgmv_expand_kwargs(self, add_inputs: bool):
608 + assert len(self.lora_weights_lst) == 1
609 + self.to_device(self.input.device)
610 +
611 + _, num_tokens, _, _ = self.metadata()
612 + # Sanity check shapes
613 + i_shape, lw_shape, o_shape = self.input.shape, self.lora_weights_lst[
614 + 0].shape, self.output.shape
615 + # Expected input shape [num_tokens, lora_rank]
616 + assert len(i_shape) == 2
617 + assert i_shape[0] == num_tokens
618 + lora_rank = i_shape[1]
619 + # Expected lora weight shape [num_loras, hidden_size, lora_rank]
620 + assert len(lw_shape) == 3
621 + assert lw_shape[2] == lora_rank
622 + hidden_size = lw_shape[1]
623 + # Expected output shape [num_tokens, hidden_size]
624 + assert len(o_shape) == 2
625 + assert o_shape == (num_tokens, hidden_size)
626 +
627 + return {
628 + 'inputs': self.input,
629 + 'lora_b_weights': self.lora_weights_lst[0],
630 + 'output_tensor': self.output,
631 + 'lora_indices_tensor': self.token_lora_mapping,
632 + 'add_inputs': add_inputs
633 + }
634 +
635 + def as_bgmv_expand_slice_kwargs(self, add_inputs: bool) -> Dict[str, Any]:
636 +
637 + _, num_tokens, _, num_slices = self.metadata()
638 + # Sanity check shapes
639 + i_shape, lw_shape, o_shape = self.input.shape, self.lora_weights_lst[
640 + 0].shape, self.output.shape
641 + # Expected input shape [num_slices, num_tokens, lora_rank]
642 + assert len(i_shape) == 3
643 + assert i_shape[0] == num_slices
644 + assert i_shape[1] == num_tokens
645 + lora_rank = i_shape[2]
646 + # Expected lora weight shape [num_loras, hidden_size, lora_rank]
647 + assert len(lw_shape) == 3
648 + assert lw_shape[2] == lora_rank
649 + hidden_size = lw_shape[1]
650 + # Expected output shape [num_tokens, hidden_size * num_slices]
651 + assert len(o_shape) == 2
652 + assert o_shape == (num_tokens, hidden_size * num_slices)
653 +
654 + self.to_device(self.input.device)
655 +
656 + kwargs_list = []
657 + for i in range(num_slices):
658 + kwargs_list.append({
659 + 'inputs': self.input[i],
660 + 'lora_b_weights': self.lora_weights_lst[i],
661 + 'output_tensor': self.output,
662 + 'lora_indices_tensor': self.token_lora_mapping,
663 + 'slice_offset': i * hidden_size,
664 + 'slice_size': hidden_size,
665 + 'add_inputs': add_inputs,
666 + })
667 + return {'kwargs_list': kwargs_list}
668 +
669 + def bench_fn_kwargs(self,
670 + op_type: OpType,
671 + add_inputs: Optional[bool] = None) -> Dict[str, Any]:
672 + if op_type.is_shrink_fn():
673 + assert add_inputs is None
674 + else:
675 + assert add_inputs is not None
676 +
677 + if op_type == OpType.SGMV_SHRINK:
678 + return self.as_sgmv_shrink_kwargs()
679 + if op_type == OpType.SGMV_EXPAND:
680 + return self.as_sgmv_expand_kwargs(add_inputs)
681 + if op_type == OpType.BGMV_SHRINK:
682 + return self.as_bgmv_shrink_kwargs()
683 + if op_type == OpType.BGMV_EXPAND:
684 + return self.as_bgmv_expand_kwargs(add_inputs)
685 + if op_type == OpType.BGMV_EXPAND_SLICE:
686 + return self.as_bgmv_expand_slice_kwargs(add_inputs)
687 + raise ValueError(f"Unrecognized optype {self}")
688 +
689 + def test_correctness(self, op_type: OpType,
690 + expand_fn_add_inputs: Optional[bool]) -> bool:
691 + """
692 + Test correctness of op_type implementation against a grouped gemm
693 + reference implementation.
694 + """
695 + seq_lens_cpu = self.seq_lens.to(device="cpu")
696 + prompt_lora_mapping_cpu = self.prompt_lora_mapping.to(device="cpu")
697 + ref_output = self.output.clone()
698 +
699 + self.output.zero_()
700 + op_type.bench_fn()(
701 + **self.bench_fn_kwargs(op_type, expand_fn_add_inputs))
702 +
703 + op_type.run_ref_group_gemm(
704 + ref_output,
705 + self.input,
706 + self.lora_weights_lst,
707 + seq_lens_cpu=seq_lens_cpu,
708 + prompt_lora_mapping_cpu=prompt_lora_mapping_cpu,
709 + scaling=1.0,
710 + add_inputs=expand_fn_add_inputs)
711 +
712 + rtol, atol = {
713 + torch.float16: (6e-2, 6e-2),
714 + torch.bfloat16: (6e-2, 6e-2),
715 + torch.float32: (1e-2, 1e-2),
716 + }[self.output.dtype]
717 +
718 + return torch.allclose(ref_output, self.output, rtol=rtol, atol=atol)
719 +
720 +
721 +def bench_optype(ctx: BenchmarkContext,
722 + arg_pool_size: int,
723 + op_type: OpType,
724 + cuda_graph_nops: Optional[int] = None,
725 + expand_fn_add_inputs: Optional[bool] = None,
726 + test_correctness: bool = False) -> TMeasurement:
727 +
728 + assert arg_pool_size >= 1
729 + if op_type.is_shrink_fn():
730 + assert expand_fn_add_inputs is None
731 + else:
732 + assert expand_fn_add_inputs is not None
733 +
734 + # BenchmarkContext -> BenchmarkTensors
735 + bench_tensors : List[BenchmarkTensors] = \
736 + [BenchmarkTensors.make(ctx, op_type) for _ in range(arg_pool_size)]
737 + for bt in bench_tensors:
738 + bt.sanity_check()
739 +
740 + # Test correctness of our implementation.
741 + if test_correctness:
742 + assert all([
743 + bt.test_correctness(op_type, expand_fn_add_inputs)
744 + for bt in bench_tensors
745 + ])
746 +
747 + # BenchmarkTensors -> Dict (kwargs)
748 + kwargs_list = [
749 + bt.bench_fn_kwargs(op_type, add_inputs=expand_fn_add_inputs)
750 + for bt in bench_tensors
751 + ]
752 +
753 + # Clear LoRA optimization hash-maps.
754 + _LORA_A_PTR_DICT.clear()
755 + _LORA_B_PTR_DICT.clear()
756 + # Run bench function so that _LORA_A_PTR_DICT and _LORA_B_PTR_DICT are setup
757 + for kwargs in kwargs_list:
758 + op_type.bench_fn()(**kwargs)
759 + torch.cuda.synchronize()
760 +
761 + # Merge into a single kwargs and qualify arguments as ArgPool
762 + kwargs = {k: ArgPool([]) for k in kwargs_list[0]}
763 + for _kwargs in kwargs_list:
764 + for k, v in _kwargs.items():
765 + kwargs[k].values.append(v)
766 +
767 + describe_args = (f"add_inputs={expand_fn_add_inputs}"
768 + if expand_fn_add_inputs is not None else "")
769 + description = (
770 + f"{op_type.name}({describe_args}) ({bench_tensors[0].io_types()})")
771 +
772 + cuda_graph_params = None
773 + if cuda_graph_nops:
774 + cuda_graph_params = CudaGraphBenchParams(cuda_graph_nops)
775 + timer = None
776 + with Bench(cuda_graph_params,
777 + ctx.bench_label(), ctx.bench_sublabel(op_type), description,
778 + op_type.bench_fn(), **kwargs) as bench:
779 + timer = bench.run()
780 + return timer
781 +
782 +
783 +def bench_torch_mm(ctx: BenchmarkContext,
784 + arg_pool_size: int,
785 + op_type: OpType,
786 + cuda_graph_nops: Optional[int] = None) -> TMeasurement:
787 + """
788 + Benchmark basic torch.mm as a roofline.
789 +
790 + When all the input tokens have the same LoRA ID, the LoRA kernels are just
791 + a matmul. This torch.mm benchmark serves as a roofline for that case.
792 +
793 + input op_type is used in determining the m, k, n dimensions for the matmul.
794 + """
795 +
796 + batch_size, hidden_size, lora_rank, seq_length, dtype = (ctx.batch_size,
797 + ctx.hidden_size,
798 + ctx.lora_rank,
799 + ctx.seq_length,
800 + ctx.dtype)
801 +
802 + m, k, n = op_type.mkn(batch_size, seq_length, hidden_size, lora_rank)
803 + # For a fairer comparison.
804 + n = n * ctx.num_slices
805 +
806 + # Get matmul input and output tensors for A x B = C
807 + As, Bs, Cs = [], [], []
808 + for _ in range(arg_pool_size):
809 + As.append(torch.rand((m, k), dtype=dtype).to("cuda"))
810 + Bs.append(torch.rand((n, k), dtype=dtype).to("cuda").t())
811 + Cs.append(torch.rand((m, n), dtype=dtype).to("cuda"))
812 +
813 + # Make torch.mm kwargs
814 + mm_kwargs = {'input': ArgPool(As), 'mat2': ArgPool(Bs), 'out': ArgPool(Cs)}
815 +
816 + description = (
817 + f"single-lora roofline using torch.mm ({dtype_to_str(dtype)}"
818 + f"x{dtype_to_str(dtype)}"
819 + f"=>{dtype_to_str(dtype)})")
820 + cuda_graph_params = None
821 + if cuda_graph_nops:
822 + cuda_graph_params = CudaGraphBenchParams(cuda_graph_nops)
823 + with Bench(cuda_graph_params, ctx.bench_label(),
824 + ctx.bench_sublabel(op_type), description, torch.mm,
825 + **mm_kwargs) as bench:
826 + return bench.run()
827 +
828 +
829 +# runner
830 +def use_cuda_graph_recommendation() -> str:
831 + return """
832 + Triton kernels have a significant launch overhead with
833 + launched directly via python. This overhead is more noticeable
834 + for small the problem sizes. For these cases, it is recommended
835 + to use the script with `--cuda-graph-nops N` to benchmark N
836 + consecutive invocations of the benchmarking operations from
837 + inside a CUDA Graph. Note that the returned measurement is for N
838 + invocations of the operation.
839 + """
840 +
841 +
842 +def print_timers(timers: List[TMeasurement],
843 + args: Optional[argparse.Namespace] = None):
844 + compare = TBenchmark.Compare(timers)
845 + compare.print()
846 +
847 + if args and args.cuda_graph_nops:
848 + print(
849 + f"Note : The timings reported above is for {args.cuda_graph_nops} "
850 + "consecutive invocations of the benchmarking functions. "
851 + f"Please divide by {args.cuda_graph_nops} for single invocation "
852 + "timings.")
853 +
854 + print("Note on Comparison with torch.mm : The torch.mm numbers are "
855 + "benchmark numbers of a simple matmul emulating the single lora "
856 + "case. It is provided as a roofline for comparing our LoRA Kernel "
857 + "implementations. It is expected that the LoRA kernels will be "
858 + "slower than torch.mm in cases where num_loras is big. But for "
859 + "small num_loras the goal should be to match the torch.mm numbers.")
860 +
861 +
862 +def run(args: argparse.Namespace, bench_ctxs: List[BenchmarkContext]):
863 +
864 + if args.cuda_graph_nops is not None:
865 + assert args.cuda_graph_nops > 0
866 + print(f"Benchmarking {args.cuda_graph_nops} invocations inside a CUDA "
867 + "Graph")
868 + else:
869 + print(f"CUDA Graphs not enabled.\n{use_cuda_graph_recommendation()}")
870 +
871 + timers = []
872 + for bench_ctx in bench_ctxs:
873 + for seq_len in args.seq_lengths:
874 + bench_ops: List[OpType] = []
875 + if seq_len == 1:
876 + # bench all decode ops
877 + bench_ops = [op for op in args.op_types if op.is_decode_op()]
878 + else:
879 + # bench all prefill ops
880 + bench_ops = [op for op in args.op_types if op.is_prefill_op()]
881 +
882 + seq_len_timers = []
883 + for bench_op in bench_ops:
884 + for num_slices in bench_op.num_slices():
885 + _ctx = bench_ctx.with_seq_length(seq_len).with_num_slices(
886 + num_slices)
887 + # Benchmark torch.mm as a roofline
888 + seq_len_timers.append(
889 + bench_torch_mm(_ctx, args.arg_pool_size, bench_op,
890 + args.cuda_graph_nops))
891 +
892 + # Benchmark bench_op
893 + expand_fn_add_inputs = [
894 + None
895 + ] if bench_op.is_shrink_fn() else args.expand_fn_add_inputs
896 + for add_input_arg in expand_fn_add_inputs:
897 + seq_len_timers.append(
898 + bench_optype(_ctx, args.arg_pool_size, bench_op,
899 + args.cuda_graph_nops, add_input_arg,
900 + args.test_correctness))
901 +
902 + print_timers(seq_len_timers)
903 + timers.extend(seq_len_timers)
904 +
905 + # Result stdout dump
906 + print("== All Results ====")
907 + print_timers(timers, args)
908 +
909 + if args.output_directory:
910 + # Result file dump
911 + od = Path(args.output_directory)
912 + if not od.exists():
913 + od.mkdir()
914 +
915 + timestamp = int(time.time())
916 + pkl_file = od / f"lora_bench-{timestamp}.pkl"
917 + print(f"Writing benchmarks to {pkl_file}")
918 + with open(pkl_file, "wb") as f:
919 + pickle.dump(timers, f)
920 +
921 +
922 +def as_benchmark_contexts(hidden_sizes: List[int], lora_ranks: List[int],
923 + args: argparse.Namespace) -> List[BenchmarkContext]:
924 +
925 + ctxs: List[BenchmarkContext] = []
926 + for batch_size, hidden_size, lora_rank, num_loras, sort_by_lora_id in product( # noqa
927 + args.batch_sizes, list(hidden_sizes), lora_ranks, args.num_loras,
928 + args.sort_by_lora_id):
929 + ctxs.append(
930 + BenchmarkContext(
931 + batch_size=batch_size,
932 + hidden_size=hidden_size,
933 + lora_rank=lora_rank,
934 + num_loras=num_loras,
935 + num_active_loras=args.num_active_loras
936 + if args.num_active_loras else num_loras,
937 + # To be filled based on the OpType to benchmark
938 + seq_length=None,
939 + sort_by_lora_id=sort_by_lora_id,
940 + dtype=args.dtype,
941 + # To be filled based on the OpType to benchmark
942 + num_slices=None))
943 +
944 + return ctxs
945 +
946 +
947 +def run_list_bench(args: argparse.Namespace):
948 + print(args)
949 +
950 + print("List bench :\n"
951 + f" Hidden Sizes {args.hidden_sizes}"
952 + f" LoRA Ranks {args.lora_ranks}")
953 +
954 + # Get all benchmarking contexts
955 + bench_contexts: List[BenchmarkContext] = as_benchmark_contexts(
956 + hidden_sizes=args.hidden_sizes, lora_ranks=args.lora_ranks, args=args)
957 +
958 + run(args, bench_contexts)
959 +
960 +
961 +def run_range_bench(args: argparse.Namespace):
962 + print(args)
963 +
964 + hidden_sizes = list(
965 + range(args.hidden_sizes_start, args.hidden_sizes_end + 1,
966 + args.hidden_sizes_increment))
967 + lora_ranks = list(
968 + range(args.lora_ranks_start, args.lora_ranks_end + 1,
969 + args.lora_ranks_increment))
970 +
971 + print("Range bench :\n"
972 + f" Hidden Sizes {hidden_sizes}"
973 + f" LoRA Ranks {lora_ranks}")
974 +
975 + # Get all benchmarking contexts
976 + bench_contexts: List[BenchmarkContext] = as_benchmark_contexts(
977 + hidden_sizes=hidden_sizes, lora_ranks=lora_ranks, args=args)
978 +
979 + run(args, bench_contexts)
980 +
981 +
982 +def run_model_bench(args: argparse.Namespace):
983 + print(args)
984 +
985 + def hidden_sizes_from_model(model: str, tp_size: int) -> set[int]:
986 + hidden_sizes = set()
987 + for KN, tp_split_dim in WEIGHT_SHAPES[model]:
988 + KN[tp_split_dim] = KN[tp_split_dim] // tp_size
989 + hidden_sizes.add(KN[1])
990 + return hidden_sizes
991 +
992 + # Get all hidden sizes
993 + hidden_sizes: set[int] = set()
994 + for model_name, tp_size in product(args.models, args.tp_sizes):
995 + hidden_sizes = hidden_sizes.union(
996 + hidden_sizes_from_model(model_name, tp_size))
997 +
998 + print("Model bench :\n"
999 + f" Hidden Sizes {hidden_sizes}"
1000 + f" LoRA Ranks {args.lora_ranks}")
1001 +
1002 + # Get all benchmarking contexts
1003 + bench_contexts: List[BenchmarkContext] = as_benchmark_contexts(
1004 + hidden_sizes=hidden_sizes, lora_ranks=args.lora_ranks, args=args)
1005 +
1006 + run(args, bench_contexts)
1007 +
1008 +
1009 +if __name__ == '__main__':
1010 +
1011 + def to_torch_dtype(dt):
1012 + if dt == "torch.float16":
1013 + return torch.float16
1014 + if dt == "torch.bfloat16":
1015 + return torch.bfloat16
1016 + raise ValueError("unsupported dtype")
1017 +
1018 + def get_bool(s: str) -> bool:
1019 + return s.lower() in ['true', '1']
1020 +
1021 + def add_common_command_args(p: argparse.ArgumentParser):
1022 + p.add_argument(
1023 + "--dtype",
1024 + type=to_torch_dtype,
1025 + required=True,
1026 + help="Available options are ['torch.float16', 'torch.bfloat16']")
1027 +
1028 + p.add_argument(
1029 + "--arg-pool-size",
1030 + type=int,
1031 + default=32,
1032 + help="Run profiles with a pool of input/output/meta tensors instead"
1033 + "of simply reusing the same tensors for all runs. A bigger arg-pool"
1034 + "mitigates hardware caching effects during benchmarking.")
1035 +
1036 + p.add_argument(
1037 + "--cuda-graph-nops",
1038 + type=int,
1039 + help=("when set profiling is done using cudagraph, "
1040 + "with the given number of operations in a graph."
1041 + "Note that the measurement returned is the time "
1042 + "taken for N consecutive executions of the benchmarking "
1043 + "functions, where N is the value of this argument."))
1044 + p.add_argument("--num-loras",
1045 + nargs="+",
1046 + type=int,
1047 + default=DEFAULT_NUM_LORAS)
1048 + p.add_argument("--num-active-loras",
1049 + type=int,
1050 + default=None,
1051 + help="Active LoRAs. When None, all LoRAs are active")
1052 + p.add_argument("--sort-by-lora-id",
1053 + nargs="+",
1054 + type=get_bool,
1055 + default=DEFAULT_SORT_BY_LORA_IDS)
1056 + p.add_argument("--op-types",
1057 + nargs="+",
1058 + type=OpType.from_str,
1059 + default=list(OpType))
1060 + p.add_argument('--seq-lengths',
1061 + nargs="+",
1062 + type=int,
1063 + default=DEFAULT_SEQ_LENGTHS)
1064 + p.add_argument("--batch-sizes",
1065 + nargs="+",
1066 + type=int,
1067 + default=DEFAULT_BATCH_SIZES)
1068 + p.add_argument("--expand-fn-add-inputs",
1069 + nargs="+",
1070 + type=get_bool,
1071 + default=DEFAULT_EXPAND_FN_ADD_INPUTS)
1072 + p.add_argument(
1073 + '-o',
1074 + '--output-directory',
1075 + type=str,
1076 + help=("Output directory to store a the list of benchmarking"
1077 + "TMeasurement objects as a pickle file"))
1078 +
1079 + p.add_argument(
1080 + "--test-correctness",
1081 + action='store_true',
1082 + help=("When enabled, the benchmarking functions are tested"
1083 + "for correctness before the actual benchmarking"))
1084 +
1085 + parser = FlexibleArgumentParser(
1086 + description=f"""
1087 +Benchmark LoRA kernels:
1088 + {use_cuda_graph_recommendation()}
1089 +
1090 + list_bench example:
1091 + python3 benchmarks/kernels/benchmark_lora.py list_bench --arg-pool-size 32 --batch-sizes 1 16 32 --dtype torch.float16 --hidden-sizes 2048 --lora-ranks 16 --num-loras 1 4 --op-types bgmv_shrink bgmv_expand sgmv_shrink sgmv_expand bgmv_expand_slice --seq-lengths 1 16 --sort-by-lora-id 1 --cuda-graph-nops 32
1092 +
1093 + model_bench example:
1094 + python3 benchmarks/kernels/benchmark_lora.py model_bench --models meta-llama/Llama-3-8b --arg-pool-size 32 --batch-sizes 1 16 32 --dtype torch.float16 --lora-ranks 16 --num-loras 1 4 --op-types bgmv_shrink bgmv_expand sgmv_shrink sgmv_expand bgmv_expand_slice --seq-lengths 1 16 --sort-by-lora-id 1 --cuda-graph-nops 32
1095 +
1096 + range_bench example:
1097 + python3 benchmarks/kernels/benchmark_lora.py range_bench --arg-pool-size 32 --batch-sizes 1 16 32 --dtype torch.float16 --num-loras 1 4 --op-types bgmv_shrink bgmv_expand sgmv_shrink sgmv_expand bgmv_expand_slice --seq-lengths 1 16 --sort-by-lora-id 1 --cuda-graph-nops 32 --hidden-sizes-start 1024 --hidden-sizes-end 4096 --hidden-sizes-increment 1024 --lora-ranks-start 8 --lora-ranks-end 24 --lora-ranks-increment 8
1098 + """, # noqa: E501
1099 + formatter_class=argparse.RawTextHelpFormatter)
1100 +
1101 + subparsers = parser.add_subparsers(dest="cmd", required=True)
1102 +
1103 + list_parser = subparsers.add_parser("list_bench")
1104 + list_parser.add_argument("--hidden-sizes",
1105 + nargs="+",
1106 + type=int,
1107 + default=DEFAULT_HIDDEN_SIZES)
1108 + list_parser.add_argument("--lora-ranks",
1109 + nargs="+",
1110 + type=int,
1111 + default=DEFAULT_LORA_RANKS)
1112 + add_common_command_args(list_parser)
1113 + list_parser.set_defaults(func=run_list_bench)
1114 +
1115 + range_parser = subparsers.add_parser("range_bench")
1116 + range_parser.add_argument("--hidden-sizes-start", type=int, required=True)
1117 + range_parser.add_argument("--hidden-sizes-end", type=int, required=True)
1118 + range_parser.add_argument("--hidden-sizes-increment",
1119 + type=int,
1120 + required=True)
1121 + range_parser.add_argument("--lora-ranks-start", type=int, required=True)
1122 + range_parser.add_argument("--lora-ranks-end", type=int, required=True)
1123 + range_parser.add_argument("--lora-ranks-increment",
1124 + type=int,
1125 + required=True)
1126 + add_common_command_args(range_parser)
1127 + range_parser.set_defaults(func=run_range_bench)
1128 +
1129 + model_parser = subparsers.add_parser("model_bench")
1130 + model_parser.add_argument("--models",
1131 + nargs="+",
1132 + type=str,
1133 + default=DEFAULT_MODELS,
1134 + choices=WEIGHT_SHAPES.keys())
1135 + model_parser.add_argument("--tp-sizes",
1136 + nargs="+",
1137 + type=int,
1138 + default=DEFAULT_TP_SIZES)
1139 + model_parser.add_argument("--lora-ranks",
1140 + nargs="+",
1141 + type=int,
1142 + default=DEFAULT_LORA_RANKS)
1143 + add_common_command_args(model_parser)
1144 + model_parser.set_defaults(func=run_model_bench)
1145 +
1146 + args = parser.parse_args()
1147 + args.func(args)
benchmarks/kernels/utils.py new
+210
@@ -0,0 +1,210 @@
1 +import dataclasses
2 +from typing import Any, Callable, Iterable, Optional
3 +
4 +import torch
5 +import torch.utils.benchmark as TBenchmark
6 +from torch.utils.benchmark import Measurement as TMeasurement
7 +
8 +
9 +@dataclasses.dataclass
10 +class CudaGraphBenchParams:
11 + num_ops_in_cuda_graph: int
12 +
13 +
14 +@dataclasses.dataclass
15 +class ArgPool:
16 + """
17 + When some argument of the benchmarking function is annotated with this type,
18 + the benchmarking class (BenchMM) will collapse the argument to a pick a
19 + single value from the given list of values, during function invocation.
20 + For every invocation during a benchmarking run, it will choose a
21 + different value from the list.
22 + """
23 + values: Iterable[Any]
24 +
25 + def __getitem__(self, index):
26 + return self.values[index]
27 +
28 +
29 +class Bench:
30 +
31 + class ArgsIterator:
32 +
33 + def __init__(self, args_list, kwargs_list):
34 + assert len(args_list) == len(kwargs_list)
35 + self.args_list = args_list
36 + self.kwargs_list = kwargs_list
37 + self.n = len(self.args_list)
38 + self.idx = 0
39 +
40 + def __next__(self):
41 + while True:
42 + yield (self.args_list[self.idx], self.kwargs_list[self.idx])
43 + self.idx += 1
44 + self.idx = self.idx % self.n
45 +
46 + def reset(self):
47 + self.idx = 0
48 +
49 + @property
50 + def n_args(self):
51 + return self.n
52 +
53 + def __init__(self, cuda_graph_params: Optional[CudaGraphBenchParams],
54 + label: str, sub_label: str, description: str, fn: Callable,
55 + *args, **kwargs):
56 +
57 + self.cuda_graph_params = cuda_graph_params
58 + self.use_cuda_graph = self.cuda_graph_params is not None
59 + self.label = label
60 + self.sub_label = sub_label
61 + self.description = description
62 + self.fn = fn
63 +
64 + # Process args
65 + self._args = args
66 + self._kwargs = kwargs
67 + self.args_list, self.kwargs_list = self.collapse_argpool(
68 + *args, **kwargs)
69 + self.args_iterator = self.ArgsIterator(self.args_list,
70 + self.kwargs_list)
71 +
72 + # Cudagraph runner
73 + self.g = None
74 + if self.use_cuda_graph:
75 + self.g = self.get_cuda_graph_runner()
76 +
77 + # benchmark run params
78 + self.min_run_time = 1
79 +
80 + def collapse_argpool(self, *args, **kwargs):
81 + argpool_args = [arg for arg in args if isinstance(arg, ArgPool)] + [
82 + arg for arg in kwargs.values() if isinstance(arg, ArgPool)
83 + ]
84 + if len(argpool_args) == 0:
85 + return [args], [kwargs]
86 +
87 + # Make sure all argpools are of the same size
88 + argpool_size = len(argpool_args[0].values)
89 + assert all([argpool_size == len(arg.values) for arg in argpool_args])
90 +
91 + # create copies of the args
92 + args_list = []
93 + kwargs_list = []
94 + for _ in range(argpool_size):
95 + args_list.append(args)
96 + kwargs_list.append(kwargs.copy())
97 +
98 + for i in range(argpool_size):
99 + # collapse args; Just pick the ith value
100 + args_list[i] = tuple([
101 + arg[i] if isinstance(arg, ArgPool) else arg
102 + for arg in args_list[i]
103 + ])
104 +
105 + # collapse kwargs
106 + kwargs_i = kwargs_list[i]
107 + arg_pool_keys = [
108 + k for k, v in kwargs_i.items() if isinstance(v, ArgPool)
109 + ]
110 + for k in arg_pool_keys:
111 + # again just pick the ith value
112 + kwargs_i[k] = kwargs_i[k][i]
113 + kwargs_list[i] = kwargs_i
114 +
115 + return args_list, kwargs_list
116 +
117 + def get_cuda_graph_runner(self):
118 + assert self.use_cuda_graph
119 + assert self.args_iterator is not None
120 +
121 + num_graph_ops = self.cuda_graph_params.num_ops_in_cuda_graph
122 +
123 + # warmup
124 + args_it = self.args_iterator.__next__()
125 + for _ in range(2):
126 + args, kwargs = next(args_it)
127 + self.fn(*args, **kwargs)
128 +
129 + self.args_iterator.reset()
130 + args_it = self.args_iterator.__next__()
131 + stream = torch.cuda.Stream()
132 + with torch.cuda.stream(stream):
133 + g = torch.cuda.CUDAGraph()
134 + with torch.cuda.graph(g):
135 + for _ in range(num_graph_ops):
136 + args, kwargs = next(args_it)
137 + self.fn(*args, **kwargs)
138 + return g
139 +
140 + def run_cudagrah(self) -> TMeasurement:
141 + assert self.use_cuda_graph
142 + globals = {'g': self.g}
143 +
144 + return TBenchmark.Timer(
145 + stmt="g.replay()",
146 + globals=globals,
147 + label=(
148 + f"{self.label}"
149 + f" | cugraph {self.cuda_graph_params.num_ops_in_cuda_graph} ops"
150 + ),
151 + sub_label=self.sub_label,
152 + description=self.description,
153 + ).blocked_autorange(min_run_time=self.min_run_time)
154 +
155 + def run_eager(self) -> TMeasurement:
156 + setup = None
157 + stmt = None
158 + globals = None
159 +
160 + has_arg_pool = self.args_iterator.n_args > 1
161 + if has_arg_pool:
162 + setup = '''
163 + args_iterator.reset()
164 + args_it = args_iterator.__next__()
165 + '''
166 + stmt = '''
167 + args, kwargs = next(args_it)
168 + fn(*args, **kwargs)
169 + '''
170 + globals = {'fn': self.fn, 'args_iterator': self.args_iterator}
171 + else:
172 + # no arg pool. Just use the args and kwargs directly
173 + self.args_iterator.reset()
174 + args_it = self.args_iterator.__next__()
175 + args, kwargs = next(args_it)
176 +
177 + setup = ""
178 + stmt = '''
179 + fn(*args, **kwargs)
180 + '''
181 + globals = {'fn': self.fn, 'args': args, 'kwargs': kwargs}
182 +
183 + return TBenchmark.Timer(
184 + stmt=stmt,
185 + setup=setup,
186 + globals=globals,
187 + label=self.label,
188 + sub_label=self.sub_label,
189 + description=self.description,
190 + ).blocked_autorange(min_run_time=self.min_run_time)
191 +
192 + def run(self) -> TMeasurement:
193 + timer = None
194 + if self.use_cuda_graph: # noqa SIM108
195 + timer = self.run_cudagrah()
196 + else:
197 + timer = self.run_eager()
198 + if not timer.meets_confidence() or timer.has_warnings:
199 + print("Doesn't meet confidence - re-running bench ...")
200 + return self.run()
201 + return timer
202 +
203 + def __enter__(self):
204 + return self
205 +
206 + def __exit__(self, exc_type, exc_value, traceback):
207 + if exc_type:
208 + print(f"exc type {exc_type}")
209 + print(f"exc value {exc_value}")
210 + print(f"exc traceback {traceback}")