A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 02:00 UTC
[misc] Add LoRA kernel micro benchmarks (#11579)
Varun Sundar Rabindranath committed
Jan 16, 2025 at 21:21 UTC
5fd24ec02e0365f96301ac73a31ef06976c256e8
2 files changed
+1357
benchmarks/kernels/benchmark_lora.py
new
+1147
@@ -0,0 +1,1147 @@
1
+import argparse
2
+import copy
3
+import json
4
+import pickle
5
+import time
6
+from dataclasses import dataclass
7
+from enum import Enum, auto
8
+from itertools import product
9
+from pathlib import Path
10
+from typing import Any, Callable, Dict, List, Optional, Tuple
11
+
12
+import torch
13
+import torch.utils.benchmark as TBenchmark
14
+from torch.utils.benchmark import Measurement as TMeasurement
15
+from utils import ArgPool, Bench, CudaGraphBenchParams
16
+from weight_shapes import WEIGHT_SHAPES
17
+
18
+from vllm.lora.ops.triton_ops.bgmv_expand import bgmv_expand
19
+from vllm.lora.ops.triton_ops.bgmv_expand_slice import bgmv_expand_slice
20
+from vllm.lora.ops.triton_ops.bgmv_shrink import bgmv_shrink
21
+from vllm.lora.ops.triton_ops.sgmv_expand import sgmv_expand
22
+from vllm.lora.ops.triton_ops.sgmv_shrink import sgmv_shrink
23
+from vllm.lora.ops.triton_ops.utils import _LORA_A_PTR_DICT, _LORA_B_PTR_DICT
24
+from vllm.utils import FlexibleArgumentParser
25
+
26
+DEFAULT_MODELS = list(WEIGHT_SHAPES.keys())
27
+DEFAULT_TP_SIZES = [1]
28
+DEFAULT_BATCH_SIZES = [
29
+ 1, 16, 32, 64, 128, 192, 256, 320, 384, 448, 512, 640, 768, 896, 1024,
30
+ 2048, 3072, 4096, 5120, 6144, 7168, 8192
31
+]
32
+DEFAULT_HIDDEN_SIZES = [1024, 2048, 4096, 8192, 16384]
33
+DEFAULT_LORA_RANKS = [16]
34
+DEFAULT_NUM_LORAS = [1, 2, 3, 4]
35
+DEFAULT_SORT_BY_LORA_IDS = [False, True]
36
+DEFAULT_SEQ_LENGTHS = [1]
37
+DEFAULT_EXPAND_FN_ADD_INPUTS = [True, False]
38
+
39
+
40
+# Utilities
41
+def dtype_to_str(dtype: torch.dtype):
42
+ if dtype == torch.float16:
43
+ return "f16"
44
+ if dtype == torch.bfloat16:
45
+ return "bf16"
46
+ if dtype == torch.float32:
47
+ return "f32"
48
+ raise ValueError(f"Unsupported dtype {dtype}")
49
+
50
+
51
+def make_rand_lora_weight_tensor(k: int,
52
+ n: int,
53
+ num_loras: int,
54
+ dtype: torch.dtype,
55
+ device: str = "cuda") -> torch.Tensor:
56
+
57
+ # LoRA weights column major
58
+ return torch.rand((num_loras, n, k), dtype=dtype).to(device)
59
+
60
+
61
+def make_rand_tensors(
62
+ a_shape: Tuple[int],
63
+ b_shape: Tuple[int],
64
+ c_shape: Tuple[int],
65
+ a_dtype: torch.dtype,
66
+ b_dtype: torch.dtype,
67
+ c_dtype: torch.dtype,
68
+ num_slices: int,
69
+ device: str = "cuda",
70
+) -> Tuple[torch.Tensor, List[torch.Tensor], torch.Tensor]:
71
+ """
72
+ Make LoRA input/output matrices.
73
+ """
74
+ A = torch.rand(a_shape, dtype=a_dtype).to(device)
75
+
76
+ # LoRA weights column major
77
+ Bs = [
78
+ torch.rand(b_shape, dtype=b_dtype).to(device)
79
+ for _ in range(num_slices)
80
+ ]
81
+
82
+ C = torch.zeros(c_shape, dtype=c_dtype).to(device)
83
+ return A, Bs, C
84
+
85
+
86
+def make_prompt_lora_mapping(num_prompts: int, num_active_loras: int,
87
+ sort_by_lora_id: bool,
88
+ device: str) -> torch.Tensor:
89
+ """
90
+ All prompts are mapped to a Lora ID in range [0, num_active_loras).
91
+ where 0 refers to first lora, 1 refers to second lora and so on.
92
+ """
93
+ assert num_active_loras > 0
94
+
95
+ if not sort_by_lora_id:
96
+ return torch.randint(0,
97
+ num_active_loras, (num_prompts, ),
98
+ dtype=torch.long)
99
+
100
+ # Divide LoRAs equally and in order.
101
+ part_size = num_prompts // num_active_loras
102
+ part_size = max(part_size, 1)
103
+
104
+ lora_id = 0
105
+ prompt_lora_mapping = []
106
+ while len(prompt_lora_mapping) < num_prompts:
107
+ prompt_lora_mapping.extend([lora_id] * part_size)
108
+ lora_id = lora_id + 1 if lora_id + 1 < num_active_loras else lora_id
109
+ return torch.tensor(prompt_lora_mapping[:num_prompts],
110
+ dtype=torch.long,
111
+ device=device)
112
+
113
+
114
+def make_token_lora_mapping(num_tokens: int, num_prompts: int,
115
+ prompt_lora_mapping: torch.Tensor,
116
+ seq_len_tensor: torch.Tensor, device: str):
117
+ """
118
+ Make token_lora_mapping from prompt_lora_mapping and seq_lens_tensor
119
+ """
120
+ assert prompt_lora_mapping.shape[0] == num_prompts
121
+
122
+ # token to lora index mapping
123
+ token_lora_mapping = [0] * num_tokens
124
+ current_offset = 0
125
+ for b_id in range(num_prompts):
126
+ lora_index = prompt_lora_mapping[b_id].item()
127
+ s = current_offset
128
+ e = s + seq_len_tensor[b_id].item()
129
+ token_lora_mapping[s:e] = [lora_index] * (e - s)
130
+ current_offset += seq_len_tensor[b_id].item()
131
+
132
+ return torch.tensor(token_lora_mapping, dtype=torch.long, device=device)
133
+
134
+
135
+def ref_group_gemm(ref_out: torch.Tensor, input: torch.Tensor,
136
+ lora_weights: List[torch.Tensor],
137
+ seq_lens_cpu: torch.Tensor,
138
+ prompt_lora_mapping_cpu: torch.Tensor, scaling: float,
139
+ add_inputs: Optional[bool]):
140
+ """
141
+ Torch group gemm reference implementation to test correctness of
142
+ benchmarking operations.
143
+ """
144
+ batches = seq_lens_cpu.size(0)
145
+ out_list = []
146
+ current_offset = 0
147
+ for lora_index, b_length in zip(range(batches), seq_lens_cpu):
148
+ x = input[current_offset:b_length + current_offset, :]
149
+ current_offset += b_length
150
+ w = lora_weights[prompt_lora_mapping_cpu[lora_index]]
151
+ result = torch.nn.functional.linear(x, w)
152
+ result *= scaling
153
+ out_list.append(result)
154
+ torch.cat(out_list, dim=0)
155
+
156
+ cat_result = torch.cat(out_list, dim=0)
157
+
158
+ if add_inputs:
159
+ ref_out += cat_result
160
+ else:
161
+ ref_out.copy_(cat_result)
162
+
163
+
164
+class OpType(Enum):
165
+ """
166
+ LoRA Ops to benchmark and its properties.
167
+ """
168
+ SGMV_SHRINK = auto()
169
+ BGMV_SHRINK = auto()
170
+ SGMV_EXPAND = auto()
171
+ BGMV_EXPAND = auto()
172
+ BGMV_EXPAND_SLICE = auto()
173
+
174
+ @staticmethod
175
+ def from_str(s: str) -> "OpType":
176
+ if s.lower() == 'sgmv_shrink':
177
+ return OpType.SGMV_SHRINK
178
+ if s.lower() == 'sgmv_expand':
179
+ return OpType.SGMV_EXPAND
180
+ if s.lower() == 'bgmv_shrink':
181
+ return OpType.BGMV_SHRINK
182
+ if s.lower() == 'bgmv_expand':
183
+ return OpType.BGMV_EXPAND
184
+ if s.lower() == "bgmv_expand_slice":
185
+ return OpType.BGMV_EXPAND_SLICE
186
+ raise ValueError(f"Unrecognized str {s} to convert to OpType")
187
+
188
+ def is_shrink_fn(self) -> bool:
189
+ return self in [OpType.SGMV_SHRINK, OpType.BGMV_SHRINK]
190
+
191
+ def is_expand_fn(self) -> bool:
192
+ return self in [OpType.SGMV_EXPAND, OpType.BGMV_EXPAND]
193
+
194
+ def is_prefill_op(self) -> bool:
195
+ return self in [OpType.SGMV_SHRINK, OpType.SGMV_EXPAND]
196
+
197
+ def is_decode_op(self) -> bool:
198
+ return self in [
199
+ OpType.BGMV_SHRINK, OpType.BGMV_EXPAND, OpType.BGMV_EXPAND_SLICE
200
+ ]
201
+
202
+ def is_expand_slice_fn(self) -> bool:
203
+ return self in [OpType.BGMV_EXPAND_SLICE]
204
+
205
+ def num_slices(self) -> List[int]:
206
+ if self in [OpType.SGMV_EXPAND, OpType.SGMV_SHRINK]:
207
+ # SGMV kernels supports slices
208
+ return [1, 2, 3]
209
+ if self in [OpType.BGMV_SHRINK, OpType.BGMV_EXPAND]:
210
+ return [1]
211
+ if self in [OpType.BGMV_EXPAND_SLICE]:
212
+ return [2, 3]
213
+ raise ValueError(f"Unrecognized OpType {self}")
214
+
215
+ def mkn(self, batch_size: int, seq_length: int, hidden_size: int,
216
+ lora_rank: int) -> Tuple[int, int, int]:
217
+ num_tokens = batch_size * seq_length
218
+ if self.is_shrink_fn():
219
+ m = num_tokens
220
+ k = hidden_size
221
+ n = lora_rank
222
+ else:
223
+ assert self.is_expand_fn() or self.is_expand_slice_fn()
224
+ m = num_tokens
225
+ k = lora_rank
226
+ n = hidden_size
227
+ return m, k, n
228
+
229
+ def matmul_dtypes(
230
+ self, op_dtype: torch.dtype
231
+ ) -> Tuple[torch.dtype, torch.dtype, torch.dtype]:
232
+ """
233
+ return a type, b type and c type for A x B = C
234
+ """
235
+ if self.is_shrink_fn():
236
+ return op_dtype, op_dtype, torch.float32
237
+ else:
238
+ assert self.is_expand_fn() or self.is_expand_slice_fn()
239
+ return torch.float32, op_dtype, op_dtype
240
+
241
+ def matmul_shapes(
242
+ self, batch_size: int, seq_length: int, hidden_size: int,
243
+ lora_rank: int, num_loras: int,
244
+ num_slices: int) -> Tuple[Tuple[int], Tuple[int], Tuple[int]]:
245
+ """
246
+ Given num_slices, return the shapes of the A, B, and C matrices
247
+ in A x B = C, for the op_type
248
+ """
249
+ m, k, n = self.mkn(batch_size, seq_length, hidden_size, lora_rank)
250
+
251
+ b_shape = (num_loras, n, k) # col-major
252
+ if self == OpType.SGMV_SHRINK:
253
+ # SGMV shrink supports num_slices inherently in the kernel
254
+ return ((m, k), b_shape, (num_slices, m, n))
255
+ if self == OpType.SGMV_EXPAND:
256
+ # SGMV expand supports num_slices inherently in the kernel
257
+ return ((num_slices, m, k), b_shape, (m, n * num_slices))
258
+ if self == OpType.BGMV_SHRINK:
259
+ return ((m, k), b_shape, (m, n))
260
+ if self == OpType.BGMV_EXPAND:
261
+ return ((m, k), b_shape, (m, n))
262
+ if self == OpType.BGMV_EXPAND_SLICE:
263
+ return ((num_slices, m, k), b_shape, (m, n * num_slices))
264
+
265
+ raise ValueError(f"Unrecognized op_type {self}")
266
+
267
+ def bench_fn(self) -> Callable:
268
+
269
+ def emulate_bgmv_expand_slice(kwargs_list: List[Dict[str, Any]]):
270
+ for x in kwargs_list:
271
+ bgmv_expand_slice(**x)
272
+
273
+ if self == OpType.SGMV_SHRINK:
274
+ return sgmv_shrink
275
+ if self == OpType.SGMV_EXPAND:
276
+ return sgmv_expand
277
+ if self == OpType.BGMV_SHRINK:
278
+ return bgmv_shrink
279
+ if self == OpType.BGMV_EXPAND:
280
+ return bgmv_expand
281
+ if self == OpType.BGMV_EXPAND_SLICE:
282
+ return emulate_bgmv_expand_slice
283
+ raise ValueError(f"Unrecognized optype {self}")
284
+
285
+ def run_ref_group_gemm(self, output: torch.Tensor, input: torch.Tensor,
286
+ lora_weights: List[torch.Tensor],
287
+ **kwargs) -> Callable:
288
+ """Each benchmark operation expected the input, lora_weights and outputs
289
+ in a slightly different format. Refer to self.matmul_shapes().
290
+ run_ref_group_gemm accounts for those differences in executing a
291
+ reference group gemm for correctness testing.
292
+ """
293
+ w_dtype = lora_weights[0].dtype
294
+ num_slices = len(lora_weights)
295
+ if self == OpType.SGMV_SHRINK:
296
+ for slice_idx in range(num_slices):
297
+ ref_group_gemm(ref_out=output[slice_idx, :],
298
+ input=input,
299
+ lora_weights=lora_weights[slice_idx],
300
+ **kwargs)
301
+ if self == OpType.SGMV_EXPAND:
302
+ hidden_size = lora_weights[0].shape[1]
303
+ for slice_idx in range(num_slices):
304
+ slice_offset = slice_idx * hidden_size
305
+ ref_group_gemm(
306
+ ref_out=output[:, slice_offset:slice_offset + hidden_size],
307
+ input=input[slice_idx].clone().to(dtype=w_dtype),
308
+ lora_weights=lora_weights[slice_idx],
309
+ **kwargs)
310
+ if self == OpType.BGMV_SHRINK:
311
+ assert num_slices == 1
312
+ ref_group_gemm(ref_out=output,
313
+ input=input,
314
+ lora_weights=lora_weights[0],
315
+ **kwargs)
316
+ if self == OpType.BGMV_EXPAND:
317
+ assert num_slices == 1
318
+ ref_group_gemm(ref_out=output,
319
+ input=input.clone().to(dtype=w_dtype),
320
+ lora_weights=lora_weights[0],
321
+ **kwargs)
322
+ if self == OpType.BGMV_EXPAND_SLICE:
323
+ hidden_size = lora_weights[0].shape[1]
324
+ for slice_idx in range(num_slices):
325
+ slice_offset = slice_idx * hidden_size
326
+ ref_group_gemm(
327
+ ref_out=output[:, slice_offset:slice_offset + hidden_size],
328
+ input=input[slice_idx].clone().to(dtype=w_dtype),
329
+ lora_weights=lora_weights[slice_idx],
330
+ **kwargs)
331
+ raise ValueError(f"Unrecognized optype {self}")
332
+
333
+
334
+@dataclass
335
+class BenchmarkContext:
336
+ """
337
+ LoRA benchmark context
338
+ """
339
+ batch_size: int
340
+ hidden_size: int
341
+ num_loras: int
342
+ num_active_loras: int
343
+ lora_rank: int
344
+ sort_by_lora_id: bool
345
+ dtype: torch.dtype
346
+ seq_length: Optional[int] = None
347
+ num_slices: Optional[int] = None # num_slices for slice based ops
348
+
349
+ def with_seq_length(self, seq_length: int) -> "BenchmarkContext":
350
+ ctx = copy.copy(self)
351
+ ctx.seq_length = seq_length
352
+ return ctx
353
+
354
+ def with_num_slices(self, num_slices: int) -> "BenchmarkContext":
355
+ ctx = copy.copy(self)
356
+ ctx.num_slices = num_slices
357
+ return ctx
358
+
359
+ def bench_label(self) -> str:
360
+ return f"lora-{self.dtype}"
361
+
362
+ def bench_sublabel(self, op_type: OpType) -> str:
363
+ m, k, n = op_type.mkn(self.batch_size, self.seq_length,
364
+ self.hidden_size, self.lora_rank)
365
+ desc = {
366
+ 'bs': self.batch_size,
367
+ 'sl': self.seq_length,
368
+ 'm': m,
369
+ 'k': k,
370
+ 'n': n,
371
+ 'num_loras': self.num_loras,
372
+ 'sort_by_lora': self.sort_by_lora_id,
373
+ 'num_slices': self.num_slices,
374
+ }
375
+ return json.dumps(desc)
376
+
377
+
378
+@dataclass
379
+class BenchmarkTensors:
380
+ """
381
+ Input/Output tensors used for benchmarks
382
+ """
383
+ # matmul tensors
384
+ input: torch.Tensor
385
+ lora_weights_lst: List[torch.Tensor]
386
+ output: torch.Tensor
387
+ # metadata tensors
388
+ seq_lens: torch.Tensor
389
+ seq_start_loc: torch.Tensor
390
+ prompt_lora_mapping: torch.Tensor
391
+ token_lora_mapping: torch.Tensor
392
+
393
+ def io_types(self) -> str:
394
+ return (f"{dtype_to_str(self.input.dtype)}x"
395
+ f"{dtype_to_str(self.lora_weights_lst[0].dtype)}=>"
396
+ f"{dtype_to_str(self.output.dtype)}")
397
+
398
+ @staticmethod
399
+ def make(ctx: BenchmarkContext,
400
+ op_type: OpType,
401
+ device: str = "cuda") -> "BenchmarkTensors":
402
+
403
+ # Make input / output matmul tensors.
404
+ a_shape, b_shape, c_shape = op_type.matmul_shapes(
405
+ ctx.batch_size, ctx.seq_length, ctx.hidden_size, ctx.lora_rank,
406
+ ctx.num_loras, ctx.num_slices)
407
+ a_type, b_type, c_type = op_type.matmul_dtypes(ctx.dtype)
408
+ input_tensor, lora_weights, output_tensor = \
409
+ make_rand_tensors(a_shape, b_shape, c_shape, a_type, b_type, c_type,
410
+ num_slices = ctx.num_slices)
411
+
412
+ # Make metadata tensors.
413
+ # Keep the metadata tensors in the CPU for further processing if needed.
414
+ # The tensors get moved to the GPU before benchmarking.
415
+ assert ctx.num_active_loras <= ctx.num_loras
416
+ total_tokens = ctx.batch_size * ctx.seq_length
417
+
418
+ # Prepare seq lens tensor
419
+ seq_len_tensor = torch.randint(ctx.seq_length, ctx.seq_length + 1,
420
+ (ctx.batch_size, ))
421
+ # Prepare seq_start_loc tensor
422
+ seq_start_loc_tensor = torch.cumsum(torch.tensor(
423
+ [0] + seq_len_tensor[:-1].tolist(), dtype=torch.long),
424
+ dim=0)
425
+ assert total_tokens == seq_len_tensor.sum()
426
+ # Prepare prompt lora indices tensor
427
+ prompt_lora_indices_tensor = make_prompt_lora_mapping(
428
+ ctx.batch_size, ctx.num_active_loras, ctx.sort_by_lora_id, "cpu")
429
+ # Prepare token lora indices tensor
430
+ token_lora_indices_tensor = make_token_lora_mapping(
431
+ total_tokens, ctx.batch_size, prompt_lora_indices_tensor,
432
+ seq_len_tensor, "cpu")
433
+
434
+ return BenchmarkTensors(input_tensor, lora_weights, output_tensor,
435
+ seq_len_tensor, seq_start_loc_tensor,
436
+ prompt_lora_indices_tensor,
437
+ token_lora_indices_tensor)
438
+
439
+ def sanity_check(self) -> None:
440
+ """
441
+ Fails asserts when non-conformality is detected.
442
+ """
443
+ num_tokens = self.input.shape[-2]
444
+ # check metadata tensors
445
+ assert torch.sum(self.seq_lens) == num_tokens
446
+ num_seqs = self.seq_lens.shape[0]
447
+ assert self.seq_start_loc.shape[0] == num_seqs
448
+ assert self.prompt_lora_mapping.shape[0] == num_seqs
449
+ assert self.token_lora_mapping.shape[0] == num_tokens
450
+
451
+ def to_device(self, device: str):
452
+ """
453
+ Transfer tensors to device if the tensors aren't already on the device
454
+ """
455
+
456
+ def to_device(tensor: torch.Tensor):
457
+ if tensor.device != device:
458
+ tensor = tensor.to(device=device)
459
+ return tensor
460
+
461
+ self.input = to_device(self.input)
462
+ self.output = to_device(self.output)
463
+ self.seq_lens = to_device(self.seq_lens)
464
+ self.seq_start_loc = to_device(self.seq_start_loc)
465
+ self.prompt_lora_mapping = to_device(self.prompt_lora_mapping)
466
+ self.token_lora_mapping = to_device(self.token_lora_mapping)
467
+ for i in range(len(self.lora_weights_lst)):
468
+ self.lora_weights_lst[i] = to_device(self.lora_weights_lst[i])
469
+
470
+ def metadata(self) -> Tuple[int, int, int]:
471
+ """
472
+ Return num_seqs, num_tokens and max_seq_len
473
+ """
474
+ num_seqs = self.seq_lens.shape[0]
475
+ num_tokens = self.token_lora_mapping.shape[0]
476
+ max_seq_len = torch.max(self.seq_lens).item()
477
+ num_slices = len(self.lora_weights_lst)
478
+ return num_seqs, num_tokens, max_seq_len, num_slices
479
+
480
+ def convert_to_sgmv_benchmark_tensors(self):
481
+ """
482
+ For sgmv punica kernels, when consecutive sequences have the
483
+ same LoRA ID, we just merge them together.
484
+ This happens in punica.py::compute_metadata
485
+ """
486
+
487
+ # Collapse seq_lens and seq_start_loc
488
+ _, seq_lens = torch.unique_consecutive(self.token_lora_mapping,
489
+ return_counts=True)
490
+ cum_result = torch.cumsum(seq_lens, dim=0)
491
+ seq_start_loc = torch.zeros_like(seq_lens)
492
+ seq_start_loc[1:].copy_(cum_result[:-1])
493
+
494
+ # Collapse prompt mapping
495
+ prompt_lora_mapping = torch.unique_consecutive(
496
+ self.prompt_lora_mapping)
497
+
498
+ assert torch.sum(seq_lens) == torch.sum(self.seq_lens), \
499
+ f"dont match - new {torch.sum(seq_lens)} vs {torch.sum(self.seq_lens)}"
500
+
501
+ self.prompt_lora_mapping = prompt_lora_mapping.to(
502
+ dtype=self.prompt_lora_mapping.dtype)
503
+ self.seq_lens = seq_lens.to(dtype=self.seq_lens.dtype)
504
+ self.seq_start_loc = seq_start_loc.to(dtype=self.seq_start_loc.dtype)
505
+
506
+ def as_sgmv_shrink_kwargs(self) -> Dict[str, Any]:
507
+ self.convert_to_sgmv_benchmark_tensors()
508
+ self.sanity_check()
509
+ self.to_device(self.input.device)
510
+
511
+ num_seqs, num_tokens, max_seq_len, num_slices = self.metadata()
512
+
513
+ # Sanity check matrix shapes.
514
+ i_shape, lw_shape, o_shape = self.input.shape, self.lora_weights_lst[
515
+ 0].shape, self.output.shape
516
+ # Expected input shape [num_tokens, hidden_size]
517
+ assert len(i_shape) == 2
518
+ assert i_shape[0] == num_tokens
519
+ hidden_size = i_shape[1]
520
+ # Expected lora weight shape [num_loras, lora_rank, hidden_size]
521
+ assert len(lw_shape) == 3
522
+ assert lw_shape[2] == hidden_size
523
+ lora_rank = lw_shape[1]
524
+ # Expected output shape [num_slices, num_tokens, lora_rank]
525
+ assert len(o_shape) == 3
526
+ assert o_shape == (num_slices, num_tokens, lora_rank)
527
+
528
+ return {
529
+ 'inputs': self.input,
530
+ 'lora_a_weights': self.lora_weights_lst,
531
+ 'output_tensor': self.output,
532
+ 'b_seq_start_loc': self.seq_start_loc,
533
+ 'seq_len_tensor': self.seq_lens,
534
+ 'lora_indices_tensor': self.prompt_lora_mapping,
535
+ 'batches': num_seqs,
536
+ 'max_seq_length': max_seq_len,
537
+ 'token_nums': num_tokens,
538
+ 'scaling': 1.0,
539
+ }
540
+
541
+ def as_sgmv_expand_kwargs(self, add_inputs: bool) -> Dict[str, Any]:
542
+
543
+ self.convert_to_sgmv_benchmark_tensors()
544
+ self.sanity_check()
545
+ self.to_device(self.input.device)
546
+
547
+ num_seqs, num_tokens, max_seq_len, num_slices = self.metadata()
548
+
549
+ # Sanity check matrix shapes.
550
+ i_shape, lw_shape, o_shape = self.input.shape, self.lora_weights_lst[
551
+ 0].shape, self.output.shape
552
+ # Expected input shape : [num_slices, num_tokens, lora_rank]
553
+ assert len(i_shape) == 3
554
+ assert i_shape[0] == num_slices
555
+ assert i_shape[1] == num_tokens
556
+ lora_rank = i_shape[2]
557
+ # Expected lora weight shape : [num_lora, hidden_size, lora_rank]
558
+ assert len(lw_shape) == 3
559
+ assert lw_shape[2] == lora_rank
560
+ hidden_size = lw_shape[1]
561
+ # Expected output shape : [num_tokens, hidden_size * num_slices]
562
+ assert len(o_shape) == 2
563
+ assert o_shape == (num_tokens, hidden_size * num_slices)
564
+
565
+ return {
566
+ 'inputs': self.input,
567
+ 'lora_b_weights': self.lora_weights_lst,
568
+ 'output_tensor': self.output,
569
+ 'b_seq_start_loc': self.seq_start_loc,
570
+ 'seq_len_tensor': self.seq_lens,
571
+ 'lora_indices_tensor': self.prompt_lora_mapping,
572
+ 'batches': num_seqs,
573
+ 'max_seq_length': max_seq_len,
574
+ 'token_nums': num_tokens,
575
+ 'offset_start': 0,
576
+ 'add_inputs': add_inputs,
577
+ }
578
+
579
+ def as_bgmv_shrink_kwargs(self) -> Dict[str, Any]:
580
+ assert len(self.lora_weights_lst) == 1
581
+ self.to_device(self.input.device)
582
+
583
+ _, num_tokens, _, _ = self.metadata()
584
+ # Sanity check shapes
585
+ i_shape, lw_shape, o_shape = self.input.shape, self.lora_weights_lst[
586
+ 0].shape, self.output.shape
587
+ # Expected input shape [num_tokens, hidden_size]
588
+ assert len(i_shape) == 2
589
+ assert i_shape[0] == num_tokens
590
+ hidden_size = i_shape[1]
591
+ # Expected lora weight shape [num_loras, lora_rank, hidden_size]
592
+ assert len(lw_shape) == 3
593
+ assert lw_shape[2] == hidden_size
594
+ lora_rank = lw_shape[1]
595
+ # Expected output shape [num_tokens, lora_rank]
596
+ assert len(o_shape) == 2
597
+ assert o_shape == (num_tokens, lora_rank)
598
+
599
+ return {
600
+ 'inputs': self.input,
601
+ 'lora_a_weights': self.lora_weights_lst[0],
602
+ 'output_tensor': self.output,
603
+ 'lora_indices_tensor': self.token_lora_mapping,
604
+ 'scaling': 1.0
605
+ }
606
+
607
+ def as_bgmv_expand_kwargs(self, add_inputs: bool):
608
+ assert len(self.lora_weights_lst) == 1
609
+ self.to_device(self.input.device)
610
+
611
+ _, num_tokens, _, _ = self.metadata()
612
+ # Sanity check shapes
613
+ i_shape, lw_shape, o_shape = self.input.shape, self.lora_weights_lst[
614
+ 0].shape, self.output.shape
615
+ # Expected input shape [num_tokens, lora_rank]
616
+ assert len(i_shape) == 2
617
+ assert i_shape[0] == num_tokens
618
+ lora_rank = i_shape[1]
619
+ # Expected lora weight shape [num_loras, hidden_size, lora_rank]
620
+ assert len(lw_shape) == 3
621
+ assert lw_shape[2] == lora_rank
622
+ hidden_size = lw_shape[1]
623
+ # Expected output shape [num_tokens, hidden_size]
624
+ assert len(o_shape) == 2
625
+ assert o_shape == (num_tokens, hidden_size)
626
+
627
+ return {
628
+ 'inputs': self.input,
629
+ 'lora_b_weights': self.lora_weights_lst[0],
630
+ 'output_tensor': self.output,
631
+ 'lora_indices_tensor': self.token_lora_mapping,
632
+ 'add_inputs': add_inputs
633
+ }
634
+
635
+ def as_bgmv_expand_slice_kwargs(self, add_inputs: bool) -> Dict[str, Any]:
636
+
637
+ _, num_tokens, _, num_slices = self.metadata()
638
+ # Sanity check shapes
639
+ i_shape, lw_shape, o_shape = self.input.shape, self.lora_weights_lst[
640
+ 0].shape, self.output.shape
641
+ # Expected input shape [num_slices, num_tokens, lora_rank]
642
+ assert len(i_shape) == 3
643
+ assert i_shape[0] == num_slices
644
+ assert i_shape[1] == num_tokens
645
+ lora_rank = i_shape[2]
646
+ # Expected lora weight shape [num_loras, hidden_size, lora_rank]
647
+ assert len(lw_shape) == 3
648
+ assert lw_shape[2] == lora_rank
649
+ hidden_size = lw_shape[1]
650
+ # Expected output shape [num_tokens, hidden_size * num_slices]
651
+ assert len(o_shape) == 2
652
+ assert o_shape == (num_tokens, hidden_size * num_slices)
653
+
654
+ self.to_device(self.input.device)
655
+
656
+ kwargs_list = []
657
+ for i in range(num_slices):
658
+ kwargs_list.append({
659
+ 'inputs': self.input[i],
660
+ 'lora_b_weights': self.lora_weights_lst[i],
661
+ 'output_tensor': self.output,
662
+ 'lora_indices_tensor': self.token_lora_mapping,
663
+ 'slice_offset': i * hidden_size,
664
+ 'slice_size': hidden_size,
665
+ 'add_inputs': add_inputs,
666
+ })
667
+ return {'kwargs_list': kwargs_list}
668
+
669
+ def bench_fn_kwargs(self,
670
+ op_type: OpType,
671
+ add_inputs: Optional[bool] = None) -> Dict[str, Any]:
672
+ if op_type.is_shrink_fn():
673
+ assert add_inputs is None
674
+ else:
675
+ assert add_inputs is not None
676
+
677
+ if op_type == OpType.SGMV_SHRINK:
678
+ return self.as_sgmv_shrink_kwargs()
679
+ if op_type == OpType.SGMV_EXPAND:
680
+ return self.as_sgmv_expand_kwargs(add_inputs)
681
+ if op_type == OpType.BGMV_SHRINK:
682
+ return self.as_bgmv_shrink_kwargs()
683
+ if op_type == OpType.BGMV_EXPAND:
684
+ return self.as_bgmv_expand_kwargs(add_inputs)
685
+ if op_type == OpType.BGMV_EXPAND_SLICE:
686
+ return self.as_bgmv_expand_slice_kwargs(add_inputs)
687
+ raise ValueError(f"Unrecognized optype {self}")
688
+
689
+ def test_correctness(self, op_type: OpType,
690
+ expand_fn_add_inputs: Optional[bool]) -> bool:
691
+ """
692
+ Test correctness of op_type implementation against a grouped gemm
693
+ reference implementation.
694
+ """
695
+ seq_lens_cpu = self.seq_lens.to(device="cpu")
696
+ prompt_lora_mapping_cpu = self.prompt_lora_mapping.to(device="cpu")
697
+ ref_output = self.output.clone()
698
+
699
+ self.output.zero_()
700
+ op_type.bench_fn()(
701
+ **self.bench_fn_kwargs(op_type, expand_fn_add_inputs))
702
+
703
+ op_type.run_ref_group_gemm(
704
+ ref_output,
705
+ self.input,
706
+ self.lora_weights_lst,
707
+ seq_lens_cpu=seq_lens_cpu,
708
+ prompt_lora_mapping_cpu=prompt_lora_mapping_cpu,
709
+ scaling=1.0,
710
+ add_inputs=expand_fn_add_inputs)
711
+
712
+ rtol, atol = {
713
+ torch.float16: (6e-2, 6e-2),
714
+ torch.bfloat16: (6e-2, 6e-2),
715
+ torch.float32: (1e-2, 1e-2),
716
+ }[self.output.dtype]
717
+
718
+ return torch.allclose(ref_output, self.output, rtol=rtol, atol=atol)
719
+
720
+
721
+def bench_optype(ctx: BenchmarkContext,
722
+ arg_pool_size: int,
723
+ op_type: OpType,
724
+ cuda_graph_nops: Optional[int] = None,
725
+ expand_fn_add_inputs: Optional[bool] = None,
726
+ test_correctness: bool = False) -> TMeasurement:
727
+
728
+ assert arg_pool_size >= 1
729
+ if op_type.is_shrink_fn():
730
+ assert expand_fn_add_inputs is None
731
+ else:
732
+ assert expand_fn_add_inputs is not None
733
+
734
+ # BenchmarkContext -> BenchmarkTensors
735
+ bench_tensors : List[BenchmarkTensors] = \
736
+ [BenchmarkTensors.make(ctx, op_type) for _ in range(arg_pool_size)]
737
+ for bt in bench_tensors:
738
+ bt.sanity_check()
739
+
740
+ # Test correctness of our implementation.
741
+ if test_correctness:
742
+ assert all([
743
+ bt.test_correctness(op_type, expand_fn_add_inputs)
744
+ for bt in bench_tensors
745
+ ])
746
+
747
+ # BenchmarkTensors -> Dict (kwargs)
748
+ kwargs_list = [
749
+ bt.bench_fn_kwargs(op_type, add_inputs=expand_fn_add_inputs)
750
+ for bt in bench_tensors
751
+ ]
752
+
753
+ # Clear LoRA optimization hash-maps.
754
+ _LORA_A_PTR_DICT.clear()
755
+ _LORA_B_PTR_DICT.clear()
756
+ # Run bench function so that _LORA_A_PTR_DICT and _LORA_B_PTR_DICT are setup
757
+ for kwargs in kwargs_list:
758
+ op_type.bench_fn()(**kwargs)
759
+ torch.cuda.synchronize()
760
+
761
+ # Merge into a single kwargs and qualify arguments as ArgPool
762
+ kwargs = {k: ArgPool([]) for k in kwargs_list[0]}
763
+ for _kwargs in kwargs_list:
764
+ for k, v in _kwargs.items():
765
+ kwargs[k].values.append(v)
766
+
767
+ describe_args = (f"add_inputs={expand_fn_add_inputs}"
768
+ if expand_fn_add_inputs is not None else "")
769
+ description = (
770
+ f"{op_type.name}({describe_args}) ({bench_tensors[0].io_types()})")
771
+
772
+ cuda_graph_params = None
773
+ if cuda_graph_nops:
774
+ cuda_graph_params = CudaGraphBenchParams(cuda_graph_nops)
775
+ timer = None
776
+ with Bench(cuda_graph_params,
777
+ ctx.bench_label(), ctx.bench_sublabel(op_type), description,
778
+ op_type.bench_fn(), **kwargs) as bench:
779
+ timer = bench.run()
780
+ return timer
781
+
782
+
783
+def bench_torch_mm(ctx: BenchmarkContext,
784
+ arg_pool_size: int,
785
+ op_type: OpType,
786
+ cuda_graph_nops: Optional[int] = None) -> TMeasurement:
787
+ """
788
+ Benchmark basic torch.mm as a roofline.
789
+
790
+ When all the input tokens have the same LoRA ID, the LoRA kernels are just
791
+ a matmul. This torch.mm benchmark serves as a roofline for that case.
792
+
793
+ input op_type is used in determining the m, k, n dimensions for the matmul.
794
+ """
795
+
796
+ batch_size, hidden_size, lora_rank, seq_length, dtype = (ctx.batch_size,
797
+ ctx.hidden_size,
798
+ ctx.lora_rank,
799
+ ctx.seq_length,
800
+ ctx.dtype)
801
+
802
+ m, k, n = op_type.mkn(batch_size, seq_length, hidden_size, lora_rank)
803
+ # For a fairer comparison.
804
+ n = n * ctx.num_slices
805
+
806
+ # Get matmul input and output tensors for A x B = C
807
+ As, Bs, Cs = [], [], []
808
+ for _ in range(arg_pool_size):
809
+ As.append(torch.rand((m, k), dtype=dtype).to("cuda"))
810
+ Bs.append(torch.rand((n, k), dtype=dtype).to("cuda").t())
811
+ Cs.append(torch.rand((m, n), dtype=dtype).to("cuda"))
812
+
813
+ # Make torch.mm kwargs
814
+ mm_kwargs = {'input': ArgPool(As), 'mat2': ArgPool(Bs), 'out': ArgPool(Cs)}
815
+
816
+ description = (
817
+ f"single-lora roofline using torch.mm ({dtype_to_str(dtype)}"
818
+ f"x{dtype_to_str(dtype)}"
819
+ f"=>{dtype_to_str(dtype)})")
820
+ cuda_graph_params = None
821
+ if cuda_graph_nops:
822
+ cuda_graph_params = CudaGraphBenchParams(cuda_graph_nops)
823
+ with Bench(cuda_graph_params, ctx.bench_label(),
824
+ ctx.bench_sublabel(op_type), description, torch.mm,
825
+ **mm_kwargs) as bench:
826
+ return bench.run()
827
+
828
+
829
+# runner
830
+def use_cuda_graph_recommendation() -> str:
831
+ return """
832
+ Triton kernels have a significant launch overhead with
833
+ launched directly via python. This overhead is more noticeable
834
+ for small the problem sizes. For these cases, it is recommended
835
+ to use the script with `--cuda-graph-nops N` to benchmark N
836
+ consecutive invocations of the benchmarking operations from
837
+ inside a CUDA Graph. Note that the returned measurement is for N
838
+ invocations of the operation.
839
+ """
840
+
841
+
842
+def print_timers(timers: List[TMeasurement],
843
+ args: Optional[argparse.Namespace] = None):
844
+ compare = TBenchmark.Compare(timers)
845
+ compare.print()
846
+
847
+ if args and args.cuda_graph_nops:
848
+ print(
849
+ f"Note : The timings reported above is for {args.cuda_graph_nops} "
850
+ "consecutive invocations of the benchmarking functions. "
851
+ f"Please divide by {args.cuda_graph_nops} for single invocation "
852
+ "timings.")
853
+
854
+ print("Note on Comparison with torch.mm : The torch.mm numbers are "
855
+ "benchmark numbers of a simple matmul emulating the single lora "
856
+ "case. It is provided as a roofline for comparing our LoRA Kernel "
857
+ "implementations. It is expected that the LoRA kernels will be "
858
+ "slower than torch.mm in cases where num_loras is big. But for "
859
+ "small num_loras the goal should be to match the torch.mm numbers.")
860
+
861
+
862
+def run(args: argparse.Namespace, bench_ctxs: List[BenchmarkContext]):
863
+
864
+ if args.cuda_graph_nops is not None:
865
+ assert args.cuda_graph_nops > 0
866
+ print(f"Benchmarking {args.cuda_graph_nops} invocations inside a CUDA "
867
+ "Graph")
868
+ else:
869
+ print(f"CUDA Graphs not enabled.\n{use_cuda_graph_recommendation()}")
870
+
871
+ timers = []
872
+ for bench_ctx in bench_ctxs:
873
+ for seq_len in args.seq_lengths:
874
+ bench_ops: List[OpType] = []
875
+ if seq_len == 1:
876
+ # bench all decode ops
877
+ bench_ops = [op for op in args.op_types if op.is_decode_op()]
878
+ else:
879
+ # bench all prefill ops
880
+ bench_ops = [op for op in args.op_types if op.is_prefill_op()]
881
+
882
+ seq_len_timers = []
883
+ for bench_op in bench_ops:
884
+ for num_slices in bench_op.num_slices():
885
+ _ctx = bench_ctx.with_seq_length(seq_len).with_num_slices(
886
+ num_slices)
887
+ # Benchmark torch.mm as a roofline
888
+ seq_len_timers.append(
889
+ bench_torch_mm(_ctx, args.arg_pool_size, bench_op,
890
+ args.cuda_graph_nops))
891
+
892
+ # Benchmark bench_op
893
+ expand_fn_add_inputs = [
894
+ None
895
+ ] if bench_op.is_shrink_fn() else args.expand_fn_add_inputs
896
+ for add_input_arg in expand_fn_add_inputs:
897
+ seq_len_timers.append(
898
+ bench_optype(_ctx, args.arg_pool_size, bench_op,
899
+ args.cuda_graph_nops, add_input_arg,
900
+ args.test_correctness))
901
+
902
+ print_timers(seq_len_timers)
903
+ timers.extend(seq_len_timers)
904
+
905
+ # Result stdout dump
906
+ print("== All Results ====")
907
+ print_timers(timers, args)
908
+
909
+ if args.output_directory:
910
+ # Result file dump
911
+ od = Path(args.output_directory)
912
+ if not od.exists():
913
+ od.mkdir()
914
+
915
+ timestamp = int(time.time())
916
+ pkl_file = od / f"lora_bench-{timestamp}.pkl"
917
+ print(f"Writing benchmarks to {pkl_file}")
918
+ with open(pkl_file, "wb") as f:
919
+ pickle.dump(timers, f)
920
+
921
+
922
+def as_benchmark_contexts(hidden_sizes: List[int], lora_ranks: List[int],
923
+ args: argparse.Namespace) -> List[BenchmarkContext]:
924
+
925
+ ctxs: List[BenchmarkContext] = []
926
+ for batch_size, hidden_size, lora_rank, num_loras, sort_by_lora_id in product( # noqa
927
+ args.batch_sizes, list(hidden_sizes), lora_ranks, args.num_loras,
928
+ args.sort_by_lora_id):
929
+ ctxs.append(
930
+ BenchmarkContext(
931
+ batch_size=batch_size,
932
+ hidden_size=hidden_size,
933
+ lora_rank=lora_rank,
934
+ num_loras=num_loras,
935
+ num_active_loras=args.num_active_loras
936
+ if args.num_active_loras else num_loras,
937
+ # To be filled based on the OpType to benchmark
938
+ seq_length=None,
939
+ sort_by_lora_id=sort_by_lora_id,
940
+ dtype=args.dtype,
941
+ # To be filled based on the OpType to benchmark
942
+ num_slices=None))
943
+
944
+ return ctxs
945
+
946
+
947
+def run_list_bench(args: argparse.Namespace):
948
+ print(args)
949
+
950
+ print("List bench :\n"
951
+ f" Hidden Sizes {args.hidden_sizes}"
952
+ f" LoRA Ranks {args.lora_ranks}")
953
+
954
+ # Get all benchmarking contexts
955
+ bench_contexts: List[BenchmarkContext] = as_benchmark_contexts(
956
+ hidden_sizes=args.hidden_sizes, lora_ranks=args.lora_ranks, args=args)
957
+
958
+ run(args, bench_contexts)
959
+
960
+
961
+def run_range_bench(args: argparse.Namespace):
962
+ print(args)
963
+
964
+ hidden_sizes = list(
965
+ range(args.hidden_sizes_start, args.hidden_sizes_end + 1,
966
+ args.hidden_sizes_increment))
967
+ lora_ranks = list(
968
+ range(args.lora_ranks_start, args.lora_ranks_end + 1,
969
+ args.lora_ranks_increment))
970
+
971
+ print("Range bench :\n"
972
+ f" Hidden Sizes {hidden_sizes}"
973
+ f" LoRA Ranks {lora_ranks}")
974
+
975
+ # Get all benchmarking contexts
976
+ bench_contexts: List[BenchmarkContext] = as_benchmark_contexts(
977
+ hidden_sizes=hidden_sizes, lora_ranks=lora_ranks, args=args)
978
+
979
+ run(args, bench_contexts)
980
+
981
+
982
+def run_model_bench(args: argparse.Namespace):
983
+ print(args)
984
+
985
+ def hidden_sizes_from_model(model: str, tp_size: int) -> set[int]:
986
+ hidden_sizes = set()
987
+ for KN, tp_split_dim in WEIGHT_SHAPES[model]:
988
+ KN[tp_split_dim] = KN[tp_split_dim] // tp_size
989
+ hidden_sizes.add(KN[1])
990
+ return hidden_sizes
991
+
992
+ # Get all hidden sizes
993
+ hidden_sizes: set[int] = set()
994
+ for model_name, tp_size in product(args.models, args.tp_sizes):
995
+ hidden_sizes = hidden_sizes.union(
996
+ hidden_sizes_from_model(model_name, tp_size))
997
+
998
+ print("Model bench :\n"
999
+ f" Hidden Sizes {hidden_sizes}"
1000
+ f" LoRA Ranks {args.lora_ranks}")
1001
+
1002
+ # Get all benchmarking contexts
1003
+ bench_contexts: List[BenchmarkContext] = as_benchmark_contexts(
1004
+ hidden_sizes=hidden_sizes, lora_ranks=args.lora_ranks, args=args)
1005
+
1006
+ run(args, bench_contexts)
1007
+
1008
+
1009
+if __name__ == '__main__':
1010
+
1011
+ def to_torch_dtype(dt):
1012
+ if dt == "torch.float16":
1013
+ return torch.float16
1014
+ if dt == "torch.bfloat16":
1015
+ return torch.bfloat16
1016
+ raise ValueError("unsupported dtype")
1017
+
1018
+ def get_bool(s: str) -> bool:
1019
+ return s.lower() in ['true', '1']
1020
+
1021
+ def add_common_command_args(p: argparse.ArgumentParser):
1022
+ p.add_argument(
1023
+ "--dtype",
1024
+ type=to_torch_dtype,
1025
+ required=True,
1026
+ help="Available options are ['torch.float16', 'torch.bfloat16']")
1027
+
1028
+ p.add_argument(
1029
+ "--arg-pool-size",
1030
+ type=int,
1031
+ default=32,
1032
+ help="Run profiles with a pool of input/output/meta tensors instead"
1033
+ "of simply reusing the same tensors for all runs. A bigger arg-pool"
1034
+ "mitigates hardware caching effects during benchmarking.")
1035
+
1036
+ p.add_argument(
1037
+ "--cuda-graph-nops",
1038
+ type=int,
1039
+ help=("when set profiling is done using cudagraph, "
1040
+ "with the given number of operations in a graph."
1041
+ "Note that the measurement returned is the time "
1042
+ "taken for N consecutive executions of the benchmarking "
1043
+ "functions, where N is the value of this argument."))
1044
+ p.add_argument("--num-loras",
1045
+ nargs="+",
1046
+ type=int,
1047
+ default=DEFAULT_NUM_LORAS)
1048
+ p.add_argument("--num-active-loras",
1049
+ type=int,
1050
+ default=None,
1051
+ help="Active LoRAs. When None, all LoRAs are active")
1052
+ p.add_argument("--sort-by-lora-id",
1053
+ nargs="+",
1054
+ type=get_bool,
1055
+ default=DEFAULT_SORT_BY_LORA_IDS)
1056
+ p.add_argument("--op-types",
1057
+ nargs="+",
1058
+ type=OpType.from_str,
1059
+ default=list(OpType))
1060
+ p.add_argument('--seq-lengths',
1061
+ nargs="+",
1062
+ type=int,
1063
+ default=DEFAULT_SEQ_LENGTHS)
1064
+ p.add_argument("--batch-sizes",
1065
+ nargs="+",
1066
+ type=int,
1067
+ default=DEFAULT_BATCH_SIZES)
1068
+ p.add_argument("--expand-fn-add-inputs",
1069
+ nargs="+",
1070
+ type=get_bool,
1071
+ default=DEFAULT_EXPAND_FN_ADD_INPUTS)
1072
+ p.add_argument(
1073
+ '-o',
1074
+ '--output-directory',
1075
+ type=str,
1076
+ help=("Output directory to store a the list of benchmarking"
1077
+ "TMeasurement objects as a pickle file"))
1078
+
1079
+ p.add_argument(
1080
+ "--test-correctness",
1081
+ action='store_true',
1082
+ help=("When enabled, the benchmarking functions are tested"
1083
+ "for correctness before the actual benchmarking"))
1084
+
1085
+ parser = FlexibleArgumentParser(
1086
+ description=f"""
1087
+Benchmark LoRA kernels:
1088
+ {use_cuda_graph_recommendation()}
1089
+
1090
+ list_bench example:
1091
+ python3 benchmarks/kernels/benchmark_lora.py list_bench --arg-pool-size 32 --batch-sizes 1 16 32 --dtype torch.float16 --hidden-sizes 2048 --lora-ranks 16 --num-loras 1 4 --op-types bgmv_shrink bgmv_expand sgmv_shrink sgmv_expand bgmv_expand_slice --seq-lengths 1 16 --sort-by-lora-id 1 --cuda-graph-nops 32
1092
+
1093
+ model_bench example:
1094
+ python3 benchmarks/kernels/benchmark_lora.py model_bench --models meta-llama/Llama-3-8b --arg-pool-size 32 --batch-sizes 1 16 32 --dtype torch.float16 --lora-ranks 16 --num-loras 1 4 --op-types bgmv_shrink bgmv_expand sgmv_shrink sgmv_expand bgmv_expand_slice --seq-lengths 1 16 --sort-by-lora-id 1 --cuda-graph-nops 32
1095
+
1096
+ range_bench example:
1097
+ python3 benchmarks/kernels/benchmark_lora.py range_bench --arg-pool-size 32 --batch-sizes 1 16 32 --dtype torch.float16 --num-loras 1 4 --op-types bgmv_shrink bgmv_expand sgmv_shrink sgmv_expand bgmv_expand_slice --seq-lengths 1 16 --sort-by-lora-id 1 --cuda-graph-nops 32 --hidden-sizes-start 1024 --hidden-sizes-end 4096 --hidden-sizes-increment 1024 --lora-ranks-start 8 --lora-ranks-end 24 --lora-ranks-increment 8
1098
+ """, # noqa: E501
1099
+ formatter_class=argparse.RawTextHelpFormatter)
1100
+
1101
+ subparsers = parser.add_subparsers(dest="cmd", required=True)
1102
+
1103
+ list_parser = subparsers.add_parser("list_bench")
1104
+ list_parser.add_argument("--hidden-sizes",
1105
+ nargs="+",
1106
+ type=int,
1107
+ default=DEFAULT_HIDDEN_SIZES)
1108
+ list_parser.add_argument("--lora-ranks",
1109
+ nargs="+",
1110
+ type=int,
1111
+ default=DEFAULT_LORA_RANKS)
1112
+ add_common_command_args(list_parser)
1113
+ list_parser.set_defaults(func=run_list_bench)
1114
+
1115
+ range_parser = subparsers.add_parser("range_bench")
1116
+ range_parser.add_argument("--hidden-sizes-start", type=int, required=True)
1117
+ range_parser.add_argument("--hidden-sizes-end", type=int, required=True)
1118
+ range_parser.add_argument("--hidden-sizes-increment",
1119
+ type=int,
1120
+ required=True)
1121
+ range_parser.add_argument("--lora-ranks-start", type=int, required=True)
1122
+ range_parser.add_argument("--lora-ranks-end", type=int, required=True)
1123
+ range_parser.add_argument("--lora-ranks-increment",
1124
+ type=int,
1125
+ required=True)
1126
+ add_common_command_args(range_parser)
1127
+ range_parser.set_defaults(func=run_range_bench)
1128
+
1129
+ model_parser = subparsers.add_parser("model_bench")
1130
+ model_parser.add_argument("--models",
1131
+ nargs="+",
1132
+ type=str,
1133
+ default=DEFAULT_MODELS,
1134
+ choices=WEIGHT_SHAPES.keys())
1135
+ model_parser.add_argument("--tp-sizes",
1136
+ nargs="+",
1137
+ type=int,
1138
+ default=DEFAULT_TP_SIZES)
1139
+ model_parser.add_argument("--lora-ranks",
1140
+ nargs="+",
1141
+ type=int,
1142
+ default=DEFAULT_LORA_RANKS)
1143
+ add_common_command_args(model_parser)
1144
+ model_parser.set_defaults(func=run_model_bench)
1145
+
1146
+ args = parser.parse_args()
1147
+ args.func(args)
benchmarks/kernels/utils.py
new
+210
@@ -0,0 +1,210 @@
1
+import dataclasses
2
+from typing import Any, Callable, Iterable, Optional
3
+
4
+import torch
5
+import torch.utils.benchmark as TBenchmark
6
+from torch.utils.benchmark import Measurement as TMeasurement
7
+
8
+
9
+@dataclasses.dataclass
10
+class CudaGraphBenchParams:
11
+ num_ops_in_cuda_graph: int
12
+
13
+
14
+@dataclasses.dataclass
15
+class ArgPool:
16
+ """
17
+ When some argument of the benchmarking function is annotated with this type,
18
+ the benchmarking class (BenchMM) will collapse the argument to a pick a
19
+ single value from the given list of values, during function invocation.
20
+ For every invocation during a benchmarking run, it will choose a
21
+ different value from the list.
22
+ """
23
+ values: Iterable[Any]
24
+
25
+ def __getitem__(self, index):
26
+ return self.values[index]
27
+
28
+
29
+class Bench:
30
+
31
+ class ArgsIterator:
32
+
33
+ def __init__(self, args_list, kwargs_list):
34
+ assert len(args_list) == len(kwargs_list)
35
+ self.args_list = args_list
36
+ self.kwargs_list = kwargs_list
37
+ self.n = len(self.args_list)
38
+ self.idx = 0
39
+
40
+ def __next__(self):
41
+ while True:
42
+ yield (self.args_list[self.idx], self.kwargs_list[self.idx])
43
+ self.idx += 1
44
+ self.idx = self.idx % self.n
45
+
46
+ def reset(self):
47
+ self.idx = 0
48
+
49
+ @property
50
+ def n_args(self):
51
+ return self.n
52
+
53
+ def __init__(self, cuda_graph_params: Optional[CudaGraphBenchParams],
54
+ label: str, sub_label: str, description: str, fn: Callable,
55
+ *args, **kwargs):
56
+
57
+ self.cuda_graph_params = cuda_graph_params
58
+ self.use_cuda_graph = self.cuda_graph_params is not None
59
+ self.label = label
60
+ self.sub_label = sub_label
61
+ self.description = description
62
+ self.fn = fn
63
+
64
+ # Process args
65
+ self._args = args
66
+ self._kwargs = kwargs
67
+ self.args_list, self.kwargs_list = self.collapse_argpool(
68
+ *args, **kwargs)
69
+ self.args_iterator = self.ArgsIterator(self.args_list,
70
+ self.kwargs_list)
71
+
72
+ # Cudagraph runner
73
+ self.g = None
74
+ if self.use_cuda_graph:
75
+ self.g = self.get_cuda_graph_runner()
76
+
77
+ # benchmark run params
78
+ self.min_run_time = 1
79
+
80
+ def collapse_argpool(self, *args, **kwargs):
81
+ argpool_args = [arg for arg in args if isinstance(arg, ArgPool)] + [
82
+ arg for arg in kwargs.values() if isinstance(arg, ArgPool)
83
+ ]
84
+ if len(argpool_args) == 0:
85
+ return [args], [kwargs]
86
+
87
+ # Make sure all argpools are of the same size
88
+ argpool_size = len(argpool_args[0].values)
89
+ assert all([argpool_size == len(arg.values) for arg in argpool_args])
90
+
91
+ # create copies of the args
92
+ args_list = []
93
+ kwargs_list = []
94
+ for _ in range(argpool_size):
95
+ args_list.append(args)
96
+ kwargs_list.append(kwargs.copy())
97
+
98
+ for i in range(argpool_size):
99
+ # collapse args; Just pick the ith value
100
+ args_list[i] = tuple([
101
+ arg[i] if isinstance(arg, ArgPool) else arg
102
+ for arg in args_list[i]
103
+ ])
104
+
105
+ # collapse kwargs
106
+ kwargs_i = kwargs_list[i]
107
+ arg_pool_keys = [
108
+ k for k, v in kwargs_i.items() if isinstance(v, ArgPool)
109
+ ]
110
+ for k in arg_pool_keys:
111
+ # again just pick the ith value
112
+ kwargs_i[k] = kwargs_i[k][i]
113
+ kwargs_list[i] = kwargs_i
114
+
115
+ return args_list, kwargs_list
116
+
117
+ def get_cuda_graph_runner(self):
118
+ assert self.use_cuda_graph
119
+ assert self.args_iterator is not None
120
+
121
+ num_graph_ops = self.cuda_graph_params.num_ops_in_cuda_graph
122
+
123
+ # warmup
124
+ args_it = self.args_iterator.__next__()
125
+ for _ in range(2):
126
+ args, kwargs = next(args_it)
127
+ self.fn(*args, **kwargs)
128
+
129
+ self.args_iterator.reset()
130
+ args_it = self.args_iterator.__next__()
131
+ stream = torch.cuda.Stream()
132
+ with torch.cuda.stream(stream):
133
+ g = torch.cuda.CUDAGraph()
134
+ with torch.cuda.graph(g):
135
+ for _ in range(num_graph_ops):
136
+ args, kwargs = next(args_it)
137
+ self.fn(*args, **kwargs)
138
+ return g
139
+
140
+ def run_cudagrah(self) -> TMeasurement:
141
+ assert self.use_cuda_graph
142
+ globals = {'g': self.g}
143
+
144
+ return TBenchmark.Timer(
145
+ stmt="g.replay()",
146
+ globals=globals,
147
+ label=(
148
+ f"{self.label}"
149
+ f" | cugraph {self.cuda_graph_params.num_ops_in_cuda_graph} ops"
150
+ ),
151
+ sub_label=self.sub_label,
152
+ description=self.description,
153
+ ).blocked_autorange(min_run_time=self.min_run_time)
154
+
155
+ def run_eager(self) -> TMeasurement:
156
+ setup = None
157
+ stmt = None
158
+ globals = None
159
+
160
+ has_arg_pool = self.args_iterator.n_args > 1
161
+ if has_arg_pool:
162
+ setup = '''
163
+ args_iterator.reset()
164
+ args_it = args_iterator.__next__()
165
+ '''
166
+ stmt = '''
167
+ args, kwargs = next(args_it)
168
+ fn(*args, **kwargs)
169
+ '''
170
+ globals = {'fn': self.fn, 'args_iterator': self.args_iterator}
171
+ else:
172
+ # no arg pool. Just use the args and kwargs directly
173
+ self.args_iterator.reset()
174
+ args_it = self.args_iterator.__next__()
175
+ args, kwargs = next(args_it)
176
+
177
+ setup = ""
178
+ stmt = '''
179
+ fn(*args, **kwargs)
180
+ '''
181
+ globals = {'fn': self.fn, 'args': args, 'kwargs': kwargs}
182
+
183
+ return TBenchmark.Timer(
184
+ stmt=stmt,
185
+ setup=setup,
186
+ globals=globals,
187
+ label=self.label,
188
+ sub_label=self.sub_label,
189
+ description=self.description,
190
+ ).blocked_autorange(min_run_time=self.min_run_time)
191
+
192
+ def run(self) -> TMeasurement:
193
+ timer = None
194
+ if self.use_cuda_graph: # noqa SIM108
195
+ timer = self.run_cudagrah()
196
+ else:
197
+ timer = self.run_eager()
198
+ if not timer.meets_confidence() or timer.has_warnings:
199
+ print("Doesn't meet confidence - re-running bench ...")
200
+ return self.run()
201
+ return timer
202
+
203
+ def __enter__(self):
204
+ return self
205
+
206
+ def __exit__(self, exc_type, exc_value, traceback):
207
+ if exc_type:
208
+ print(f"exc type {exc_type}")
209
+ print(f"exc value {exc_value}")
210
+ print(f"exc traceback {traceback}")