A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 15:45 UTC
[Kernel][LoRA] Add assertion for punica sgmv kernels (#7585)
Jee Jee Li committed
Sep 24, 2024 at 02:57 UTC
9b0e3ec970f6a19427be358848a2ed663fd735e1
8 files changed
+64
-38
tests/lora/test_punica_sizes.py
+5
@@ -169,6 +169,7 @@ def test_punica_sgmv(
169
device,
170
)
171
max_seq_length = seq_len_tensor.max()
172
+ token_nums = seq_len_tensor.sum().item()
173
if isinstance(max_seq_length, tuple):
174
max_seq_length = max_seq_length[0].item()
175
else:
@@ -183,6 +184,7 @@ def test_punica_sgmv(
184
lora_indices_tensor,
185
batches,
186
max_seq_length,
187
+ token_nums,
188
scaling,
189
)
190
else:
@@ -195,6 +197,7 @@ def test_punica_sgmv(
197
lora_indices_tensor,
198
batches,
199
max_seq_length,
200
+ token_nums,
201
add_inputs=True,
202
)
203
ref_torch_groupgemm(
@@ -347,6 +350,7 @@ def test_punica_expand_nslices(
350
device,
351
)
352
max_seq_length = seq_len_tensor.max()
353
+ token_nums = seq_len_tensor.sum().item()
354
if isinstance(max_seq_length, tuple):
355
max_seq_length = max_seq_length[0].item()
356
else:
@@ -364,6 +368,7 @@ def test_punica_expand_nslices(
368
lora_indices_tensor,
369
batches,
370
max_seq_length,
371
+ token_nums,
372
slice_offset,
373
hidden_size,
374
add_inputs=True,
tests/lora/test_punica_variation.py
+5
@@ -84,6 +84,7 @@ def test_punica_sgmv(
84
device,
85
)
86
max_seq_length = seq_len_tensor.max()
87
+ token_nums = seq_len_tensor.sum().item()
88
if isinstance(max_seq_length, tuple):
89
max_seq_length = max_seq_length[0].item()
90
else:
@@ -98,6 +99,7 @@ def test_punica_sgmv(
99
lora_indices_tensor,
100
batches,
101
max_seq_length,
102
+ token_nums,
103
scaling,
104
)
105
else:
@@ -110,6 +112,7 @@ def test_punica_sgmv(
112
lora_indices_tensor,
113
batches,
114
max_seq_length,
115
+ token_nums,
116
add_inputs=True,
117
)
118
ref_torch_groupgemm(
@@ -262,6 +265,7 @@ def test_punica_expand_nslices(
265
device,
266
)
267
max_seq_length = seq_len_tensor.max()
268
+ token_nums = seq_len_tensor.sum().item()
269
if isinstance(max_seq_length, tuple):
270
max_seq_length = max_seq_length[0].item()
271
else:
@@ -279,6 +283,7 @@ def test_punica_expand_nslices(
283
lora_indices_tensor,
284
batches,
285
max_seq_length,
286
+ token_nums,
287
slice_offset,
288
hidden_size,
289
add_inputs=True,
vllm/lora/ops/bgmv_expand.py
+1
-1
@@ -100,7 +100,7 @@ def _bgmv_expand(
100
corresponding to each batch, An index of -1 means no lora should be
101
applied.
102
batches (int): batch size
103
- add_inputs (bool, optional): Defaults to False. adds the final lora
103
+ add_inputs (bool, optional): Defaults to False, adds the final lora
104
results to the output.
105
"""
106
assert inputs.dtype in [torch.float16, torch.bfloat16, torch.float32]
vllm/lora/ops/bgmv_expand_slice.py
+1
-1
@@ -104,7 +104,7 @@ def _bgmv_expand_slice(
104
lora_indices_tensor (torch.Tensor): (batch_size,). The LoRA index
105
corresponding to each batch, An index of -1 means no lora should be
106
applied.
107
- slice_offst (int): output_tensor's offst
107
+ slice_offset (int): output_tensor's offset
108
slice_size (int): current output_tensor's size
109
batches (int): batch size
110
add_inputs (bool, optional): Defaults to False.
vllm/lora/ops/sgmv_expand.py
+10
-6
@@ -106,6 +106,7 @@ def _sgmv_expand(
106
lora_indices_tensor: torch.Tensor,
107
batches: int,
108
max_seq_length: int,
109
+ token_nums: int,
110
add_inputs: bool = False,
111
) -> None:
112
"""
@@ -115,17 +116,19 @@ def _sgmv_expand(
116
output_tensor (torch.Tensor): output tensor
117
b_seq_start_loc (torch.Tensor): (batch_size,). The cumulative
118
sequence lengths of the sequences in the batch, used to index
118
- into sequence. E.g.,if the sequence length is [4, 6], it is
119
+ into sequence. E.g., if the sequence length is [4, 6], it is
120
[0, 4, 10].
120
- seq_len_tensor (torch.Tensor): (batch_size,). record the sequence
121
- length of the sequences in the batch
121
+ seq_len_tensor (torch.Tensor): (batch_size,). Record the sequence
122
+ length of the sequences in the batch.
123
lora_indices_tensor (torch.Tensor): (batch_size,). The LoRA index
124
corresponding to each batch. An index of -1 means no lora should be
125
applied.
126
batches (int): batch size
126
- max_seq_length (int): The max sequence lengths of the sequences
127
- in the batch
128
- add_inputs (bool, optional): Defaults to False. adds the final lora
127
+ max_seq_length (int): The max sequence lengths of the sequences in the
128
+ batch.
129
+ token_nums (int): The token numbers in the batch. Used to verify if the
130
+ token numbers in the inputs matches the one in the metadata.
131
+ add_inputs (bool, optional): Defaults to False, adds the final lora
132
results to the output.
133
"""
134
@@ -134,6 +137,7 @@ def _sgmv_expand(
137
torch.float16,
138
torch.bfloat16,
139
]
140
+ assert inputs.size(0) == token_nums
141
assert inputs.size(1) == lora_b_weights.size(-1)
142
assert b_seq_start_loc.size(0) == batches
143
assert lora_indices_tensor.size(0) == batches
vllm/lora/ops/sgmv_expand_slice.py
+11
-7
@@ -112,6 +112,7 @@ def _sgmv_expand_slice(
112
lora_indices_tensor: torch.Tensor,
113
batches: int,
114
max_seq_length: int,
115
+ token_nums: int,
116
slice_offset: int,
117
slice_size: int,
118
add_inputs: bool = False,
@@ -124,20 +125,22 @@ def _sgmv_expand_slice(
125
output_tensor (torch.Tensor): output tensor
126
b_seq_start_loc (torch.Tensor): (batch_size,). The cumulative
127
sequence lengths of the sequences in the batch, used to index
127
- into sequence. E.g.,if the sequence length is [4, 6], it is
128
+ into sequence. E.g., if the sequence length is [4, 6], it is
129
[0, 4, 10].
129
- seq_len_tensor (torch.Tensor): (batch_size,). record the sequence
130
- length of the sequences in the batch
130
+ seq_len_tensor (torch.Tensor): (batch_size,). Record the sequence
131
+ length of the sequences in the batch
132
lora_indices_tensor (torch.Tensor): (batch_size,). The LoRA index
133
corresponding to each batch. An index of -1 means no lora should be
134
applied.
135
batches (int): batch size
135
- max_seq_length (int): The max sequence lengths of the sequences
136
+ max_seq_length (int): The max sequence lengths of the sequences
137
in the batch
137
- slice_offst (int): output_tensor's offst
138
+ token_nums (int): The token numbers in the batch. Used to verify if the
139
+ token numbers in the inputs matches the one in the metadata.
140
+ slice_offset (int): output_tensor's offset
141
slice_size (int): current output_tensor's size
139
- add_inputs (bool, optional): Defaults to False. adds the final lora
140
- results to the output..
142
+ add_inputs (bool, optional): Defaults to False, adds the final lora
143
+ results to the output.
144
"""
145
146
assert inputs.dtype in [torch.float16, torch.bfloat16, torch.float32]
@@ -145,6 +148,7 @@ def _sgmv_expand_slice(
148
torch.float16,
149
torch.bfloat16,
150
]
151
+ assert inputs.size(0) == token_nums
152
assert inputs.size(1) == lora_b_weights.size(-1)
153
assert b_seq_start_loc.size(0) == batches
154
assert lora_indices_tensor.size(0) == batches
vllm/lora/ops/sgmv_shrink.py
+10
-6
@@ -110,6 +110,7 @@ def _sgmv_shrink(
110
lora_indices_tensor: torch.Tensor,
111
batches: int,
112
max_seq_length: int,
113
+ token_nums: int,
114
scaling: float,
115
) -> None:
116
"""
@@ -120,17 +121,19 @@ def _sgmv_shrink(
121
output_tensor (torch.Tensor): output tensor
122
b_seq_start_loc (torch.Tensor): (batch_size,). The cumulative
123
sequence lengths of the sequences in the batch, used to index
123
- into sequence. E.g.,if the sequence length is [4, 6], it is
124
+ into sequence. E.g., if the sequence length is [4, 6], it is
125
[0, 4].
125
- seq_len_tensor (torch.Tensor): (batch_size,). record the sequence
126
- length of the sequences in the batch
126
+ seq_len_tensor (torch.Tensor): (batch_size,). Record the sequence
127
+ length of the sequences in the batch.
128
lora_indices_tensor (torch.Tensor): (batch_size,). The LoRA index
129
corresponding to each batch. An index of -1 means no lora should be
130
applied.
131
batches (int): batch size
131
- max_seq_length (int): The max sequence lengths of the sequences
132
- in the batch
133
- scaling (float): Scaling factor.
132
+ max_seq_length (int): The max sequence lengths of the sequences in the
133
+ batch.
134
+ token_nums (int): The token numbers in the batch. Used to verify if the
135
+ token numbers in the inputs matches the one in the metadata.
136
+ scaling (float): Scaling factor.
137
"""
138
assert inputs.dtype == lora_a_weights.dtype
139
assert inputs.dtype in [torch.float16, torch.bfloat16]
@@ -138,6 +141,7 @@ def _sgmv_shrink(
141
torch.float16,
142
torch.bfloat16,
143
]
144
+ assert inputs.size(0) == token_nums
145
assert inputs.size(1) == lora_a_weights.size(-1)
146
assert b_seq_start_loc.size(0) == batches
147
assert lora_indices_tensor.size(0) == batches
vllm/lora/punica.py
+21
-17
@@ -27,7 +27,7 @@ if TYPE_CHECKING:
27
28
def compute_meta(
29
token_lora_tensor: torch.Tensor
30
-) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, int, int, bool]:
30
+) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, int, int, int, bool]:
31
"""
32
Get the information required for the sgmv kernel. With the features:
33
1. If consecutive requests in the batch use the same LoRA, this function
@@ -43,7 +43,7 @@ def compute_meta(
43
b_seq_start_tensor = torch.zeros_like(seq_length_tensor)
44
b_seq_start_tensor[1:].copy_(cum_result[:-1])
45
max_length = seq_length_tensor.max().item()
46
-
46
+ token_nums = seq_length_tensor.sum().item()
47
batch_size = lora_indices_tensor.size(0)
48
no_lora = False
49
# -1 means no lora should be applied. Use `no_lora` to determine whether
@@ -52,7 +52,7 @@ def compute_meta(
52
if batch_size == 1 and lora_indices_tensor == -1:
53
no_lora = True
54
return (b_seq_start_tensor, seq_length_tensor, lora_indices_tensor,
55
- batch_size, max_length, no_lora)
55
+ batch_size, max_length, token_nums, no_lora)
56
57
58
# TODO see if this can be vectorized
@@ -178,7 +178,7 @@ def convert_mapping(
178
class PunicaWrapper:
179
"""
180
PunicaWrapper is designed to manage and provide metadata for the punica
181
- kernel. The main function is to maintain the state information for
181
+ kernel. The main function is to maintain the state information for
182
Multi-LoRA, and to provide the interface for the punica kernel.
183
"""
184
@@ -216,6 +216,7 @@ class PunicaWrapper:
216
dtype=torch.long,
217
device=device)
218
self.max_length: int = 0
219
+ self.token_nums: int = 0
220
self.batch_size: int = -1
221
self.is_prefill = False
222
self.no_lora = False
@@ -276,13 +277,13 @@ class PunicaWrapper:
277
long_lora_offsets_tensor)
278
else:
279
self._long_lora_indices.zero_()
279
-
280
self.indices_len[:] = indices_len
281
282
def _update_prefill_metada(self, token_lora_tensor: torch.Tensor) -> None:
283
284
(b_seq_start_tensor, seq_length_tensor, lora_indices_tensor,
285
- batch_size, max_length, no_lora) = compute_meta(token_lora_tensor)
285
+ batch_size, max_length, token_nums,
286
+ no_lora) = compute_meta(token_lora_tensor)
287
288
self._seq_start_locs[:b_seq_start_tensor.shape[0]].copy_(
289
b_seq_start_tensor)
@@ -291,25 +292,28 @@ class PunicaWrapper:
292
lora_indices_tensor)
293
self.batch_size = batch_size
294
self.max_length = max_length
295
+ self.token_nums = token_nums
296
self.no_lora = no_lora
297
298
@property
299
def prefill_metadata(
298
- self) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, int, int]:
300
+ self
301
+ ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, int, int, int]:
302
"""
303
This property provides a convenient way to access the necessary
304
metadata for prefill-related kernel computations.
302
- 1. seq_start_locs: Tensor of sequence start positions
303
- 2. seq_lengths: Tensor of sequence lengths
305
+ 1. seq_start_locs: Tensor of sequence start positions.
306
+ 2. seq_lengths: Tensor of sequence lengths.
307
3. lora_indices_per_batch: Tensor of lora indices, and an index of
308
-1 means no lora should be applied.
306
- 4. batch_size: batch size after clustering identical lora indices
307
- 5. max_length: The maximum sequence length in the batch
309
+ 4. batch_size: Batch size after clustering identical lora indices.
310
+ 5. max_length: The maximum sequence length in the batch.
311
+ 6. token_nums: The token numbers in the batch.
312
"""
313
return (self._seq_start_locs[:self.batch_size],
314
self._seq_lengths[:self.batch_size],
315
self._lora_indices_per_batch[:self.batch_size],
312
- self.batch_size, self.max_length)
316
+ self.batch_size, self.max_length, self.token_nums)
317
318
@property
319
def token_lora_indices(self) -> torch.Tensor:
@@ -324,7 +328,7 @@ class PunicaWrapper:
328
def sampler_indices(self) -> torch.Tensor:
329
"""
330
This property is used to access the lora indices specifically for
327
- LogitsProcessorWithLoRA
331
+ LogitsProcessorWithLoRA.
332
"""
333
sampler_indices_len = self.indices_len[1]
334
return self._sampler_indices[:sampler_indices_len]
@@ -332,7 +336,7 @@ class PunicaWrapper:
336
@property
337
def sampler_indices_padded(self) -> torch.Tensor:
338
"""
335
- This property provides access to padded sampler indices
339
+ This property provides access to padded sampler indices.
340
"""
341
indices_padded_len = self.indices_len[2]
342
return self._sampler_indices_padded[:indices_padded_len]
@@ -341,7 +345,7 @@ class PunicaWrapper:
345
def embeddings_indices(self) -> torch.Tensor:
346
"""
347
This property provides access to the indices used for lora embeddings,
344
- specifically for VocabParallelEmbeddingWithLoRA
348
+ specifically for VocabParallelEmbeddingWithLoRA.
349
"""
350
embeddings_indices_len = self.indices_len[3]
351
return self._embeddings_indices[:, :embeddings_indices_len]
@@ -350,7 +354,7 @@ class PunicaWrapper:
354
def long_lora_indices(self) -> torch.Tensor:
355
"""
356
This property provides access to the indices used for long context
353
- lora, specifically for LinearScalingRotaryEmbeddingWithLora
357
+ lora, specifically for LinearScalingRotaryEmbeddingWithLora.
358
"""
359
long_lora_len = self.indices_len[4]
360
return self._long_lora_indices[:long_lora_len]
@@ -524,7 +528,7 @@ class PunicaWrapper:
528
scale (float): Scaling factor.
529
y_offset (Optional[int], optional): Offset to apply to the starting
530
column of y.
527
- y_slice_size (Optional[int], optional): Size of the y column slice..
531
+ y_slice_size (Optional[int], optional): Size of the y column slice.
532
buffer (Optional[torch.Tensor], optional): Defaults to None.
533
"""
534
y_org = y