@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 9b0e3ec970
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 15:45 UTC

[Kernel][LoRA] Add assertion for punica sgmv kernels (#7585)

Jee Jee Li committed Sep 24, 2024 at 02:57 UTC 9b0e3ec970f6a19427be358848a2ed663fd735e1
8 files changed +64 -38
tests/lora/test_punica_sizes.py
+5
@@ -169,6 +169,7 @@ def test_punica_sgmv(
169 device,
170 )
171 max_seq_length = seq_len_tensor.max()
172 + token_nums = seq_len_tensor.sum().item()
173 if isinstance(max_seq_length, tuple):
174 max_seq_length = max_seq_length[0].item()
175 else:
@@ -183,6 +184,7 @@ def test_punica_sgmv(
184 lora_indices_tensor,
185 batches,
186 max_seq_length,
187 + token_nums,
188 scaling,
189 )
190 else:
@@ -195,6 +197,7 @@ def test_punica_sgmv(
197 lora_indices_tensor,
198 batches,
199 max_seq_length,
200 + token_nums,
201 add_inputs=True,
202 )
203 ref_torch_groupgemm(
@@ -347,6 +350,7 @@ def test_punica_expand_nslices(
350 device,
351 )
352 max_seq_length = seq_len_tensor.max()
353 + token_nums = seq_len_tensor.sum().item()
354 if isinstance(max_seq_length, tuple):
355 max_seq_length = max_seq_length[0].item()
356 else:
@@ -364,6 +368,7 @@ def test_punica_expand_nslices(
368 lora_indices_tensor,
369 batches,
370 max_seq_length,
371 + token_nums,
372 slice_offset,
373 hidden_size,
374 add_inputs=True,
tests/lora/test_punica_variation.py
+5
@@ -84,6 +84,7 @@ def test_punica_sgmv(
84 device,
85 )
86 max_seq_length = seq_len_tensor.max()
87 + token_nums = seq_len_tensor.sum().item()
88 if isinstance(max_seq_length, tuple):
89 max_seq_length = max_seq_length[0].item()
90 else:
@@ -98,6 +99,7 @@ def test_punica_sgmv(
99 lora_indices_tensor,
100 batches,
101 max_seq_length,
102 + token_nums,
103 scaling,
104 )
105 else:
@@ -110,6 +112,7 @@ def test_punica_sgmv(
112 lora_indices_tensor,
113 batches,
114 max_seq_length,
115 + token_nums,
116 add_inputs=True,
117 )
118 ref_torch_groupgemm(
@@ -262,6 +265,7 @@ def test_punica_expand_nslices(
265 device,
266 )
267 max_seq_length = seq_len_tensor.max()
268 + token_nums = seq_len_tensor.sum().item()
269 if isinstance(max_seq_length, tuple):
270 max_seq_length = max_seq_length[0].item()
271 else:
@@ -279,6 +283,7 @@ def test_punica_expand_nslices(
283 lora_indices_tensor,
284 batches,
285 max_seq_length,
286 + token_nums,
287 slice_offset,
288 hidden_size,
289 add_inputs=True,
vllm/lora/ops/bgmv_expand.py
+1 -1
@@ -100,7 +100,7 @@ def _bgmv_expand(
100 corresponding to each batch, An index of -1 means no lora should be
101 applied.
102 batches (int): batch size
103 - add_inputs (bool, optional): Defaults to False. adds the final lora
103 + add_inputs (bool, optional): Defaults to False, adds the final lora
104 results to the output.
105 """
106 assert inputs.dtype in [torch.float16, torch.bfloat16, torch.float32]
vllm/lora/ops/bgmv_expand_slice.py
+1 -1
@@ -104,7 +104,7 @@ def _bgmv_expand_slice(
104 lora_indices_tensor (torch.Tensor): (batch_size,). The LoRA index
105 corresponding to each batch, An index of -1 means no lora should be
106 applied.
107 - slice_offst (int): output_tensor's offst
107 + slice_offset (int): output_tensor's offset
108 slice_size (int): current output_tensor's size
109 batches (int): batch size
110 add_inputs (bool, optional): Defaults to False.
vllm/lora/ops/sgmv_expand.py
+10 -6
@@ -106,6 +106,7 @@ def _sgmv_expand(
106 lora_indices_tensor: torch.Tensor,
107 batches: int,
108 max_seq_length: int,
109 + token_nums: int,
110 add_inputs: bool = False,
111 ) -> None:
112 """
@@ -115,17 +116,19 @@ def _sgmv_expand(
116 output_tensor (torch.Tensor): output tensor
117 b_seq_start_loc (torch.Tensor): (batch_size,). The cumulative
118 sequence lengths of the sequences in the batch, used to index
118 - into sequence. E.g.,if the sequence length is [4, 6], it is
119 + into sequence. E.g., if the sequence length is [4, 6], it is
120 [0, 4, 10].
120 - seq_len_tensor (torch.Tensor): (batch_size,). record the sequence
121 - length of the sequences in the batch
121 + seq_len_tensor (torch.Tensor): (batch_size,). Record the sequence
122 + length of the sequences in the batch.
123 lora_indices_tensor (torch.Tensor): (batch_size,). The LoRA index
124 corresponding to each batch. An index of -1 means no lora should be
125 applied.
126 batches (int): batch size
126 - max_seq_length (int): The max sequence lengths of the sequences
127 - in the batch
128 - add_inputs (bool, optional): Defaults to False. adds the final lora
127 + max_seq_length (int): The max sequence lengths of the sequences in the
128 + batch.
129 + token_nums (int): The token numbers in the batch. Used to verify if the
130 + token numbers in the inputs matches the one in the metadata.
131 + add_inputs (bool, optional): Defaults to False, adds the final lora
132 results to the output.
133 """
134
@@ -134,6 +137,7 @@ def _sgmv_expand(
137 torch.float16,
138 torch.bfloat16,
139 ]
140 + assert inputs.size(0) == token_nums
141 assert inputs.size(1) == lora_b_weights.size(-1)
142 assert b_seq_start_loc.size(0) == batches
143 assert lora_indices_tensor.size(0) == batches
vllm/lora/ops/sgmv_expand_slice.py
+11 -7
@@ -112,6 +112,7 @@ def _sgmv_expand_slice(
112 lora_indices_tensor: torch.Tensor,
113 batches: int,
114 max_seq_length: int,
115 + token_nums: int,
116 slice_offset: int,
117 slice_size: int,
118 add_inputs: bool = False,
@@ -124,20 +125,22 @@ def _sgmv_expand_slice(
125 output_tensor (torch.Tensor): output tensor
126 b_seq_start_loc (torch.Tensor): (batch_size,). The cumulative
127 sequence lengths of the sequences in the batch, used to index
127 - into sequence. E.g.,if the sequence length is [4, 6], it is
128 + into sequence. E.g., if the sequence length is [4, 6], it is
129 [0, 4, 10].
129 - seq_len_tensor (torch.Tensor): (batch_size,). record the sequence
130 - length of the sequences in the batch
130 + seq_len_tensor (torch.Tensor): (batch_size,). Record the sequence
131 + length of the sequences in the batch
132 lora_indices_tensor (torch.Tensor): (batch_size,). The LoRA index
133 corresponding to each batch. An index of -1 means no lora should be
134 applied.
135 batches (int): batch size
135 - max_seq_length (int): The max sequence lengths of the sequences
136 + max_seq_length (int): The max sequence lengths of the sequences
137 in the batch
137 - slice_offst (int): output_tensor's offst
138 + token_nums (int): The token numbers in the batch. Used to verify if the
139 + token numbers in the inputs matches the one in the metadata.
140 + slice_offset (int): output_tensor's offset
141 slice_size (int): current output_tensor's size
139 - add_inputs (bool, optional): Defaults to False. adds the final lora
140 - results to the output..
142 + add_inputs (bool, optional): Defaults to False, adds the final lora
143 + results to the output.
144 """
145
146 assert inputs.dtype in [torch.float16, torch.bfloat16, torch.float32]
@@ -145,6 +148,7 @@ def _sgmv_expand_slice(
148 torch.float16,
149 torch.bfloat16,
150 ]
151 + assert inputs.size(0) == token_nums
152 assert inputs.size(1) == lora_b_weights.size(-1)
153 assert b_seq_start_loc.size(0) == batches
154 assert lora_indices_tensor.size(0) == batches
vllm/lora/ops/sgmv_shrink.py
+10 -6
@@ -110,6 +110,7 @@ def _sgmv_shrink(
110 lora_indices_tensor: torch.Tensor,
111 batches: int,
112 max_seq_length: int,
113 + token_nums: int,
114 scaling: float,
115 ) -> None:
116 """
@@ -120,17 +121,19 @@ def _sgmv_shrink(
121 output_tensor (torch.Tensor): output tensor
122 b_seq_start_loc (torch.Tensor): (batch_size,). The cumulative
123 sequence lengths of the sequences in the batch, used to index
123 - into sequence. E.g.,if the sequence length is [4, 6], it is
124 + into sequence. E.g., if the sequence length is [4, 6], it is
125 [0, 4].
125 - seq_len_tensor (torch.Tensor): (batch_size,). record the sequence
126 - length of the sequences in the batch
126 + seq_len_tensor (torch.Tensor): (batch_size,). Record the sequence
127 + length of the sequences in the batch.
128 lora_indices_tensor (torch.Tensor): (batch_size,). The LoRA index
129 corresponding to each batch. An index of -1 means no lora should be
130 applied.
131 batches (int): batch size
131 - max_seq_length (int): The max sequence lengths of the sequences
132 - in the batch
133 - scaling (float): Scaling factor.
132 + max_seq_length (int): The max sequence lengths of the sequences in the
133 + batch.
134 + token_nums (int): The token numbers in the batch. Used to verify if the
135 + token numbers in the inputs matches the one in the metadata.
136 + scaling (float): Scaling factor.
137 """
138 assert inputs.dtype == lora_a_weights.dtype
139 assert inputs.dtype in [torch.float16, torch.bfloat16]
@@ -138,6 +141,7 @@ def _sgmv_shrink(
141 torch.float16,
142 torch.bfloat16,
143 ]
144 + assert inputs.size(0) == token_nums
145 assert inputs.size(1) == lora_a_weights.size(-1)
146 assert b_seq_start_loc.size(0) == batches
147 assert lora_indices_tensor.size(0) == batches
vllm/lora/punica.py
+21 -17
@@ -27,7 +27,7 @@ if TYPE_CHECKING:
27
28 def compute_meta(
29 token_lora_tensor: torch.Tensor
30 -) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, int, int, bool]:
30 +) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, int, int, int, bool]:
31 """
32 Get the information required for the sgmv kernel. With the features:
33 1. If consecutive requests in the batch use the same LoRA, this function
@@ -43,7 +43,7 @@ def compute_meta(
43 b_seq_start_tensor = torch.zeros_like(seq_length_tensor)
44 b_seq_start_tensor[1:].copy_(cum_result[:-1])
45 max_length = seq_length_tensor.max().item()
46 -
46 + token_nums = seq_length_tensor.sum().item()
47 batch_size = lora_indices_tensor.size(0)
48 no_lora = False
49 # -1 means no lora should be applied. Use `no_lora` to determine whether
@@ -52,7 +52,7 @@ def compute_meta(
52 if batch_size == 1 and lora_indices_tensor == -1:
53 no_lora = True
54 return (b_seq_start_tensor, seq_length_tensor, lora_indices_tensor,
55 - batch_size, max_length, no_lora)
55 + batch_size, max_length, token_nums, no_lora)
56
57
58 # TODO see if this can be vectorized
@@ -178,7 +178,7 @@ def convert_mapping(
178 class PunicaWrapper:
179 """
180 PunicaWrapper is designed to manage and provide metadata for the punica
181 - kernel. The main function is to maintain the state information for
181 + kernel. The main function is to maintain the state information for
182 Multi-LoRA, and to provide the interface for the punica kernel.
183 """
184
@@ -216,6 +216,7 @@ class PunicaWrapper:
216 dtype=torch.long,
217 device=device)
218 self.max_length: int = 0
219 + self.token_nums: int = 0
220 self.batch_size: int = -1
221 self.is_prefill = False
222 self.no_lora = False
@@ -276,13 +277,13 @@ class PunicaWrapper:
277 long_lora_offsets_tensor)
278 else:
279 self._long_lora_indices.zero_()
279 -
280 self.indices_len[:] = indices_len
281
282 def _update_prefill_metada(self, token_lora_tensor: torch.Tensor) -> None:
283
284 (b_seq_start_tensor, seq_length_tensor, lora_indices_tensor,
285 - batch_size, max_length, no_lora) = compute_meta(token_lora_tensor)
285 + batch_size, max_length, token_nums,
286 + no_lora) = compute_meta(token_lora_tensor)
287
288 self._seq_start_locs[:b_seq_start_tensor.shape[0]].copy_(
289 b_seq_start_tensor)
@@ -291,25 +292,28 @@ class PunicaWrapper:
292 lora_indices_tensor)
293 self.batch_size = batch_size
294 self.max_length = max_length
295 + self.token_nums = token_nums
296 self.no_lora = no_lora
297
298 @property
299 def prefill_metadata(
298 - self) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, int, int]:
300 + self
301 + ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, int, int, int]:
302 """
303 This property provides a convenient way to access the necessary
304 metadata for prefill-related kernel computations.
302 - 1. seq_start_locs: Tensor of sequence start positions
303 - 2. seq_lengths: Tensor of sequence lengths
305 + 1. seq_start_locs: Tensor of sequence start positions.
306 + 2. seq_lengths: Tensor of sequence lengths.
307 3. lora_indices_per_batch: Tensor of lora indices, and an index of
308 -1 means no lora should be applied.
306 - 4. batch_size: batch size after clustering identical lora indices
307 - 5. max_length: The maximum sequence length in the batch
309 + 4. batch_size: Batch size after clustering identical lora indices.
310 + 5. max_length: The maximum sequence length in the batch.
311 + 6. token_nums: The token numbers in the batch.
312 """
313 return (self._seq_start_locs[:self.batch_size],
314 self._seq_lengths[:self.batch_size],
315 self._lora_indices_per_batch[:self.batch_size],
312 - self.batch_size, self.max_length)
316 + self.batch_size, self.max_length, self.token_nums)
317
318 @property
319 def token_lora_indices(self) -> torch.Tensor:
@@ -324,7 +328,7 @@ class PunicaWrapper:
328 def sampler_indices(self) -> torch.Tensor:
329 """
330 This property is used to access the lora indices specifically for
327 - LogitsProcessorWithLoRA
331 + LogitsProcessorWithLoRA.
332 """
333 sampler_indices_len = self.indices_len[1]
334 return self._sampler_indices[:sampler_indices_len]
@@ -332,7 +336,7 @@ class PunicaWrapper:
336 @property
337 def sampler_indices_padded(self) -> torch.Tensor:
338 """
335 - This property provides access to padded sampler indices
339 + This property provides access to padded sampler indices.
340 """
341 indices_padded_len = self.indices_len[2]
342 return self._sampler_indices_padded[:indices_padded_len]
@@ -341,7 +345,7 @@ class PunicaWrapper:
345 def embeddings_indices(self) -> torch.Tensor:
346 """
347 This property provides access to the indices used for lora embeddings,
344 - specifically for VocabParallelEmbeddingWithLoRA
348 + specifically for VocabParallelEmbeddingWithLoRA.
349 """
350 embeddings_indices_len = self.indices_len[3]
351 return self._embeddings_indices[:, :embeddings_indices_len]
@@ -350,7 +354,7 @@ class PunicaWrapper:
354 def long_lora_indices(self) -> torch.Tensor:
355 """
356 This property provides access to the indices used for long context
353 - lora, specifically for LinearScalingRotaryEmbeddingWithLora
357 + lora, specifically for LinearScalingRotaryEmbeddingWithLora.
358 """
359 long_lora_len = self.indices_len[4]
360 return self._long_lora_indices[:long_lora_len]
@@ -524,7 +528,7 @@ class PunicaWrapper:
528 scale (float): Scaling factor.
529 y_offset (Optional[int], optional): Offset to apply to the starting
530 column of y.
527 - y_slice_size (Optional[int], optional): Size of the y column slice..
531 + y_slice_size (Optional[int], optional): Size of the y column slice.
532 buffer (Optional[torch.Tensor], optional): Defaults to None.
533 """
534 y_org = y