@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / ca871491ed
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 05:45 UTC

[Misc][LoRA] Abstract PunicaWrapper (#10955)

Signed-off-by: Jee Jee Li <pandaleefree@gmail.com>

Jee Jee Li committed Dec 10, 2024 at 04:54 UTC ca871491edb0fba11fe9aa94300bd8d282fa29e1
9 files changed +1058 -749
tests/lora/test_layers.py
+33 -16
@@ -28,7 +28,7 @@ from vllm.lora.layers import (BaseLayerWithLoRA, ColumnParallelLinearWithLoRA,
28 # yapf: enable
29 from vllm.lora.models import (LongContextLoRAContext, LoRALayerWeights,
30 PackedLoRALayerWeights)
31 -from vllm.lora.punica import PunicaWrapper
31 +from vllm.lora.punica_wrapper import get_punica_wrapper
32 from vllm.model_executor.layers.linear import (ColumnParallelLinear,
33 MergedColumnParallelLinear,
34 QKVParallelLinear,
@@ -48,11 +48,12 @@ TOLERANCES = {
48 torch.float32: (5e-3, 5e-3),
49 torch.bfloat16: (3e-2, 2e-2),
50 }
51 -CUDA_DEVICES = [
51 +# TODO: Modify this based on platform
52 +DEVICES = [
53 f"cuda:{i}" for i in range(1 if torch.cuda.device_count() == 1 else 2)
54 ]
55
55 -# We will launch different triton kernels between the prefill and decode
56 +#For GPU, we will launch different triton kernels between the prefill and decode
57 # stages, so we need to verify this. prefill stage(True) or decode stage(False)
58 STAGES = [True, False]
59
@@ -192,9 +193,18 @@ def create_random_inputs(
193 return inputs, index_mapping, prompt_mapping
194
195
196 +def check_punica_wrapper(punica_wrapper) -> bool:
197 + if current_platform.is_cuda_alike():
198 + from vllm.lora.punica_wrapper.punica_gpu import PunicaWrapperGPU
199 +
200 + return type(punica_wrapper) is PunicaWrapperGPU
201 + else:
202 + return False
203 +
204 +
205 @torch.inference_mode()
206 @pytest.mark.parametrize("num_loras", [1, 2, 4, 8])
197 -@pytest.mark.parametrize("device", CUDA_DEVICES)
207 +@pytest.mark.parametrize("device", DEVICES)
208 @pytest.mark.parametrize("vocab_size", [512, 32000, 64000, 128000])
209 @pytest.mark.parametrize("stage", STAGES)
210 def test_embeddings(dist_init, num_loras, device, vocab_size, stage) -> None:
@@ -205,7 +215,8 @@ def test_embeddings(dist_init, num_loras, device, vocab_size, stage) -> None:
215
216 torch.set_default_device(device)
217 max_loras = 8
208 - punica_wrapper = PunicaWrapper(8192, 256, device)
218 + punica_wrapper = get_punica_wrapper(8192, 256, device)
219 + assert check_punica_wrapper(punica_wrapper)
220 lora_config = LoRAConfig(max_loras=max_loras,
221 max_lora_rank=8,
222 lora_dtype=torch.float16)
@@ -296,7 +307,7 @@ def test_embeddings(dist_init, num_loras, device, vocab_size, stage) -> None:
307 # @pytest.mark.skip(
308 # reason="Fails when loras are in any slot other than the first.")
309 @pytest.mark.parametrize("num_loras", [1, 2, 4, 8])
299 -@pytest.mark.parametrize("device", CUDA_DEVICES)
310 +@pytest.mark.parametrize("device", DEVICES)
311 @pytest.mark.parametrize("vocab_size", [512, 32000, 64000, 128000])
312 @pytest.mark.parametrize("stage", STAGES)
313 def test_embeddings_with_new_embeddings(dist_init, num_loras, device,
@@ -305,7 +316,8 @@ def test_embeddings_with_new_embeddings(dist_init, num_loras, device,
316 torch.cuda.set_device(device)
317 torch.set_default_device(device)
318 max_loras = 8
308 - punica_wrapper = PunicaWrapper(8192, 256, device)
319 + punica_wrapper = get_punica_wrapper(8192, 256, device)
320 + assert check_punica_wrapper(punica_wrapper)
321 lora_config = LoRAConfig(max_loras=max_loras,
322 max_lora_rank=8,
323 lora_dtype=torch.float16)
@@ -432,7 +444,7 @@ def test_embeddings_with_new_embeddings(dist_init, num_loras, device,
444
445 @torch.inference_mode()
446 @pytest.mark.parametrize("num_loras", [1, 2, 4, 8])
435 -@pytest.mark.parametrize("device", CUDA_DEVICES)
447 +@pytest.mark.parametrize("device", DEVICES)
448 @pytest.mark.parametrize("vocab_size", [512, 32000, 64000, 256512])
449 @pytest.mark.parametrize("stage", STAGES)
450 def test_lm_head_logits_processor(dist_init, num_loras, device, vocab_size,
@@ -441,7 +453,8 @@ def test_lm_head_logits_processor(dist_init, num_loras, device, vocab_size,
453 torch.cuda.set_device(device)
454 torch.set_default_device(device)
455 max_loras = 8
444 - punica_wrapper = PunicaWrapper(8192, 256, device)
456 + punica_wrapper = get_punica_wrapper(8192, 256, device)
457 + assert check_punica_wrapper(punica_wrapper)
458 lora_config = LoRAConfig(max_loras=max_loras,
459 max_lora_rank=8,
460 lora_dtype=torch.float16)
@@ -563,7 +576,7 @@ def test_lm_head_logits_processor(dist_init, num_loras, device, vocab_size,
576
577 @torch.inference_mode()
578 @pytest.mark.parametrize("num_loras", [1, 2, 4, 8])
566 -@pytest.mark.parametrize("device", CUDA_DEVICES)
579 +@pytest.mark.parametrize("device", DEVICES)
580 @pytest.mark.parametrize("stage", STAGES)
581 @pytest.mark.parametrize("bias_enabled", [True, False])
582 def test_linear_replicated(dist_init, num_loras, device, stage,
@@ -571,7 +584,8 @@ def test_linear_replicated(dist_init, num_loras, device, stage,
584
585 torch.cuda.set_device(device)
586 torch.set_default_device(device)
574 - punica_wrapper = PunicaWrapper(8192, 256, device)
587 + punica_wrapper = get_punica_wrapper(8192, 256, device)
588 + assert check_punica_wrapper(punica_wrapper)
589 max_loras = 8
590 lora_config = LoRAConfig(max_loras=max_loras,
591 max_lora_rank=8,
@@ -675,7 +689,7 @@ def test_linear_replicated(dist_init, num_loras, device, stage,
689 @pytest.mark.parametrize("num_loras", [1, 2, 4, 8])
690 @pytest.mark.parametrize("orientation", ["row", "column"])
691 @pytest.mark.parametrize("fully_shard", [True, False])
678 -@pytest.mark.parametrize("device", CUDA_DEVICES)
692 +@pytest.mark.parametrize("device", DEVICES)
693 @pytest.mark.parametrize("stage", STAGES)
694 @pytest.mark.parametrize("bias_enabled", [True, False])
695 def test_linear_parallel(dist_init, num_loras, orientation, fully_shard,
@@ -683,7 +697,8 @@ def test_linear_parallel(dist_init, num_loras, orientation, fully_shard,
697
698 torch.cuda.set_device(device)
699 torch.set_default_device(device)
686 - punica_wrapper = PunicaWrapper(8192, 256, device)
700 + punica_wrapper = get_punica_wrapper(8192, 256, device)
701 + assert check_punica_wrapper(punica_wrapper)
702 max_loras = 8
703 lora_config = LoRAConfig(max_loras=max_loras,
704 max_lora_rank=8,
@@ -797,7 +812,7 @@ def test_linear_parallel(dist_init, num_loras, orientation, fully_shard,
812 @pytest.mark.parametrize("num_loras", [1, 2, 4, 8])
813 @pytest.mark.parametrize("repeats", [1, 2, 3])
814 @pytest.mark.parametrize("fully_shard", [True, False])
800 -@pytest.mark.parametrize("device", CUDA_DEVICES)
815 +@pytest.mark.parametrize("device", DEVICES)
816 @pytest.mark.parametrize("stage", STAGES)
817 @pytest.mark.parametrize("bias_enabled", [True, False])
818 def test_column_parallel_packed(dist_init, num_loras, repeats, fully_shard,
@@ -805,7 +820,8 @@ def test_column_parallel_packed(dist_init, num_loras, repeats, fully_shard,
820
821 torch.cuda.set_device(device)
822 torch.set_default_device(device)
808 - punica_wrapper = PunicaWrapper(8192, 256, device)
823 + punica_wrapper = get_punica_wrapper(8192, 256, device)
824 + assert check_punica_wrapper(punica_wrapper)
825 max_loras = 8
826 lora_config = LoRAConfig(max_loras=max_loras,
827 max_lora_rank=8,
@@ -963,7 +979,8 @@ def test_rotary_embedding_long_context(dist_init, num_loras, device,
979 seed = 0
980 current_platform.seed_everything(seed)
981 torch.set_default_device(device)
966 - punica_wrapper = PunicaWrapper(8192, 256, device)
982 + punica_wrapper = get_punica_wrapper(8192, 256, device)
983 + assert check_punica_wrapper(punica_wrapper)
984 max_loras = 8
985 lora_config = LoRAConfig(max_loras=max_loras,
986 max_lora_rank=8,
vllm/lora/layers.py
+3 -4
@@ -17,7 +17,6 @@ from vllm.distributed import (get_tensor_model_parallel_rank,
17 tensor_model_parallel_all_reduce,
18 tensor_model_parallel_gather)
19 from vllm.distributed.utils import divide
20 -from vllm.lora.punica import PunicaWrapper
20 # yapf: disable
21 from vllm.model_executor.layers.linear import (ColumnParallelLinear,
22 LinearBase,
@@ -33,7 +32,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
32 VocabParallelEmbedding)
33
34 if TYPE_CHECKING:
36 - pass
35 + from vllm.lora.punica_wrapper import PunicaWrapperBase
36
37
38 def _get_lora_device(base_layer: nn.Module) -> torch.device:
@@ -115,9 +114,9 @@ class BaseLayerWithLoRA(nn.Module):
114
115 def set_mapping(
116 self,
118 - punica_wrapper: PunicaWrapper,
117 + punica_wrapper,
118 ):
120 - self.punica_wrapper: PunicaWrapper = punica_wrapper
119 + self.punica_wrapper: PunicaWrapperBase = punica_wrapper
120
121 @classmethod
122 def can_replace_layer(
vllm/lora/models.py
+4 -4
@@ -21,7 +21,7 @@ from vllm.lora.layers import (BaseLayerWithLoRA,
21 LinearScalingRotaryEmbeddingWithLora,
22 LoRAMapping)
23 from vllm.lora.lora import LoRALayerWeights, PackedLoRALayerWeights
24 -from vllm.lora.punica import PunicaWrapper
24 +from vllm.lora.punica_wrapper import get_punica_wrapper
25 from vllm.lora.utils import (from_layer, from_layer_logits_processor,
26 is_regex_target_modules,
27 parse_fine_tuned_lora_name, replace_submodule)
@@ -331,9 +331,9 @@ class LoRAModelManager(AdapterModelManager):
331 self.lora_index_to_id: List[Optional[int]] = [None] * self.lora_slots
332 self.vocab_size = vocab_size
333 self.long_lora_context: Optional[LongContextLoRAContext] = None
334 - self.punica_wrapper = PunicaWrapper(max_num_batched_tokens,
335 - max_batches=self.max_num_seqs,
336 - device=self.device)
334 + self.punica_wrapper = get_punica_wrapper(max_num_batched_tokens,
335 + max_batches=self.max_num_seqs,
336 + device=self.device)
337 # Scaling factor -> offset to the sin_cos_cache to it.
338 # Used for long context lora.
339 self.scaling_factor_to_offset: Dict[float, int] = {}
vllm/lora/punica.py deleted
-725
@@ -1,725 +0,0 @@
1 -"""
2 -Based on:
3 -Chen, L., Ye, Z., Wu, Y., Zhuo, D., Ceze, L., & Krishnamurthy, A. (2023).
4 -Punica: Multi-Tenant LoRA Serving.
5 -https://arxiv.org/abs/2310.18547
6 -"""
7 -
8 -from typing import TYPE_CHECKING, Callable, List, Optional, Tuple, Union
9 -
10 -import torch
11 -
12 -from vllm.triton_utils import HAS_TRITON
13 -
14 -if HAS_TRITON:
15 - from vllm.lora.ops.bgmv_expand import bgmv_expand
16 - from vllm.lora.ops.bgmv_expand_slice import bgmv_expand_slice
17 - from vllm.lora.ops.bgmv_shrink import bgmv_shrink
18 - from vllm.lora.ops.sgmv_expand import sgmv_expand
19 - from vllm.lora.ops.sgmv_expand_slice import sgmv_expand_slice
20 - from vllm.lora.ops.sgmv_shrink import sgmv_shrink
21 -
22 -if TYPE_CHECKING:
23 - # avoid circuit import
24 - from vllm.lora.layers import LoRAMapping
25 - from vllm.lora.models import LongContextLoRAContext
26 -
27 -
28 -def compute_meta(
29 - token_lora_tensor: torch.Tensor
30 -) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, int, int, int, bool]:
31 - """
32 - Get the information required for the sgmv kernel. With the features:
33 - 1. If consecutive requests in the batch use the same LoRA, this function
34 - will combine them into a single request, improving sgmv kernel inference
35 - performance.
36 - 2. At the beginning of each prefill stage inference, recalculations are
37 - needed based on the input, but only once.
38 - """
39 -
40 - lora_indices_tensor, seq_length_tensor = torch.unique_consecutive(
41 - token_lora_tensor, return_counts=True)
42 - cum_result = torch.cumsum(seq_length_tensor, dim=0)
43 - b_seq_start_tensor = torch.zeros_like(seq_length_tensor)
44 - b_seq_start_tensor[1:].copy_(cum_result[:-1])
45 - max_length = seq_length_tensor.max().item()
46 - token_nums = seq_length_tensor.sum().item()
47 - batch_size = lora_indices_tensor.size(0)
48 - no_lora = False
49 - # -1 means no lora should be applied. Use `no_lora` to determine whether
50 - # the current step requires LoRA. If LoRA is not needed, the prefill stage
51 - # does not need to launch the triton kernel, which can improve performance
52 - if batch_size == 1 and lora_indices_tensor == -1:
53 - no_lora = True
54 - return (b_seq_start_tensor, seq_length_tensor, lora_indices_tensor,
55 - batch_size, max_length, token_nums, no_lora)
56 -
57 -
58 -# TODO see if this can be vectorized
59 -def convert_mapping(
60 - mapping: "LoRAMapping",
61 - lora_index_to_id: List[Optional[int]],
62 - max_loras: int,
63 - vocab_size: int,
64 - extra_vocab_size: int,
65 - device: torch.device,
66 - long_lora_context: Optional["LongContextLoRAContext"] = None,
67 -) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor,
68 - Optional[torch.Tensor], List[int]]:
69 - """Converts LoRAMapping to index tensors.
70 -
71 - Args:
72 - mapping: LoRAMapping mapping rows in a batch to LoRA ids.
73 - lora_index_to_id: List mapping LoRA ids to LoRA indices.
74 - max_loras: Maximum number of LoRAs.
75 - vocab_size: Model vocab size.
76 - extra_vocab_size: Extra vocab size each LoRA can have.
77 - long_lora_context: Passed if there are long context lora in a batch.
78 -
79 - Returns:
80 - A tuple of tensors:
81 - base_indices: Tensor of shape [batch_size] mapping batch rows to
82 - LoRA indices.
83 - sampler_indices: Tensor of shape [batch_size] mapping requests to
84 - LoRA indices for sampler. For generation, this will be the
85 - same as base_indicies. For prefill, this will map requests
86 - to LoRA indices.
87 - sampler_indices_padded: Tensor of shape [batch_size] mapping
88 - requests to LoRA indices for sampler with padding.
89 - Same as sampler_indicies, but -1 is replaced with
90 - max_loras.
91 - embeddings_indices: Tensor of shape [2, batch_size] mapping
92 - requests to embedding indices. First row is for embeddings
93 - added by the LoRAs, second row is for the LoRA.lora_a
94 - embeddings.
95 - long_lora_indices: Tensor of shape [batch_size] mapping
96 - requests to RoPE offsets and rot dims for long LoRAs.
97 - None if long context lora doesn't exist.
98 - indices_len: List of lengths of the above tensors. It contains
99 - (base_indices, sampler_indices, sampler_indices_padded,
100 - embeddings_indices, long_lora_indices).
101 - """
102 - index_mapping_indices: List[int] = list(mapping.index_mapping).copy()
103 - embedding_indices = index_mapping_indices.copy()
104 - lora_indices = index_mapping_indices.copy()
105 - long_lora_offsets: Optional[torch.Tensor] = None
106 - if long_lora_context:
107 - long_lora_offsets = torch.zeros(len(index_mapping_indices),
108 - device=device,
109 - dtype=torch.long)
110 - prompt_mapping: List[int] = [
111 - lora_index_to_id.index(x) if x > 0 else -1
112 - for x in mapping.prompt_mapping
113 - ]
114 - lora_idx = None
115 - for i in range(len(index_mapping_indices)):
116 - # TODO index can be slow. optimize
117 - lora_idx = (lora_index_to_id.index(index_mapping_indices[i])
118 - if index_mapping_indices[i] > 0 else -1)
119 - embedding_indices[i] = lora_idx if index_mapping_indices[i] > 0 else 0
120 - lora_indices[i] = lora_idx
121 - if long_lora_context:
122 - assert long_lora_offsets is not None
123 - lora_offset: int = long_lora_context.offsets_by_lora_id.get(
124 - index_mapping_indices[i], 0)
125 - long_lora_offsets[i] = lora_offset
126 -
127 - indices_list: List[Union[List[int], torch.Tensor]] = [
128 - index_mapping_indices,
129 - lora_indices,
130 - embedding_indices,
131 - ]
132 - if long_lora_context:
133 - assert long_lora_offsets is not None
134 - indices_list.append(long_lora_offsets)
135 - indices = torch.tensor(indices_list, dtype=torch.long, device=device)
136 - prompt_mapping_tensor = torch.tensor(prompt_mapping,
137 - dtype=torch.long,
138 - device=device)
139 - embeddings_indices = torch.stack([
140 - indices[2] * extra_vocab_size,
141 - indices[2] * (vocab_size + extra_vocab_size),
142 - ])
143 - embeddings_indices[embeddings_indices == -1] = max_loras - 1
144 - base_indices = indices[1]
145 - sampler_indices = prompt_mapping_tensor
146 - sampler_indices_padded = sampler_indices.clone()
147 - sampler_indices_padded[sampler_indices_padded == -1] = max_loras - 1
148 - sampler_indices_padded = torch.arange(
149 - 0, len(sampler_indices_padded), device=device, dtype=torch.long) + (
150 - sampler_indices_padded * len(sampler_indices_padded))
151 - long_lora_indices = None
152 - long_lora_indices_len: Optional[int] = None
153 - if long_lora_context:
154 - long_lora_indices = indices[3]
155 - long_lora_indices_len = long_lora_indices.shape[-1]
156 - # Contain length of indices tensors. Used to index into each tensor.
157 - indices_len = [
158 - base_indices.shape[-1],
159 - sampler_indices.shape[-1],
160 - sampler_indices_padded.shape[-1],
161 - embeddings_indices.shape[-1],
162 - ]
163 - if long_lora_indices_len is not None:
164 - indices_len.append(long_lora_indices_len)
165 - else:
166 - # If long_lora doesn't exist,append None
167 - indices_len.append(None)
168 -
169 - return (
170 - base_indices,
171 - sampler_indices,
172 - sampler_indices_padded,
173 - embeddings_indices,
174 - long_lora_indices,
175 - indices_len,
176 - )
177 -
178 -
179 -class PunicaWrapper:
180 - """
181 - PunicaWrapper is designed to manage and provide metadata for the punica
182 - kernel. The main function is to maintain the state information for
183 - Multi-LoRA, and to provide the interface for the punica kernel.
184 - """
185 -
186 - def __init__(self, max_num_batched_tokens: int, max_batches: int,
187 - device: Union[torch.device, str]):
188 - self._token_lora_indices = torch.empty(max_num_batched_tokens,
189 - dtype=torch.long,
190 - device=device)
191 - self._sampler_indices = torch.empty(max_num_batched_tokens,
192 - dtype=torch.long,
193 - device=device)
194 - self._sampler_indices_padded = torch.empty(max_num_batched_tokens,
195 - dtype=torch.long,
196 - device=device)
197 - self._embeddings_indices = torch.empty(2,
198 - max_num_batched_tokens,
199 - dtype=torch.long,
200 - device=device)
201 - self._long_lora_indices = torch.empty(max_num_batched_tokens,
202 - dtype=torch.long,
203 - device=device)
204 -
205 - # 5 is the number of indicies tensors.
206 - # base_indices, sampler_indices, sampler_indices_padded,
207 - # embeddings_indices,long_lora_indices
208 - self.indices_len: List[Optional[int]] = [None] * 5
209 - # these attributes are the information required for sgmv kernel
210 - self._seq_start_locs = torch.empty(max_batches,
211 - dtype=torch.long,
212 - device=device)
213 - self._seq_lengths = torch.empty(max_batches,
214 - dtype=torch.long,
215 - device=device)
216 - self._lora_indices_per_batch = torch.empty(max_batches,
217 - dtype=torch.long,
218 - device=device)
219 - self.device: torch.device = device
220 - self.max_length: int = 0
221 - self.token_nums: int = 0
222 - self.batch_size: int = -1
223 - self.is_prefill = False
224 - self.no_lora = False
225 -
226 - def update_metadata(
227 - self,
228 - mapping: "LoRAMapping",
229 - lora_index_to_id: List[Optional[int]],
230 - max_loras: int,
231 - vocab_size: int,
232 - extra_vocab_size: int,
233 - long_lora_context: Optional["LongContextLoRAContext"] = None,
234 - ):
235 -
236 - self._update_base_metadata(mapping, lora_index_to_id, max_loras,
237 - vocab_size, extra_vocab_size,
238 - long_lora_context)
239 - if mapping.is_prefill:
240 - # Update metadata required for prefill-related operators.
241 - self._update_prefill_metada(self.token_lora_indices)
242 - self.is_prefill = True
243 - else:
244 - self.is_prefill = False
245 -
246 - def _update_base_metadata(
247 - self,
248 - mapping: "LoRAMapping",
249 - lora_index_to_id: List[Optional[int]],
250 - max_loras: int,
251 - vocab_size: int,
252 - extra_vocab_size: int,
253 - long_lora_context: Optional["LongContextLoRAContext"] = None,
254 - ):
255 - (
256 - base_indices,
257 - sampler_indices,
258 - sampler_indices_padded,
259 - embeddings_indices,
260 - long_lora_offsets_tensor,
261 - indices_len,
262 - ) = convert_mapping(
263 - mapping,
264 - lora_index_to_id,
265 - max_loras,
266 - vocab_size,
267 - extra_vocab_size,
268 - self.device,
269 - long_lora_context,
270 - )
271 - self._token_lora_indices[:base_indices.shape[0]].copy_(base_indices)
272 - self._sampler_indices[:sampler_indices.shape[0]].copy_(sampler_indices)
273 - self._sampler_indices_padded[:sampler_indices_padded.shape[0]].copy_(
274 - sampler_indices_padded)
275 - self._embeddings_indices[:embeddings_indices.
276 - shape[0], :embeddings_indices.shape[1]].copy_(
277 - embeddings_indices)
278 - if long_lora_offsets_tensor is not None:
279 - self._long_lora_indices[:long_lora_offsets_tensor.shape[0]].copy_(
280 - long_lora_offsets_tensor)
281 - else:
282 - self._long_lora_indices.zero_()
283 - self.indices_len[:] = indices_len
284 -
285 - def _update_prefill_metada(self, token_lora_tensor: torch.Tensor) -> None:
286 -
287 - (b_seq_start_tensor, seq_length_tensor, lora_indices_tensor,
288 - batch_size, max_length, token_nums,
289 - no_lora) = compute_meta(token_lora_tensor)
290 -
291 - self._seq_start_locs[:b_seq_start_tensor.shape[0]].copy_(
292 - b_seq_start_tensor)
293 - self._seq_lengths[:seq_length_tensor.shape[0]].copy_(seq_length_tensor)
294 - self._lora_indices_per_batch[:lora_indices_tensor.shape[0]].copy_(
295 - lora_indices_tensor)
296 - self.batch_size = batch_size
297 - self.max_length = max_length
298 - self.token_nums = token_nums
299 - self.no_lora = no_lora
300 -
301 - @property
302 - def prefill_metadata(
303 - self
304 - ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, int, int, int]:
305 - """
306 - This property provides a convenient way to access the necessary
307 - metadata for prefill-related kernel computations.
308 - 1. seq_start_locs: Tensor of sequence start positions.
309 - 2. seq_lengths: Tensor of sequence lengths.
310 - 3. lora_indices_per_batch: Tensor of lora indices, and an index of
311 - -1 means no lora should be applied.
312 - 4. batch_size: Batch size after clustering identical lora indices.
313 - 5. max_length: The maximum sequence length in the batch.
314 - 6. token_nums: The token numbers in the batch.
315 - """
316 - return (self._seq_start_locs[:self.batch_size],
317 - self._seq_lengths[:self.batch_size],
318 - self._lora_indices_per_batch[:self.batch_size],
319 - self.batch_size, self.max_length, self.token_nums)
320 -
321 - @property
322 - def token_lora_indices(self) -> torch.Tensor:
323 - """
324 - This property provides the lora indices corresponding to each token
325 - in the batch. An index of -1 means no lora should be applied.
326 - """
327 - token_lora_len = self.indices_len[0]
328 - return self._token_lora_indices[:token_lora_len]
329 -
330 - @property
331 - def sampler_indices(self) -> torch.Tensor:
332 - """
333 - This property is used to access the lora indices specifically for
334 - LogitsProcessorWithLoRA.
335 - """
336 - sampler_indices_len = self.indices_len[1]
337 - return self._sampler_indices[:sampler_indices_len]
338 -
339 - @property
340 - def sampler_indices_padded(self) -> torch.Tensor:
341 - """
342 - This property provides access to padded sampler indices.
343 - """
344 - indices_padded_len = self.indices_len[2]
345 - return self._sampler_indices_padded[:indices_padded_len]
346 -
347 - @property
348 - def embeddings_indices(self) -> torch.Tensor:
349 - """
350 - This property provides access to the indices used for lora embeddings,
351 - specifically for VocabParallelEmbeddingWithLoRA.
352 - """
353 - embeddings_indices_len = self.indices_len[3]
354 - return self._embeddings_indices[:, :embeddings_indices_len]
355 -
356 - @property
357 - def long_lora_indices(self) -> torch.Tensor:
358 - """
359 - This property provides access to the indices used for long context
360 - lora, specifically for LinearScalingRotaryEmbeddingWithLora.
361 - """
362 - long_lora_len = self.indices_len[4]
363 - return self._long_lora_indices[:long_lora_len]
364 -
365 - def _shrink_prefill(
366 - self,
367 - y: torch.Tensor,
368 - x: torch.Tensor,
369 - w_t_all: torch.Tensor,
370 - scale: float,
371 - ):
372 - #No LoRA request, so return directly
373 - if self.no_lora:
374 - return
375 - sgmv_shrink(
376 - x,
377 - w_t_all,
378 - y,
379 - *self.prefill_metadata,
380 - scale,
381 - )
382 -
383 - def _shrink_decode(
384 - self,
385 - y: torch.Tensor,
386 - x: torch.Tensor,
387 - w_t_all: torch.Tensor,
388 - scale: float,
389 - ):
390 - bgmv_shrink(x, w_t_all, y, self.token_lora_indices, scale)
391 -
392 - def _expand_prefill(
393 - self,
394 - y: torch.Tensor,
395 - x: torch.Tensor,
396 - w_t_all: torch.Tensor,
397 - add_input: bool,
398 - ):
399 - #No LoRA request, so return directly
400 - if self.no_lora:
401 - return
402 - sgmv_expand(
403 - x,
404 - w_t_all,
405 - y,
406 - *self.prefill_metadata,
407 - add_input,
408 - )
409 -
410 - def _expand_decode(
411 - self,
412 - y: torch.Tensor,
413 - x: torch.Tensor,
414 - w_t_all: torch.Tensor,
415 - add_input: bool,
416 - ):
417 - bgmv_expand(x, w_t_all, y, self.token_lora_indices, add_input)
418 -
419 - def _expand_slice_prefill(
420 - self,
421 - y: torch.Tensor,
422 - x: torch.Tensor,
423 - w_t_all: torch.Tensor,
424 - y_offset: Optional[int],
425 - y_slice_size: Optional[int],
426 - add_input: bool,
427 - ):
428 - #No LoRA request, so return directly
429 - if self.no_lora:
430 - return
431 - sgmv_expand_slice(
432 - x,
433 - w_t_all,
434 - y,
435 - *self.prefill_metadata,
436 - y_offset,
437 - y_slice_size,
438 - add_input,
439 - )
440 -
441 - def _expand_slice_decode(
442 - self,
443 - y: torch.Tensor,
444 - x: torch.Tensor,
445 - w_t_all: torch.Tensor,
446 - y_offset: Optional[int],
447 - y_slice_size: Optional[int],
448 - add_input: bool,
449 - ):
450 - bgmv_expand_slice(x, w_t_all, y, self.token_lora_indices, y_offset,
451 - y_slice_size, add_input)
452 -
453 - def _apply_expand(self,
454 - y: torch.Tensor,
455 - x: torch.Tensor,
456 - w_t_all: torch.Tensor,
457 - y_offset: Optional[int],
458 - y_slice_size: Optional[int],
459 - add_input: bool = True):
460 - """
461 - Perform the ` y[:,y_offset:y_offset+y_slice_size]+=x@w_t_all`
462 - computation, which is suitable for the
463 - GEMM of lora'b.
464 - """
465 -
466 - expand_slice_fun: Callable = (self._expand_slice_prefill
467 - if self.is_prefill else
468 - self._expand_slice_decode)
469 - expand_slice_fun(y, x, w_t_all, y_offset, y_slice_size, add_input)
470 -
471 - def _apply_bias(
472 - self,
473 - indices: torch.Tensor,
474 - output: torch.Tensor,
475 - output_slices: Tuple[int, ...],
476 - lora_bias_stacked: Tuple[Optional[torch.Tensor], ...],
477 - ):
478 - """Applies bias to output
479 -
480 - Input shapes:
481 - lora_bias_stacked: 3 element tuple of (num_loras, output_dim)
482 - indices: (batch_size)
483 - output: (batch_size, q_slice_size + 2*kv_slice_size)
484 - output_slices: n-1 element tuple of (slice_size...),
485 - where n is number of slices
486 - """
487 - org_output = output
488 - output = output.view(-1, output.shape[-1])
489 - indices = indices.view(-1)
490 -
491 - offset_left = 0
492 - for slice_idx, slice in enumerate(output_slices):
493 - bias = lora_bias_stacked[slice_idx]
494 - if bias is not None:
495 - bias = bias.view(-1, bias.shape[-1])
496 - bias = bias[indices]
497 - bias[indices == -1] = 0
498 - output[:, offset_left:offset_left + slice] += bias
499 - offset_left += slice
500 -
501 - return output.view_as(org_output)
502 -
503 - def _apply_shrink(
504 - self,
505 - y: torch.Tensor,
506 - x: torch.Tensor,
507 - w_t_all: torch.Tensor,
508 - scale: float,
509 - ):
510 - """
511 - Perform the ` y+=x@w_t_all` computation, which is suitable for the
512 - GEMM of lora'a.
513 - When `is_prefill is` true, it indicates that it is currently the
514 - prefill stage, and the `_shrink_prefill` function should be called.
515 - Otherwise, it is the decode stage, and the _shrink_decode function
516 - should be called.
517 - """
518 - y_org = y
519 - y = y.view(-1, y.shape[-1])
520 - shrink_fun: Callable = (self._shrink_prefill
521 - if self.is_prefill else self._shrink_decode)
522 - shrink_fun(y, x, w_t_all, scale)
523 - y = y.view_as(y_org)
524 -
525 - def add_shrink(
526 - self,
527 - y: Union[Tuple[torch.Tensor, ...], torch.Tensor],
528 - x: torch.Tensor,
529 - lora_a_stacked: Tuple[torch.Tensor, ...],
530 - scale: float,
531 - ):
532 - """
533 - Performs GEMM for multiple slices of lora_a.
534 - When `is_prefill is` true, it indicates that it is currently the
535 - prefill stage, and the `_shrink_prefill` function should be called.
536 - Otherwise, it is the decode stage, and the _shrink_decode function
537 - should be called.
538 -
539 - Semantics:
540 - for i in range(len(lora_a_stacked)):
541 - y[i] += (x @ lora_a_stacked[i]) * scale
542 -
543 - Args:
544 - y (Union[Tuple[torch.Tensor, ...], torch.Tensor]): Output tensors
545 - x (torch.Tensor): Input tensor
546 - lora_a_stacked (Tuple[torch.Tensor, ...]): lora_a's weights
547 - scale (float): Scaling factor for the operation
548 - """
549 -
550 - x = x.view(-1, x.shape[-1])
551 - # TODO fuse these kernels
552 - for slice_idx in range(len(lora_a_stacked)):
553 - self._apply_shrink(y[slice_idx], x, lora_a_stacked[slice_idx],
554 - scale)
555 -
556 - def add_expand(
557 - self,
558 - y: torch.Tensor,
559 - x: Union[Tuple[torch.Tensor, ...], torch.Tensor],
560 - lora_b_stacked: Tuple[torch.Tensor, ...],
561 - lora_bias_stacked: Optional[Tuple[torch.Tensor, ...]],
562 - output_slices: Tuple[int, ...],
563 - offset_start: int = 0,
564 - add_input=True,
565 - ) -> None:
566 - """
567 - Performs GEMM and bias addition for multiple slices of lora_b.
568 -
569 - Semantics:
570 - for i in range(len(lora_b_stacked)):
571 - slice = output_slices[i]
572 - y[:, offset:offset+slice] += x[i] @ lora_b_stacked[i] +
573 - lora_bias_stacked[i]
574 - offset += slice
575 -
576 - Args:
577 - y (torch.Tensor): Output tensor.
578 - x (Union[Tuple[torch.Tensor, ...], torch.Tensor]): Input tensors
579 - lora_b_stacked (Tuple[torch.Tensor, ...]): lora_b's weight
580 - lora_bias_stacked (Optional[Tuple[torch.Tensor, ...]]):
581 - bias's weight
582 - output_slices (Tuple[int, ...]): Every slice's size
583 - add_input (bool): Defaults to True.
584 - """
585 - y_org = y
586 - y = y.view(-1, y.shape[-1])
587 - offset_left = offset_start
588 - if lora_bias_stacked is not None:
589 - self._apply_bias(self.token_lora_indices, y, output_slices,
590 - lora_bias_stacked)
591 - for slice_idx in range(len(lora_b_stacked)):
592 - self._apply_expand(
593 - y,
594 - x[slice_idx],
595 - lora_b_stacked[slice_idx],
596 - offset_left,
597 - output_slices[slice_idx],
598 - add_input=add_input,
599 - )
600 - offset_left += output_slices[slice_idx]
601 - y = y.view_as(y_org)
602 -
603 - def add_lora_embedding(
604 - self,
605 - y: torch.Tensor,
606 - x: torch.Tensor,
607 - lora_b_stacked: torch.Tensor,
608 - add_input: bool = True,
609 - ):
610 - """
611 - Applies lora specifically for VocabParallelEmbeddingWithLoRA.
612 -
613 - Semantics:
614 - y += x @ lora_b_stacked
615 -
616 - Args:
617 - y (torch.Tensor): Output tensor.
618 - x (torch.Tensor): Input tensor.
619 - lora_b_stacked (torch.Tensor): lora_b's weights.
620 - add_input (bool): Default to True.
621 -
622 - """
623 -
624 - # Embedding layer only need expand op
625 - expand_fun: Callable = (self._expand_prefill
626 - if self.is_prefill else self._expand_decode)
627 - expand_fun(y, x, lora_b_stacked, add_input)
628 -
629 - def add_lora_linear(
630 - self,
631 - y: torch.Tensor,
632 - x: torch.Tensor,
633 - lora_a_stacked: Tuple[torch.Tensor, ...],
634 - lora_b_stacked: Tuple[torch.Tensor, ...],
635 - lora_bias_stacked: Optional[Tuple[torch.Tensor, ...]],
636 - scale: float,
637 - output_slices: Tuple[int, ...],
638 - *,
639 - buffer: Optional[Tuple[torch.Tensor, ...]] = None) -> None:
640 - """
641 - Applicable to linear-related lora.
642 -
643 - Semantics:
644 - for i in range(len(lora_a_stacked)):
645 - y[i] += (
646 - x[i].unsqueeze(0)
647 - @ lora_a_stacked[indices[i], layer_idx, :, :]
648 - @ lora_b_stacked[indices[i], layer_idx, :, :]
649 - * scale
650 - ).squeeze(0)+lora_bias_stacked[i]
651 -
652 - Args:
653 - y (torch.Tensor): Output tensor. Will be changed in-place.
654 - x (torch.Tensor): Input tensor
655 - lora_a_stacked (Tuple[torch.Tensor, ...]): lora_a's weight.
656 - lora_b_stacked (Tuple[torch.Tensor, ...]): lora_b's weight.
657 - lora_bias_stacked (Optional[Tuple[torch.Tensor, ...]]): lora's bias.
658 - scale (float): Scaling factor.
659 - output_slices (Tuple[int, ...]): Every slice's size.
660 - buffer (Optional[Tuple[torch.Tensor, ...]]): Defaults to None.
661 - """
662 -
663 - assert len(lora_a_stacked) == len(lora_b_stacked) == len(output_slices)
664 - if lora_bias_stacked is not None:
665 - assert len(lora_bias_stacked) == len(output_slices)
666 - y = self._apply_bias(self.token_lora_indices, y, output_slices,
667 - lora_bias_stacked)
668 -
669 - if buffer is None:
670 - r = lora_b_stacked[0].size(-1)
671 - # We set the buffer to be float32 by default ,refer to:
672 - # https://github.com/triton-lang/triton/issues/1387
673 - buffer = tuple(
674 - torch.zeros(
675 - (x.size(0), r), dtype=torch.float32, device=x.device)
676 - for _ in range(len(output_slices)))
677 - self.add_shrink(buffer, x, lora_a_stacked, scale)
678 - self.add_expand(y,
679 - buffer,
680 - lora_b_stacked,
681 - None,
682 - output_slices,
683 - add_input=True)
684 -
685 - def add_lora_logits(self,
686 - y: torch.Tensor,
687 - x: torch.Tensor,
688 - lora_a_stacked: torch.Tensor,
689 - lora_b_stacked: torch.Tensor,
690 - scale,
691 - *,
692 - buffer: Optional[torch.Tensor] = None) -> None:
693 - """
694 - Applies lora specifically for LogitsProcessorWithLoRA.
695 -
696 - Semantics:
697 - buffer = (x @ lora_a_stacked) * scale
698 - y += buffer @ lora_b_stacked
699 -
700 - Args:
701 - y (torch.Tensor): Output tensor.
702 - x (torch.Tensor): Input tensor.
703 - lora_a_stacked (torch.Tensor): lora_a's weights.
704 - lora_b_stacked (torch.Tensor):lora_b's weights.
705 - scale (float): Scaling factor.
706 - buffer (Optional[torch.Tensor]):Default to None.
707 - """
708 - y_org = y
709 - y = y.view(-1, y.shape[-1])
710 - x = x.view(-1, x.shape[-1])
711 - r = lora_b_stacked.size(-1)
712 - if buffer is None:
713 - # We set the buffer to be float32 by default ,refer to:
714 - # https://github.com/triton-lang/triton/issues/1387
715 - buffer = torch.zeros((x.size(0), r),
716 - dtype=torch.float32,
717 - device=x.device)
718 - # LogitsProcessorWithLoRA always using bgmv.
719 - bgmv_shrink(x, lora_a_stacked, buffer, self.sampler_indices, scale)
720 - bgmv_expand(buffer,
721 - lora_b_stacked,
722 - y,
723 - self.sampler_indices,
724 - add_inputs=True)
725 - y = y.view_as(y_org)
vllm/lora/punica_wrapper/__init__.py new
+7
@@ -0,0 +1,7 @@
1 +from vllm.lora.punica_wrapper.punica_base import PunicaWrapperBase
2 +from vllm.lora.punica_wrapper.punica_selector import get_punica_wrapper
3 +
4 +__all__ = [
5 + "PunicaWrapperBase",
6 + "get_punica_wrapper",
7 +]
vllm/lora/punica_wrapper/punica_base.py new
+480
@@ -0,0 +1,480 @@
1 +"""
2 +Based on:
3 +Chen, L., Ye, Z., Wu, Y., Zhuo, D., Ceze, L., & Krishnamurthy, A. (2023).
4 +Punica: Multi-Tenant LoRA Serving.
5 +https://arxiv.org/abs/2310.18547
6 +"""
7 +
8 +from abc import ABC, abstractmethod
9 +from typing import TYPE_CHECKING, List, Optional, Tuple, Union
10 +
11 +import torch
12 +
13 +from .utils import compute_meta, convert_mapping
14 +
15 +if TYPE_CHECKING:
16 + # avoid circuit import
17 + from vllm.lora.layers import LoRAMapping
18 + from vllm.lora.models import LongContextLoRAContext
19 +
20 +
21 +class PunicaWrapperABC(ABC):
22 + """
23 + PunicaWrapper ABC.
24 + """
25 +
26 + @abstractmethod
27 + def update_metadata(
28 + self,
29 + mapping: "LoRAMapping",
30 + lora_index_to_id: List[Optional[int]],
31 + max_loras: int,
32 + vocab_size: int,
33 + extra_vocab_size: int,
34 + long_lora_context: Optional["LongContextLoRAContext"] = None,
35 + **kwargs,
36 + ) -> None:
37 + """
38 + Update the lora-related metadata
39 + """
40 + raise NotImplementedError
41 +
42 + @abstractmethod
43 + def add_shrink(
44 + self,
45 + y: Union[Tuple[torch.Tensor, ...], torch.Tensor],
46 + x: torch.Tensor,
47 + lora_a_stacked: Tuple[torch.Tensor, ...],
48 + scale: float,
49 + **kwargs,
50 + ) -> None:
51 + """
52 + Performs GEMM for multiple slices of lora_a.
53 + """
54 +
55 + raise NotImplementedError
56 +
57 + @abstractmethod
58 + def add_expand(
59 + self,
60 + y: torch.Tensor,
61 + x: Union[Tuple[torch.Tensor, ...], torch.Tensor],
62 + lora_b_stacked: Tuple[torch.Tensor, ...],
63 + lora_bias_stacked: Optional[Tuple[torch.Tensor, ...]],
64 + output_slices: Tuple[int, ...],
65 + offset_start: int = 0,
66 + add_input=True,
67 + **kwargs,
68 + ) -> None:
69 + """
70 + Performs GEMM and bias addition for multiple slices of lora_b.
71 + """
72 + raise NotImplementedError
73 +
74 + @abstractmethod
75 + def add_lora_embedding(
76 + self,
77 + y: torch.Tensor,
78 + x: torch.Tensor,
79 + lora_b_stacked: torch.Tensor,
80 + add_input: bool = True,
81 + **kwargs,
82 + ) -> None:
83 + """
84 + Applies lora specifically for VocabParallelEmbeddingWithLoRA,
85 + and this layer only requires the expand operation.
86 + """
87 + raise NotImplementedError
88 +
89 + @abstractmethod
90 + def add_lora_linear(self,
91 + y: torch.Tensor,
92 + x: torch.Tensor,
93 + lora_a_stacked: Tuple[torch.Tensor, ...],
94 + lora_b_stacked: Tuple[torch.Tensor, ...],
95 + lora_bias_stacked: Optional[Tuple[torch.Tensor, ...]],
96 + scale: float,
97 + output_slices: Tuple[int, ...],
98 + *,
99 + buffer: Optional[Tuple[torch.Tensor, ...]] = None,
100 + **kwargs) -> None:
101 + """
102 + Applicable to linear-related lora.
103 + """
104 +
105 + raise NotImplementedError
106 +
107 + @abstractmethod
108 + def add_lora_logits(self,
109 + y: torch.Tensor,
110 + x: torch.Tensor,
111 + lora_a_stacked: torch.Tensor,
112 + lora_b_stacked: torch.Tensor,
113 + scale,
114 + *,
115 + buffer: Optional[torch.Tensor] = None,
116 + **kwargs) -> None:
117 + """
118 + Applies lora specifically for LogitsProcessorWithLoRA.
119 + """
120 + raise NotImplementedError
121 +
122 +
123 +class PunicaWrapperBase(PunicaWrapperABC):
124 + """
125 + PunicaWrapperBase is designed to manage and provide metadata for the punica
126 + kernel. The main function is to maintain the state information for
127 + Multi-LoRA, and to provide the interface for the punica.
128 + """
129 +
130 + def __init__(self, max_num_batched_tokens: int, max_batches: int,
131 + device: Union[torch.device, str], **kwargs):
132 + self._token_lora_indices = torch.empty(max_num_batched_tokens,
133 + dtype=torch.long,
134 + device=device)
135 + self._sampler_indices = torch.empty(max_num_batched_tokens,
136 + dtype=torch.long,
137 + device=device)
138 + self._sampler_indices_padded = torch.empty(max_num_batched_tokens,
139 + dtype=torch.long,
140 + device=device)
141 + self._embeddings_indices = torch.empty(2,
142 + max_num_batched_tokens,
143 + dtype=torch.long,
144 + device=device)
145 + self._long_lora_indices = torch.empty(max_num_batched_tokens,
146 + dtype=torch.long,
147 + device=device)
148 +
149 + # 5 is the number of indicies tensors.
150 + # base_indices, sampler_indices, sampler_indices_padded,
151 + # embeddings_indices,long_lora_indices
152 + self.indices_len: List[Optional[int]] = [None] * 5
153 + # these attributes are the information required for sgmv kernel
154 + self._seq_start_locs = torch.empty(max_batches,
155 + dtype=torch.long,
156 + device=device)
157 + self._seq_lengths = torch.empty(max_batches,
158 + dtype=torch.long,
159 + device=device)
160 + self._lora_indices_per_batch = torch.empty(max_batches,
161 + dtype=torch.long,
162 + device=device)
163 + self.device: torch.device = device
164 + self.max_length: int = 0
165 + self.token_nums: int = 0
166 + self.batch_size: int = -1
167 + self.is_prefill = False
168 + self.no_lora = False
169 +
170 + def _update_base_metadata(
171 + self,
172 + mapping: "LoRAMapping",
173 + lora_index_to_id: List[Optional[int]],
174 + max_loras: int,
175 + vocab_size: int,
176 + extra_vocab_size: int,
177 + long_lora_context: Optional["LongContextLoRAContext"] = None,
178 + ):
179 + (
180 + base_indices,
181 + sampler_indices,
182 + sampler_indices_padded,
183 + embeddings_indices,
184 + long_lora_offsets_tensor,
185 + indices_len,
186 + ) = convert_mapping(
187 + mapping,
188 + lora_index_to_id,
189 + max_loras,
190 + vocab_size,
191 + extra_vocab_size,
192 + self.device,
193 + long_lora_context,
194 + )
195 + self._token_lora_indices[:base_indices.shape[0]].copy_(base_indices)
196 + self._sampler_indices[:sampler_indices.shape[0]].copy_(sampler_indices)
197 + self._sampler_indices_padded[:sampler_indices_padded.shape[0]].copy_(
198 + sampler_indices_padded)
199 + self._embeddings_indices[:embeddings_indices.
200 + shape[0], :embeddings_indices.shape[1]].copy_(
201 + embeddings_indices)
202 + if long_lora_offsets_tensor is not None:
203 + self._long_lora_indices[:long_lora_offsets_tensor.shape[0]].copy_(
204 + long_lora_offsets_tensor)
205 + else:
206 + self._long_lora_indices.zero_()
207 + self.indices_len[:] = indices_len
208 +
209 + def _update_prefill_metada(self, token_lora_tensor: torch.Tensor) -> None:
210 +
211 + (b_seq_start_tensor, seq_length_tensor, lora_indices_tensor,
212 + batch_size, max_length, token_nums,
213 + no_lora) = compute_meta(token_lora_tensor)
214 +
215 + self._seq_start_locs[:b_seq_start_tensor.shape[0]].copy_(
216 + b_seq_start_tensor)
217 + self._seq_lengths[:seq_length_tensor.shape[0]].copy_(seq_length_tensor)
218 + self._lora_indices_per_batch[:lora_indices_tensor.shape[0]].copy_(
219 + lora_indices_tensor)
220 + self.batch_size = batch_size
221 + self.max_length = max_length
222 + self.token_nums = token_nums
223 + self.no_lora = no_lora
224 +
225 + def _apply_bias(
226 + self,
227 + indices: torch.Tensor,
228 + output: torch.Tensor,
229 + output_slices: Tuple[int, ...],
230 + lora_bias_stacked: Tuple[Optional[torch.Tensor], ...],
231 + ):
232 + """Applies bias to output
233 +
234 + Input shapes:
235 + lora_bias_stacked: 3 element tuple of (num_loras, output_dim)
236 + indices: (batch_size)
237 + output: (batch_size, q_slice_size + 2*kv_slice_size)
238 + output_slices: n-1 element tuple of (slice_size...),
239 + where n is number of slices
240 + """
241 + org_output = output
242 + output = output.view(-1, output.shape[-1])
243 + indices = indices.view(-1)
244 +
245 + offset_left = 0
246 + for slice_idx, slice in enumerate(output_slices):
247 + bias = lora_bias_stacked[slice_idx]
248 + if bias is not None:
249 + bias = bias.view(-1, bias.shape[-1])
250 + bias = bias[indices]
251 + bias[indices == -1] = 0
252 + output[:, offset_left:offset_left + slice] += bias
253 + offset_left += slice
254 +
255 + return output.view_as(org_output)
256 +
257 + @property
258 + def prefill_metadata(
259 + self
260 + ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, int, int, int]:
261 + """
262 + This property provides a convenient way to access the necessary
263 + metadata for prefill-related kernel computations.
264 + 1. seq_start_locs: Tensor of sequence start positions.
265 + 2. seq_lengths: Tensor of sequence lengths.
266 + 3. lora_indices_per_batch: Tensor of lora indices, and an index of
267 + -1 means no lora should be applied.
268 + 4. batch_size: Batch size after clustering identical lora indices.
269 + 5. max_length: The maximum sequence length in the batch.
270 + 6. token_nums: The token numbers in the batch.
271 + """
272 + return (self._seq_start_locs[:self.batch_size],
273 + self._seq_lengths[:self.batch_size],
274 + self._lora_indices_per_batch[:self.batch_size],
275 + self.batch_size, self.max_length, self.token_nums)
276 +
277 + @property
278 + def token_lora_indices(self) -> torch.Tensor:
279 + """
280 + This property provides the lora indices corresponding to each token
281 + in the batch. An index of -1 means no lora should be applied.
282 + """
283 + token_lora_len = self.indices_len[0]
284 + return self._token_lora_indices[:token_lora_len]
285 +
286 + @property
287 + def sampler_indices(self) -> torch.Tensor:
288 + """
289 + This property is used to access the lora indices specifically for
290 + LogitsProcessorWithLoRA.
291 + """
292 + sampler_indices_len = self.indices_len[1]
293 + return self._sampler_indices[:sampler_indices_len]
294 +
295 + @property
296 + def sampler_indices_padded(self) -> torch.Tensor:
297 + """
298 + This property provides access to padded sampler indices.
299 + """
300 + indices_padded_len = self.indices_len[2]
301 + return self._sampler_indices_padded[:indices_padded_len]
302 +
303 + @property
304 + def embeddings_indices(self) -> torch.Tensor:
305 + """
306 + This property provides access to the indices used for lora embeddings,
307 + specifically for VocabParallelEmbeddingWithLoRA.
308 + """
309 + embeddings_indices_len = self.indices_len[3]
310 + return self._embeddings_indices[:, :embeddings_indices_len]
311 +
312 + @property
313 + def long_lora_indices(self) -> torch.Tensor:
314 + """
315 + This property provides access to the indices used for long context
316 + lora, specifically for LinearScalingRotaryEmbeddingWithLora.
317 + """
318 + long_lora_len = self.indices_len[4]
319 + return self._long_lora_indices[:long_lora_len]
320 +
321 + def update_metadata(
322 + self,
323 + mapping: "LoRAMapping",
324 + lora_index_to_id: List[Optional[int]],
325 + max_loras: int,
326 + vocab_size: int,
327 + extra_vocab_size: int,
328 + long_lora_context: Optional["LongContextLoRAContext"] = None,
329 + **kwargs):
330 +
331 + self._update_base_metadata(mapping, lora_index_to_id, max_loras,
332 + vocab_size, extra_vocab_size,
333 + long_lora_context)
334 + if mapping.is_prefill:
335 + # Update metadata required for prefill-related operators.
336 + self._update_prefill_metada(self.token_lora_indices)
337 + self.is_prefill = True
338 + else:
339 + self.is_prefill = False
340 +
341 + @abstractmethod
342 + def add_shrink(self, y: Union[Tuple[torch.Tensor, ...], torch.Tensor],
343 + x: torch.Tensor, lora_a_stacked: Tuple[torch.Tensor, ...],
344 + scale: float, **kwargs) -> None:
345 + """
346 + Performs GEMM for multiple slices of lora_a.
347 +
348 + Semantics:
349 + for i in range(len(lora_a_stacked)):
350 + y[i] += (x @ lora_a_stacked[i]) * scale
351 +
352 + Args:
353 + y (Union[Tuple[torch.Tensor, ...], torch.Tensor]): Output tensors
354 + x (torch.Tensor): Input tensor
355 + lora_a_stacked (Tuple[torch.Tensor, ...]): lora_a's weights
356 + scale (float): Scaling factor for the operation
357 +
358 + """
359 + # TODO: implement it based on torch ops
360 + raise NotImplementedError
361 +
362 + @abstractmethod
363 + def add_expand(self,
364 + y: torch.Tensor,
365 + x: Union[Tuple[torch.Tensor, ...], torch.Tensor],
366 + lora_b_stacked: Tuple[torch.Tensor, ...],
367 + lora_bias_stacked: Optional[Tuple[torch.Tensor, ...]],
368 + output_slices: Tuple[int, ...],
369 + offset_start: int = 0,
370 + add_input=True,
371 + **kwargs) -> None:
372 + """
373 + Performs GEMM and bias addition for multiple slices of lora_b.
374 +
375 + Semantics:
376 + for i in range(len(lora_b_stacked)):
377 + slice = output_slices[i]
378 + y[:, offset:offset+slice] += x[i] @ lora_b_stacked[i] +
379 + lora_bias_stacked[i]
380 + offset += slice
381 +
382 + Args:
383 + y (torch.Tensor): Output tensor.
384 + x (Union[Tuple[torch.Tensor, ...], torch.Tensor]): Input tensors
385 + lora_b_stacked (Tuple[torch.Tensor, ...]): lora_b's weight
386 + lora_bias_stacked (Optional[Tuple[torch.Tensor, ...]]):
387 + bias's weight
388 + output_slices (Tuple[int, ...]): Every slice's size
389 + add_input (bool): Defaults to True.
390 +
391 + """
392 + # TODO: implement it based on torch ops
393 + raise NotImplementedError
394 +
395 + @abstractmethod
396 + def add_lora_embedding(self,
397 + y: torch.Tensor,
398 + x: torch.Tensor,
399 + lora_b_stacked: torch.Tensor,
400 + add_input: bool = True,
401 + **kwargs) -> None:
402 + """
403 + Applies lora specifically for VocabParallelEmbeddingWithLoRA.
404 + and this layer only requires the expand operation.
405 + Semantics:
406 + y += x @ lora_b_stacked
407 +
408 + Args:
409 + y (torch.Tensor): Output tensor.
410 + x (torch.Tensor): Input tensor.
411 + lora_b_stacked (torch.Tensor): lora_b's weights.
412 + add_input (bool): Default to True.
413 + """
414 + # TODO: implement it based on torch ops
415 + raise NotImplementedError
416 +
417 + @abstractmethod
418 + def add_lora_linear(self,
419 + y: torch.Tensor,
420 + x: torch.Tensor,
421 + lora_a_stacked: Tuple[torch.Tensor, ...],
422 + lora_b_stacked: Tuple[torch.Tensor, ...],
423 + lora_bias_stacked: Optional[Tuple[torch.Tensor, ...]],
424 + scale: float,
425 + output_slices: Tuple[int, ...],
426 + *,
427 + buffer: Optional[Tuple[torch.Tensor, ...]] = None,
428 + **kwargs) -> None:
429 + """
430 + Applicable to linear-related lora.
431 +
432 + Semantics:
433 + for i in range(len(lora_a_stacked)):
434 + y[i] += (
435 + x[i].unsqueeze(0)
436 + @ lora_a_stacked[indices[i], layer_idx, :, :]
437 + @ lora_b_stacked[indices[i], layer_idx, :, :]
438 + * scale
439 + ).squeeze(0)+lora_bias_stacked[i]
440 +
441 + Args:
442 + y (torch.Tensor): Output tensor. Will be changed in-place.
443 + x (torch.Tensor): Input tensor
444 + lora_a_stacked (Tuple[torch.Tensor, ...]): lora_a's weight.
445 + lora_b_stacked (Tuple[torch.Tensor, ...]): lora_b's weight.
446 + lora_bias_stacked (Optional[Tuple[torch.Tensor, ...]]): lora's bias.
447 + scale (float): Scaling factor.
448 + output_slices (Tuple[int, ...]): Every slice's size.
449 + buffer (Optional[Tuple[torch.Tensor, ...]]): Defaults to None.
450 + """
451 + # TODO: implement it based on torch ops
452 + raise NotImplementedError
453 +
454 + @abstractmethod
455 + def add_lora_logits(self,
456 + y: torch.Tensor,
457 + x: torch.Tensor,
458 + lora_a_stacked: torch.Tensor,
459 + lora_b_stacked: torch.Tensor,
460 + scale,
461 + *,
462 + buffer: Optional[torch.Tensor] = None,
463 + **kwargs) -> None:
464 + """
465 + Applies lora specifically for LogitsProcessorWithLoRA.
466 +
467 + Semantics:
468 + buffer = (x @ lora_a_stacked) * scale
469 + y += buffer @ lora_b_stacked
470 +
471 + Args:
472 + y (torch.Tensor): Output tensor.
473 + x (torch.Tensor): Input tensor.
474 + lora_a_stacked (torch.Tensor): lora_a's weights.
475 + lora_b_stacked (torch.Tensor):lora_b's weights.
476 + scale (float): Scaling factor.
477 + buffer (Optional[torch.Tensor]):Default to None.
478 + """
479 + # TODO: implement it based on torch ops
480 + raise NotImplementedError
vllm/lora/punica_wrapper/punica_gpu.py new
+358
@@ -0,0 +1,358 @@
1 +"""
2 +Based on:
3 +Chen, L., Ye, Z., Wu, Y., Zhuo, D., Ceze, L., & Krishnamurthy, A. (2023).
4 +Punica: Multi-Tenant LoRA Serving.
5 +https://arxiv.org/abs/2310.18547
6 +"""
7 +
8 +from typing import Callable, Optional, Tuple, Union, final
9 +
10 +import torch
11 +
12 +from vllm.triton_utils import HAS_TRITON
13 +
14 +if HAS_TRITON:
15 + from vllm.lora.ops.bgmv_expand import bgmv_expand
16 + from vllm.lora.ops.bgmv_expand_slice import bgmv_expand_slice
17 + from vllm.lora.ops.bgmv_shrink import bgmv_shrink
18 + from vllm.lora.ops.sgmv_expand import sgmv_expand
19 + from vllm.lora.ops.sgmv_expand_slice import sgmv_expand_slice
20 + from vllm.lora.ops.sgmv_shrink import sgmv_shrink
21 +
22 +from .punica_base import PunicaWrapperBase
23 +
24 +
25 +@final
26 +class PunicaWrapperGPU(PunicaWrapperBase):
27 + """
28 + PunicaWrapperGPU is designed to manage and provide metadata for the punica
29 + kernel. The main function is to maintain the state information for
30 + Multi-LoRA, and to provide the interface for the punica triton kernel.
31 + """
32 +
33 + def __init__(self, max_num_batched_tokens: int, max_batches: int,
34 + device: Union[torch.device, str], **kwargs):
35 + PunicaWrapperBase.__init__(self, max_num_batched_tokens, max_batches,
36 + device)
37 +
38 + def _shrink_prefill(
39 + self,
40 + y: torch.Tensor,
41 + x: torch.Tensor,
42 + w_t_all: torch.Tensor,
43 + scale: float,
44 + ):
45 + #No LoRA request, so return directly
46 + if self.no_lora:
47 + return
48 + sgmv_shrink(
49 + x,
50 + w_t_all,
51 + y,
52 + *self.prefill_metadata,
53 + scale,
54 + )
55 +
56 + def _shrink_decode(
57 + self,
58 + y: torch.Tensor,
59 + x: torch.Tensor,
60 + w_t_all: torch.Tensor,
61 + scale: float,
62 + ):
63 + bgmv_shrink(x, w_t_all, y, self.token_lora_indices, scale)
64 +
65 + def _expand_prefill(
66 + self,
67 + y: torch.Tensor,
68 + x: torch.Tensor,
69 + w_t_all: torch.Tensor,
70 + add_input: bool,
71 + ):
72 + #No LoRA request, so return directly
73 + if self.no_lora:
74 + return
75 + sgmv_expand(
76 + x,
77 + w_t_all,
78 + y,
79 + *self.prefill_metadata,
80 + add_input,
81 + )
82 +
83 + def _expand_decode(
84 + self,
85 + y: torch.Tensor,
86 + x: torch.Tensor,
87 + w_t_all: torch.Tensor,
88 + add_input: bool,
89 + ):
90 + bgmv_expand(x, w_t_all, y, self.token_lora_indices, add_input)
91 +
92 + def _expand_slice_prefill(
93 + self,
94 + y: torch.Tensor,
95 + x: torch.Tensor,
96 + w_t_all: torch.Tensor,
97 + y_offset: Optional[int],
98 + y_slice_size: Optional[int],
99 + add_input: bool,
100 + ):
101 + #No LoRA request, so return directly
102 + if self.no_lora:
103 + return
104 + sgmv_expand_slice(
105 + x,
106 + w_t_all,
107 + y,
108 + *self.prefill_metadata,
109 + y_offset,
110 + y_slice_size,
111 + add_input,
112 + )
113 +
114 + def _expand_slice_decode(
115 + self,
116 + y: torch.Tensor,
117 + x: torch.Tensor,
118 + w_t_all: torch.Tensor,
119 + y_offset: Optional[int],
120 + y_slice_size: Optional[int],
121 + add_input: bool,
122 + ):
123 + bgmv_expand_slice(x, w_t_all, y, self.token_lora_indices, y_offset,
124 + y_slice_size, add_input)
125 +
126 + def _apply_expand(
127 + self,
128 + y: torch.Tensor,
129 + x: torch.Tensor,
130 + w_t_all: torch.Tensor,
131 + y_offset: Optional[int],
132 + y_slice_size: Optional[int],
133 + add_input: bool = True,
134 + ):
135 + """
136 + Perform the ` y[:,y_offset:y_offset+y_slice_size]+=x@w_t_all`
137 + computation, which is suitable for the
138 + GEMM of lora'b.
139 + """
140 +
141 + expand_slice_fun: Callable = (self._expand_slice_prefill
142 + if self.is_prefill else
143 + self._expand_slice_decode)
144 + expand_slice_fun(y, x, w_t_all, y_offset, y_slice_size, add_input)
145 +
146 + def _apply_shrink(self, y: torch.Tensor, x: torch.Tensor,
147 + w_t_all: torch.Tensor, scale: float):
148 + """
149 + Perform the ` y+=x@w_t_all` computation, which is suitable for the
150 + GEMM of lora'a.
151 + When `is_prefill is` true, it indicates that it is currently the
152 + prefill stage, and the `_shrink_prefill` function should be called.
153 + Otherwise, it is the decode stage, and the _shrink_decode function
154 + should be called.
155 + """
156 + y_org = y
157 + y = y.view(-1, y.shape[-1])
158 + shrink_fun: Callable = (self._shrink_prefill
159 + if self.is_prefill else self._shrink_decode)
160 + shrink_fun(y, x, w_t_all, scale)
161 + y = y.view_as(y_org)
162 +
163 + def add_shrink(self, y: Union[Tuple[torch.Tensor, ...], torch.Tensor],
164 + x: torch.Tensor, lora_a_stacked: Tuple[torch.Tensor, ...],
165 + scale: float, **kwargs):
166 + """
167 + Performs GEMM for multiple slices of lora_a.
168 + When `is_prefill is` true, it indicates that it is currently the
169 + prefill stage, and the `_shrink_prefill` function should be called.
170 + Otherwise, it is the decode stage, and the _shrink_decode function
171 + should be called.
172 +
173 + Semantics:
174 + for i in range(len(lora_a_stacked)):
175 + y[i] += (x @ lora_a_stacked[i]) * scale
176 +
177 + Args:
178 + y (Union[Tuple[torch.Tensor, ...], torch.Tensor]): Output tensors
179 + x (torch.Tensor): Input tensor
180 + lora_a_stacked (Tuple[torch.Tensor, ...]): lora_a's weights
181 + scale (float): Scaling factor for the operation
182 + """
183 +
184 + x = x.view(-1, x.shape[-1])
185 + # TODO fuse these kernels
186 + for slice_idx in range(len(lora_a_stacked)):
187 + self._apply_shrink(y[slice_idx], x, lora_a_stacked[slice_idx],
188 + scale)
189 +
190 + def add_expand(self,
191 + y: torch.Tensor,
192 + x: Union[Tuple[torch.Tensor, ...], torch.Tensor],
193 + lora_b_stacked: Tuple[torch.Tensor, ...],
194 + lora_bias_stacked: Optional[Tuple[torch.Tensor, ...]],
195 + output_slices: Tuple[int, ...],
196 + offset_start: int = 0,
197 + add_input=True,
198 + **kwargs) -> None:
199 + """
200 + Performs GEMM and bias addition for multiple slices of lora_b.
201 +
202 + Semantics:
203 + for i in range(len(lora_b_stacked)):
204 + slice = output_slices[i]
205 + y[:, offset:offset+slice] += x[i] @ lora_b_stacked[i] +
206 + lora_bias_stacked[i]
207 + offset += slice
208 +
209 + Args:
210 + y (torch.Tensor): Output tensor.
211 + x (Union[Tuple[torch.Tensor, ...], torch.Tensor]): Input tensors
212 + lora_b_stacked (Tuple[torch.Tensor, ...]): lora_b's weight
213 + lora_bias_stacked (Optional[Tuple[torch.Tensor, ...]]):
214 + bias's weight
215 + output_slices (Tuple[int, ...]): Every slice's size
216 + add_input (bool): Defaults to True.
217 + """
218 + y_org = y
219 + y = y.view(-1, y.shape[-1])
220 + offset_left = offset_start
221 + if lora_bias_stacked is not None:
222 + self._apply_bias(self.token_lora_indices, y, output_slices,
223 + lora_bias_stacked)
224 + for slice_idx in range(len(lora_b_stacked)):
225 + self._apply_expand(
226 + y,
227 + x[slice_idx],
228 + lora_b_stacked[slice_idx],
229 + offset_left,
230 + output_slices[slice_idx],
231 + add_input=add_input,
232 + )
233 + offset_left += output_slices[slice_idx]
234 + y = y.view_as(y_org)
235 +
236 + def add_lora_embedding(self,
237 + y: torch.Tensor,
238 + x: torch.Tensor,
239 + lora_b_stacked: torch.Tensor,
240 + add_input: bool = True,
241 + **kwargs) -> None:
242 + """
243 + Applies lora specifically for VocabParallelEmbeddingWithLoRA.
244 +
245 + Semantics:
246 + y += x @ lora_b_stacked
247 +
248 + Args:
249 + y (torch.Tensor): Output tensor.
250 + x (torch.Tensor): Input tensor.
251 + lora_b_stacked (torch.Tensor): lora_b's weights.
252 + add_input (bool): Default to True.
253 + """
254 +
255 + # Embedding layer only need expand op
256 + expand_fun: Callable = (self._expand_prefill
257 + if self.is_prefill else self._expand_decode)
258 + expand_fun(y, x, lora_b_stacked, add_input)
259 +
260 + def add_lora_linear(self,
261 + y: torch.Tensor,
262 + x: torch.Tensor,
263 + lora_a_stacked: Tuple[torch.Tensor, ...],
264 + lora_b_stacked: Tuple[torch.Tensor, ...],
265 + lora_bias_stacked: Optional[Tuple[torch.Tensor, ...]],
266 + scale: float,
267 + output_slices: Tuple[int, ...],
268 + *,
269 + buffer: Optional[Tuple[torch.Tensor, ...]] = None,
270 + **kwargs) -> None:
271 + """
272 + Applicable to linear-related lora.
273 +
274 + Semantics:
275 + for i in range(len(lora_a_stacked)):
276 + y[i] += (
277 + x[i].unsqueeze(0)
278 + @ lora_a_stacked[indices[i], layer_idx, :, :]
279 + @ lora_b_stacked[indices[i], layer_idx, :, :]
280 + * scale
281 + ).squeeze(0)+lora_bias_stacked[i]
282 +
283 + Args:
284 + y (torch.Tensor): Output tensor. Will be changed in-place.
285 + x (torch.Tensor): Input tensor
286 + lora_a_stacked (Tuple[torch.Tensor, ...]): lora_a's weight.
287 + lora_b_stacked (Tuple[torch.Tensor, ...]): lora_b's weight.
288 + lora_bias_stacked (Optional[Tuple[torch.Tensor, ...]]): lora's bias.
289 + scale (float): Scaling factor.
290 + output_slices (Tuple[int, ...]): Every slice's size.
291 + buffer (Optional[Tuple[torch.Tensor, ...]]): Defaults to None.
292 + """
293 +
294 + assert len(lora_a_stacked) == len(lora_b_stacked) == len(output_slices)
295 + if lora_bias_stacked is not None:
296 + assert len(lora_bias_stacked) == len(output_slices)
297 + y = self._apply_bias(self.token_lora_indices, y, output_slices,
298 + lora_bias_stacked)
299 +
300 + if buffer is None:
301 + r = lora_b_stacked[0].size(-1)
302 + # We set the buffer to be float32 by default ,refer to:
303 + # https://github.com/triton-lang/triton/issues/1387
304 + buffer = tuple(
305 + torch.zeros(
306 + (x.size(0), r), dtype=torch.float32, device=x.device)
307 + for _ in range(len(output_slices)))
308 + self.add_shrink(buffer, x, lora_a_stacked, scale, **kwargs)
309 + self.add_expand(y,
310 + buffer,
311 + lora_b_stacked,
312 + None,
313 + output_slices,
314 + add_input=True,
315 + **kwargs)
316 +
317 + def add_lora_logits(self,
318 + y: torch.Tensor,
319 + x: torch.Tensor,
320 + lora_a_stacked: torch.Tensor,
321 + lora_b_stacked: torch.Tensor,
322 + scale,
323 + *,
324 + buffer: Optional[torch.Tensor] = None,
325 + **kwargs) -> None:
326 + """
327 + Applies lora specifically for LogitsProcessorWithLoRA.
328 +
329 + Semantics:
330 + buffer = (x @ lora_a_stacked) * scale
331 + y += buffer @ lora_b_stacked
332 +
333 + Args:
334 + y (torch.Tensor): Output tensor.
335 + x (torch.Tensor): Input tensor.
336 + lora_a_stacked (torch.Tensor): lora_a's weights.
337 + lora_b_stacked (torch.Tensor):lora_b's weights.
338 + scale (float): Scaling factor.
339 + buffer (Optional[torch.Tensor]):Default to None.
340 + """
341 + y_org = y
342 + y = y.view(-1, y.shape[-1])
343 + x = x.view(-1, x.shape[-1])
344 + r = lora_b_stacked.size(-1)
345 + if buffer is None:
346 + # We set the buffer to be float32 by default ,refer to:
347 + # https://github.com/triton-lang/triton/issues/1387
348 + buffer = torch.zeros((x.size(0), r),
349 + dtype=torch.float32,
350 + device=x.device)
351 + # LogitsProcessorWithLoRA always using bgmv.
352 + bgmv_shrink(x, lora_a_stacked, buffer, self.sampler_indices, scale)
353 + bgmv_expand(buffer,
354 + lora_b_stacked,
355 + y,
356 + self.sampler_indices,
357 + add_inputs=True)
358 + y = y.view_as(y_org)
vllm/lora/punica_wrapper/punica_selector.py new
+14
@@ -0,0 +1,14 @@
1 +from vllm.platforms import current_platform
2 +from vllm.utils import print_info_once
3 +
4 +from .punica_base import PunicaWrapperBase
5 +
6 +
7 +def get_punica_wrapper(*args, **kwargs) -> PunicaWrapperBase:
8 + if current_platform.is_cuda_alike():
9 + # Lazy import to avoid ImportError
10 + from vllm.lora.punica_wrapper.punica_gpu import PunicaWrapperGPU
11 + print_info_once("Using PunicaWrapperGPU.")
12 + return PunicaWrapperGPU(*args, **kwargs)
13 + else:
14 + raise NotImplementedError
vllm/lora/punica_wrapper/utils.py new
+159
@@ -0,0 +1,159 @@
1 +from typing import TYPE_CHECKING, List, Optional, Tuple, Union
2 +
3 +import torch
4 +
5 +if TYPE_CHECKING:
6 + # avoid circuit import
7 + from vllm.lora.layers import LoRAMapping
8 + from vllm.lora.models import LongContextLoRAContext
9 +
10 +
11 +def compute_meta(
12 + token_lora_tensor: torch.Tensor
13 +) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, int, int, int, bool]:
14 + """
15 + Get the information required for the sgmv kernel. With the features:
16 + 1. If consecutive requests in the batch use the same LoRA, this function
17 + will combine them into a single request, improving sgmv kernel inference
18 + performance.
19 + 2. At the beginning of each prefill stage inference, recalculations are
20 + needed based on the input, but only once.
21 + """
22 +
23 + lora_indices_tensor, seq_length_tensor = torch.unique_consecutive(
24 + token_lora_tensor, return_counts=True)
25 + cum_result = torch.cumsum(seq_length_tensor, dim=0)
26 + b_seq_start_tensor = torch.zeros_like(seq_length_tensor)
27 + b_seq_start_tensor[1:].copy_(cum_result[:-1])
28 + max_length = seq_length_tensor.max().item()
29 + token_nums = seq_length_tensor.sum().item()
30 + batch_size = lora_indices_tensor.size(0)
31 + no_lora = False
32 + # -1 means no lora should be applied. Use `no_lora` to determine whether
33 + # the current step requires LoRA. If LoRA is not needed, the prefill stage
34 + # does not need to launch the triton kernel, which can improve performance
35 + if batch_size == 1 and lora_indices_tensor == -1:
36 + no_lora = True
37 + return (b_seq_start_tensor, seq_length_tensor, lora_indices_tensor,
38 + batch_size, max_length, token_nums, no_lora)
39 +
40 +
41 +# TODO see if this can be vectorized
42 +def convert_mapping(
43 + mapping: "LoRAMapping",
44 + lora_index_to_id: List[Optional[int]],
45 + max_loras: int,
46 + vocab_size: int,
47 + extra_vocab_size: int,
48 + device: torch.device,
49 + long_lora_context: Optional["LongContextLoRAContext"] = None,
50 +) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor,
51 + Optional[torch.Tensor], List[int]]:
52 + """Converts LoRAMapping to index tensors.
53 +
54 + Args:
55 + mapping: LoRAMapping mapping rows in a batch to LoRA ids.
56 + lora_index_to_id: List mapping LoRA ids to LoRA indices.
57 + max_loras: Maximum number of LoRAs.
58 + vocab_size: Model vocab size.
59 + extra_vocab_size: Extra vocab size each LoRA can have.
60 + long_lora_context: Passed if there are long context lora in a batch.
61 +
62 + Returns:
63 + A tuple of tensors:
64 + base_indices: Tensor of shape [batch_size] mapping batch rows to
65 + LoRA indices.
66 + sampler_indices: Tensor of shape [batch_size] mapping requests to
67 + LoRA indices for sampler. For generation, this will be the
68 + same as base_indicies. For prefill, this will map requests
69 + to LoRA indices.
70 + sampler_indices_padded: Tensor of shape [batch_size] mapping
71 + requests to LoRA indices for sampler with padding.
72 + Same as sampler_indicies, but -1 is replaced with
73 + max_loras.
74 + embeddings_indices: Tensor of shape [2, batch_size] mapping
75 + requests to embedding indices. First row is for embeddings
76 + added by the LoRAs, second row is for the LoRA.lora_a
77 + embeddings.
78 + long_lora_indices: Tensor of shape [batch_size] mapping
79 + requests to RoPE offsets and rot dims for long LoRAs.
80 + None if long context lora doesn't exist.
81 + indices_len: List of lengths of the above tensors. It contains
82 + (base_indices, sampler_indices, sampler_indices_padded,
83 + embeddings_indices, long_lora_indices).
84 + """
85 + index_mapping_indices: List[int] = list(mapping.index_mapping).copy()
86 + embedding_indices = index_mapping_indices.copy()
87 + lora_indices = index_mapping_indices.copy()
88 + long_lora_offsets: Optional[torch.Tensor] = None
89 + if long_lora_context:
90 + long_lora_offsets = torch.zeros(len(index_mapping_indices),
91 + device=device,
92 + dtype=torch.long)
93 + prompt_mapping: List[int] = [
94 + lora_index_to_id.index(x) if x > 0 else -1
95 + for x in mapping.prompt_mapping
96 + ]
97 + lora_idx = None
98 + for i in range(len(index_mapping_indices)):
99 + # TODO index can be slow. optimize
100 + lora_idx = (lora_index_to_id.index(index_mapping_indices[i])
101 + if index_mapping_indices[i] > 0 else -1)
102 + embedding_indices[i] = lora_idx if index_mapping_indices[i] > 0 else 0
103 + lora_indices[i] = lora_idx
104 + if long_lora_context:
105 + assert long_lora_offsets is not None
106 + lora_offset: int = long_lora_context.offsets_by_lora_id.get(
107 + index_mapping_indices[i], 0)
108 + long_lora_offsets[i] = lora_offset
109 +
110 + indices_list: List[Union[List[int], torch.Tensor]] = [
111 + index_mapping_indices,
112 + lora_indices,
113 + embedding_indices,
114 + ]
115 + if long_lora_context:
116 + assert long_lora_offsets is not None
117 + indices_list.append(long_lora_offsets)
118 + indices = torch.tensor(indices_list, dtype=torch.long, device=device)
119 + prompt_mapping_tensor = torch.tensor(prompt_mapping,
120 + dtype=torch.long,
121 + device=device)
122 + embeddings_indices = torch.stack([
123 + indices[2] * extra_vocab_size,
124 + indices[2] * (vocab_size + extra_vocab_size),
125 + ])
126 + embeddings_indices[embeddings_indices == -1] = max_loras - 1
127 + base_indices = indices[1]
128 + sampler_indices = prompt_mapping_tensor
129 + sampler_indices_padded = sampler_indices.clone()
130 + sampler_indices_padded[sampler_indices_padded == -1] = max_loras - 1
131 + sampler_indices_padded = torch.arange(
132 + 0, len(sampler_indices_padded), device=device, dtype=torch.long) + (
133 + sampler_indices_padded * len(sampler_indices_padded))
134 + long_lora_indices = None
135 + long_lora_indices_len: Optional[int] = None
136 + if long_lora_context:
137 + long_lora_indices = indices[3]
138 + long_lora_indices_len = long_lora_indices.shape[-1]
139 + # Contain length of indices tensors. Used to index into each tensor.
140 + indices_len = [
141 + base_indices.shape[-1],
142 + sampler_indices.shape[-1],
143 + sampler_indices_padded.shape[-1],
144 + embeddings_indices.shape[-1],
145 + ]
146 + if long_lora_indices_len is not None:
147 + indices_len.append(long_lora_indices_len)
148 + else:
149 + # If long_lora doesn't exist,append None
150 + indices_len.append(None)
151 +
152 + return (
153 + base_indices,
154 + sampler_indices,
155 + sampler_indices_padded,
156 + embeddings_indices,
157 + long_lora_indices,
158 + indices_len,
159 + )