A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 05:45 UTC
[Misc][LoRA] Abstract PunicaWrapper (#10955)
Signed-off-by: Jee Jee Li <pandaleefree@gmail.com>
Jee Jee Li committed
Dec 10, 2024 at 04:54 UTC
ca871491edb0fba11fe9aa94300bd8d282fa29e1
9 files changed
+1058
-749
tests/lora/test_layers.py
+33
-16
@@ -28,7 +28,7 @@ from vllm.lora.layers import (BaseLayerWithLoRA, ColumnParallelLinearWithLoRA,
28
# yapf: enable
29
from vllm.lora.models import (LongContextLoRAContext, LoRALayerWeights,
30
PackedLoRALayerWeights)
31
-from vllm.lora.punica import PunicaWrapper
31
+from vllm.lora.punica_wrapper import get_punica_wrapper
32
from vllm.model_executor.layers.linear import (ColumnParallelLinear,
33
MergedColumnParallelLinear,
34
QKVParallelLinear,
@@ -48,11 +48,12 @@ TOLERANCES = {
48
torch.float32: (5e-3, 5e-3),
49
torch.bfloat16: (3e-2, 2e-2),
50
}
51
-CUDA_DEVICES = [
51
+# TODO: Modify this based on platform
52
+DEVICES = [
53
f"cuda:{i}" for i in range(1 if torch.cuda.device_count() == 1 else 2)
54
]
55
55
-# We will launch different triton kernels between the prefill and decode
56
+#For GPU, we will launch different triton kernels between the prefill and decode
57
# stages, so we need to verify this. prefill stage(True) or decode stage(False)
58
STAGES = [True, False]
59
@@ -192,9 +193,18 @@ def create_random_inputs(
193
return inputs, index_mapping, prompt_mapping
194
195
196
+def check_punica_wrapper(punica_wrapper) -> bool:
197
+ if current_platform.is_cuda_alike():
198
+ from vllm.lora.punica_wrapper.punica_gpu import PunicaWrapperGPU
199
+
200
+ return type(punica_wrapper) is PunicaWrapperGPU
201
+ else:
202
+ return False
203
+
204
+
205
@torch.inference_mode()
206
@pytest.mark.parametrize("num_loras", [1, 2, 4, 8])
197
-@pytest.mark.parametrize("device", CUDA_DEVICES)
207
+@pytest.mark.parametrize("device", DEVICES)
208
@pytest.mark.parametrize("vocab_size", [512, 32000, 64000, 128000])
209
@pytest.mark.parametrize("stage", STAGES)
210
def test_embeddings(dist_init, num_loras, device, vocab_size, stage) -> None:
@@ -205,7 +215,8 @@ def test_embeddings(dist_init, num_loras, device, vocab_size, stage) -> None:
215
216
torch.set_default_device(device)
217
max_loras = 8
208
- punica_wrapper = PunicaWrapper(8192, 256, device)
218
+ punica_wrapper = get_punica_wrapper(8192, 256, device)
219
+ assert check_punica_wrapper(punica_wrapper)
220
lora_config = LoRAConfig(max_loras=max_loras,
221
max_lora_rank=8,
222
lora_dtype=torch.float16)
@@ -296,7 +307,7 @@ def test_embeddings(dist_init, num_loras, device, vocab_size, stage) -> None:
307
# @pytest.mark.skip(
308
# reason="Fails when loras are in any slot other than the first.")
309
@pytest.mark.parametrize("num_loras", [1, 2, 4, 8])
299
-@pytest.mark.parametrize("device", CUDA_DEVICES)
310
+@pytest.mark.parametrize("device", DEVICES)
311
@pytest.mark.parametrize("vocab_size", [512, 32000, 64000, 128000])
312
@pytest.mark.parametrize("stage", STAGES)
313
def test_embeddings_with_new_embeddings(dist_init, num_loras, device,
@@ -305,7 +316,8 @@ def test_embeddings_with_new_embeddings(dist_init, num_loras, device,
316
torch.cuda.set_device(device)
317
torch.set_default_device(device)
318
max_loras = 8
308
- punica_wrapper = PunicaWrapper(8192, 256, device)
319
+ punica_wrapper = get_punica_wrapper(8192, 256, device)
320
+ assert check_punica_wrapper(punica_wrapper)
321
lora_config = LoRAConfig(max_loras=max_loras,
322
max_lora_rank=8,
323
lora_dtype=torch.float16)
@@ -432,7 +444,7 @@ def test_embeddings_with_new_embeddings(dist_init, num_loras, device,
444
445
@torch.inference_mode()
446
@pytest.mark.parametrize("num_loras", [1, 2, 4, 8])
435
-@pytest.mark.parametrize("device", CUDA_DEVICES)
447
+@pytest.mark.parametrize("device", DEVICES)
448
@pytest.mark.parametrize("vocab_size", [512, 32000, 64000, 256512])
449
@pytest.mark.parametrize("stage", STAGES)
450
def test_lm_head_logits_processor(dist_init, num_loras, device, vocab_size,
@@ -441,7 +453,8 @@ def test_lm_head_logits_processor(dist_init, num_loras, device, vocab_size,
453
torch.cuda.set_device(device)
454
torch.set_default_device(device)
455
max_loras = 8
444
- punica_wrapper = PunicaWrapper(8192, 256, device)
456
+ punica_wrapper = get_punica_wrapper(8192, 256, device)
457
+ assert check_punica_wrapper(punica_wrapper)
458
lora_config = LoRAConfig(max_loras=max_loras,
459
max_lora_rank=8,
460
lora_dtype=torch.float16)
@@ -563,7 +576,7 @@ def test_lm_head_logits_processor(dist_init, num_loras, device, vocab_size,
576
577
@torch.inference_mode()
578
@pytest.mark.parametrize("num_loras", [1, 2, 4, 8])
566
-@pytest.mark.parametrize("device", CUDA_DEVICES)
579
+@pytest.mark.parametrize("device", DEVICES)
580
@pytest.mark.parametrize("stage", STAGES)
581
@pytest.mark.parametrize("bias_enabled", [True, False])
582
def test_linear_replicated(dist_init, num_loras, device, stage,
@@ -571,7 +584,8 @@ def test_linear_replicated(dist_init, num_loras, device, stage,
584
585
torch.cuda.set_device(device)
586
torch.set_default_device(device)
574
- punica_wrapper = PunicaWrapper(8192, 256, device)
587
+ punica_wrapper = get_punica_wrapper(8192, 256, device)
588
+ assert check_punica_wrapper(punica_wrapper)
589
max_loras = 8
590
lora_config = LoRAConfig(max_loras=max_loras,
591
max_lora_rank=8,
@@ -675,7 +689,7 @@ def test_linear_replicated(dist_init, num_loras, device, stage,
689
@pytest.mark.parametrize("num_loras", [1, 2, 4, 8])
690
@pytest.mark.parametrize("orientation", ["row", "column"])
691
@pytest.mark.parametrize("fully_shard", [True, False])
678
-@pytest.mark.parametrize("device", CUDA_DEVICES)
692
+@pytest.mark.parametrize("device", DEVICES)
693
@pytest.mark.parametrize("stage", STAGES)
694
@pytest.mark.parametrize("bias_enabled", [True, False])
695
def test_linear_parallel(dist_init, num_loras, orientation, fully_shard,
@@ -683,7 +697,8 @@ def test_linear_parallel(dist_init, num_loras, orientation, fully_shard,
697
698
torch.cuda.set_device(device)
699
torch.set_default_device(device)
686
- punica_wrapper = PunicaWrapper(8192, 256, device)
700
+ punica_wrapper = get_punica_wrapper(8192, 256, device)
701
+ assert check_punica_wrapper(punica_wrapper)
702
max_loras = 8
703
lora_config = LoRAConfig(max_loras=max_loras,
704
max_lora_rank=8,
@@ -797,7 +812,7 @@ def test_linear_parallel(dist_init, num_loras, orientation, fully_shard,
812
@pytest.mark.parametrize("num_loras", [1, 2, 4, 8])
813
@pytest.mark.parametrize("repeats", [1, 2, 3])
814
@pytest.mark.parametrize("fully_shard", [True, False])
800
-@pytest.mark.parametrize("device", CUDA_DEVICES)
815
+@pytest.mark.parametrize("device", DEVICES)
816
@pytest.mark.parametrize("stage", STAGES)
817
@pytest.mark.parametrize("bias_enabled", [True, False])
818
def test_column_parallel_packed(dist_init, num_loras, repeats, fully_shard,
@@ -805,7 +820,8 @@ def test_column_parallel_packed(dist_init, num_loras, repeats, fully_shard,
820
821
torch.cuda.set_device(device)
822
torch.set_default_device(device)
808
- punica_wrapper = PunicaWrapper(8192, 256, device)
823
+ punica_wrapper = get_punica_wrapper(8192, 256, device)
824
+ assert check_punica_wrapper(punica_wrapper)
825
max_loras = 8
826
lora_config = LoRAConfig(max_loras=max_loras,
827
max_lora_rank=8,
@@ -963,7 +979,8 @@ def test_rotary_embedding_long_context(dist_init, num_loras, device,
979
seed = 0
980
current_platform.seed_everything(seed)
981
torch.set_default_device(device)
966
- punica_wrapper = PunicaWrapper(8192, 256, device)
982
+ punica_wrapper = get_punica_wrapper(8192, 256, device)
983
+ assert check_punica_wrapper(punica_wrapper)
984
max_loras = 8
985
lora_config = LoRAConfig(max_loras=max_loras,
986
max_lora_rank=8,
vllm/lora/layers.py
+3
-4
@@ -17,7 +17,6 @@ from vllm.distributed import (get_tensor_model_parallel_rank,
17
tensor_model_parallel_all_reduce,
18
tensor_model_parallel_gather)
19
from vllm.distributed.utils import divide
20
-from vllm.lora.punica import PunicaWrapper
20
# yapf: disable
21
from vllm.model_executor.layers.linear import (ColumnParallelLinear,
22
LinearBase,
@@ -33,7 +32,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
32
VocabParallelEmbedding)
33
34
if TYPE_CHECKING:
36
- pass
35
+ from vllm.lora.punica_wrapper import PunicaWrapperBase
36
37
38
def _get_lora_device(base_layer: nn.Module) -> torch.device:
@@ -115,9 +114,9 @@ class BaseLayerWithLoRA(nn.Module):
114
115
def set_mapping(
116
self,
118
- punica_wrapper: PunicaWrapper,
117
+ punica_wrapper,
118
):
120
- self.punica_wrapper: PunicaWrapper = punica_wrapper
119
+ self.punica_wrapper: PunicaWrapperBase = punica_wrapper
120
121
@classmethod
122
def can_replace_layer(
vllm/lora/models.py
+4
-4
@@ -21,7 +21,7 @@ from vllm.lora.layers import (BaseLayerWithLoRA,
21
LinearScalingRotaryEmbeddingWithLora,
22
LoRAMapping)
23
from vllm.lora.lora import LoRALayerWeights, PackedLoRALayerWeights
24
-from vllm.lora.punica import PunicaWrapper
24
+from vllm.lora.punica_wrapper import get_punica_wrapper
25
from vllm.lora.utils import (from_layer, from_layer_logits_processor,
26
is_regex_target_modules,
27
parse_fine_tuned_lora_name, replace_submodule)
@@ -331,9 +331,9 @@ class LoRAModelManager(AdapterModelManager):
331
self.lora_index_to_id: List[Optional[int]] = [None] * self.lora_slots
332
self.vocab_size = vocab_size
333
self.long_lora_context: Optional[LongContextLoRAContext] = None
334
- self.punica_wrapper = PunicaWrapper(max_num_batched_tokens,
335
- max_batches=self.max_num_seqs,
336
- device=self.device)
334
+ self.punica_wrapper = get_punica_wrapper(max_num_batched_tokens,
335
+ max_batches=self.max_num_seqs,
336
+ device=self.device)
337
# Scaling factor -> offset to the sin_cos_cache to it.
338
# Used for long context lora.
339
self.scaling_factor_to_offset: Dict[float, int] = {}
vllm/lora/punica.py
deleted
-725
@@ -1,725 +0,0 @@
1
-"""
2
-Based on:
3
-Chen, L., Ye, Z., Wu, Y., Zhuo, D., Ceze, L., & Krishnamurthy, A. (2023).
4
-Punica: Multi-Tenant LoRA Serving.
5
-https://arxiv.org/abs/2310.18547
6
-"""
7
-
8
-from typing import TYPE_CHECKING, Callable, List, Optional, Tuple, Union
9
-
10
-import torch
11
-
12
-from vllm.triton_utils import HAS_TRITON
13
-
14
-if HAS_TRITON:
15
- from vllm.lora.ops.bgmv_expand import bgmv_expand
16
- from vllm.lora.ops.bgmv_expand_slice import bgmv_expand_slice
17
- from vllm.lora.ops.bgmv_shrink import bgmv_shrink
18
- from vllm.lora.ops.sgmv_expand import sgmv_expand
19
- from vllm.lora.ops.sgmv_expand_slice import sgmv_expand_slice
20
- from vllm.lora.ops.sgmv_shrink import sgmv_shrink
21
-
22
-if TYPE_CHECKING:
23
- # avoid circuit import
24
- from vllm.lora.layers import LoRAMapping
25
- from vllm.lora.models import LongContextLoRAContext
26
-
27
-
28
-def compute_meta(
29
- token_lora_tensor: torch.Tensor
30
-) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, int, int, int, bool]:
31
- """
32
- Get the information required for the sgmv kernel. With the features:
33
- 1. If consecutive requests in the batch use the same LoRA, this function
34
- will combine them into a single request, improving sgmv kernel inference
35
- performance.
36
- 2. At the beginning of each prefill stage inference, recalculations are
37
- needed based on the input, but only once.
38
- """
39
-
40
- lora_indices_tensor, seq_length_tensor = torch.unique_consecutive(
41
- token_lora_tensor, return_counts=True)
42
- cum_result = torch.cumsum(seq_length_tensor, dim=0)
43
- b_seq_start_tensor = torch.zeros_like(seq_length_tensor)
44
- b_seq_start_tensor[1:].copy_(cum_result[:-1])
45
- max_length = seq_length_tensor.max().item()
46
- token_nums = seq_length_tensor.sum().item()
47
- batch_size = lora_indices_tensor.size(0)
48
- no_lora = False
49
- # -1 means no lora should be applied. Use `no_lora` to determine whether
50
- # the current step requires LoRA. If LoRA is not needed, the prefill stage
51
- # does not need to launch the triton kernel, which can improve performance
52
- if batch_size == 1 and lora_indices_tensor == -1:
53
- no_lora = True
54
- return (b_seq_start_tensor, seq_length_tensor, lora_indices_tensor,
55
- batch_size, max_length, token_nums, no_lora)
56
-
57
-
58
-# TODO see if this can be vectorized
59
-def convert_mapping(
60
- mapping: "LoRAMapping",
61
- lora_index_to_id: List[Optional[int]],
62
- max_loras: int,
63
- vocab_size: int,
64
- extra_vocab_size: int,
65
- device: torch.device,
66
- long_lora_context: Optional["LongContextLoRAContext"] = None,
67
-) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor,
68
- Optional[torch.Tensor], List[int]]:
69
- """Converts LoRAMapping to index tensors.
70
-
71
- Args:
72
- mapping: LoRAMapping mapping rows in a batch to LoRA ids.
73
- lora_index_to_id: List mapping LoRA ids to LoRA indices.
74
- max_loras: Maximum number of LoRAs.
75
- vocab_size: Model vocab size.
76
- extra_vocab_size: Extra vocab size each LoRA can have.
77
- long_lora_context: Passed if there are long context lora in a batch.
78
-
79
- Returns:
80
- A tuple of tensors:
81
- base_indices: Tensor of shape [batch_size] mapping batch rows to
82
- LoRA indices.
83
- sampler_indices: Tensor of shape [batch_size] mapping requests to
84
- LoRA indices for sampler. For generation, this will be the
85
- same as base_indicies. For prefill, this will map requests
86
- to LoRA indices.
87
- sampler_indices_padded: Tensor of shape [batch_size] mapping
88
- requests to LoRA indices for sampler with padding.
89
- Same as sampler_indicies, but -1 is replaced with
90
- max_loras.
91
- embeddings_indices: Tensor of shape [2, batch_size] mapping
92
- requests to embedding indices. First row is for embeddings
93
- added by the LoRAs, second row is for the LoRA.lora_a
94
- embeddings.
95
- long_lora_indices: Tensor of shape [batch_size] mapping
96
- requests to RoPE offsets and rot dims for long LoRAs.
97
- None if long context lora doesn't exist.
98
- indices_len: List of lengths of the above tensors. It contains
99
- (base_indices, sampler_indices, sampler_indices_padded,
100
- embeddings_indices, long_lora_indices).
101
- """
102
- index_mapping_indices: List[int] = list(mapping.index_mapping).copy()
103
- embedding_indices = index_mapping_indices.copy()
104
- lora_indices = index_mapping_indices.copy()
105
- long_lora_offsets: Optional[torch.Tensor] = None
106
- if long_lora_context:
107
- long_lora_offsets = torch.zeros(len(index_mapping_indices),
108
- device=device,
109
- dtype=torch.long)
110
- prompt_mapping: List[int] = [
111
- lora_index_to_id.index(x) if x > 0 else -1
112
- for x in mapping.prompt_mapping
113
- ]
114
- lora_idx = None
115
- for i in range(len(index_mapping_indices)):
116
- # TODO index can be slow. optimize
117
- lora_idx = (lora_index_to_id.index(index_mapping_indices[i])
118
- if index_mapping_indices[i] > 0 else -1)
119
- embedding_indices[i] = lora_idx if index_mapping_indices[i] > 0 else 0
120
- lora_indices[i] = lora_idx
121
- if long_lora_context:
122
- assert long_lora_offsets is not None
123
- lora_offset: int = long_lora_context.offsets_by_lora_id.get(
124
- index_mapping_indices[i], 0)
125
- long_lora_offsets[i] = lora_offset
126
-
127
- indices_list: List[Union[List[int], torch.Tensor]] = [
128
- index_mapping_indices,
129
- lora_indices,
130
- embedding_indices,
131
- ]
132
- if long_lora_context:
133
- assert long_lora_offsets is not None
134
- indices_list.append(long_lora_offsets)
135
- indices = torch.tensor(indices_list, dtype=torch.long, device=device)
136
- prompt_mapping_tensor = torch.tensor(prompt_mapping,
137
- dtype=torch.long,
138
- device=device)
139
- embeddings_indices = torch.stack([
140
- indices[2] * extra_vocab_size,
141
- indices[2] * (vocab_size + extra_vocab_size),
142
- ])
143
- embeddings_indices[embeddings_indices == -1] = max_loras - 1
144
- base_indices = indices[1]
145
- sampler_indices = prompt_mapping_tensor
146
- sampler_indices_padded = sampler_indices.clone()
147
- sampler_indices_padded[sampler_indices_padded == -1] = max_loras - 1
148
- sampler_indices_padded = torch.arange(
149
- 0, len(sampler_indices_padded), device=device, dtype=torch.long) + (
150
- sampler_indices_padded * len(sampler_indices_padded))
151
- long_lora_indices = None
152
- long_lora_indices_len: Optional[int] = None
153
- if long_lora_context:
154
- long_lora_indices = indices[3]
155
- long_lora_indices_len = long_lora_indices.shape[-1]
156
- # Contain length of indices tensors. Used to index into each tensor.
157
- indices_len = [
158
- base_indices.shape[-1],
159
- sampler_indices.shape[-1],
160
- sampler_indices_padded.shape[-1],
161
- embeddings_indices.shape[-1],
162
- ]
163
- if long_lora_indices_len is not None:
164
- indices_len.append(long_lora_indices_len)
165
- else:
166
- # If long_lora doesn't exist,append None
167
- indices_len.append(None)
168
-
169
- return (
170
- base_indices,
171
- sampler_indices,
172
- sampler_indices_padded,
173
- embeddings_indices,
174
- long_lora_indices,
175
- indices_len,
176
- )
177
-
178
-
179
-class PunicaWrapper:
180
- """
181
- PunicaWrapper is designed to manage and provide metadata for the punica
182
- kernel. The main function is to maintain the state information for
183
- Multi-LoRA, and to provide the interface for the punica kernel.
184
- """
185
-
186
- def __init__(self, max_num_batched_tokens: int, max_batches: int,
187
- device: Union[torch.device, str]):
188
- self._token_lora_indices = torch.empty(max_num_batched_tokens,
189
- dtype=torch.long,
190
- device=device)
191
- self._sampler_indices = torch.empty(max_num_batched_tokens,
192
- dtype=torch.long,
193
- device=device)
194
- self._sampler_indices_padded = torch.empty(max_num_batched_tokens,
195
- dtype=torch.long,
196
- device=device)
197
- self._embeddings_indices = torch.empty(2,
198
- max_num_batched_tokens,
199
- dtype=torch.long,
200
- device=device)
201
- self._long_lora_indices = torch.empty(max_num_batched_tokens,
202
- dtype=torch.long,
203
- device=device)
204
-
205
- # 5 is the number of indicies tensors.
206
- # base_indices, sampler_indices, sampler_indices_padded,
207
- # embeddings_indices,long_lora_indices
208
- self.indices_len: List[Optional[int]] = [None] * 5
209
- # these attributes are the information required for sgmv kernel
210
- self._seq_start_locs = torch.empty(max_batches,
211
- dtype=torch.long,
212
- device=device)
213
- self._seq_lengths = torch.empty(max_batches,
214
- dtype=torch.long,
215
- device=device)
216
- self._lora_indices_per_batch = torch.empty(max_batches,
217
- dtype=torch.long,
218
- device=device)
219
- self.device: torch.device = device
220
- self.max_length: int = 0
221
- self.token_nums: int = 0
222
- self.batch_size: int = -1
223
- self.is_prefill = False
224
- self.no_lora = False
225
-
226
- def update_metadata(
227
- self,
228
- mapping: "LoRAMapping",
229
- lora_index_to_id: List[Optional[int]],
230
- max_loras: int,
231
- vocab_size: int,
232
- extra_vocab_size: int,
233
- long_lora_context: Optional["LongContextLoRAContext"] = None,
234
- ):
235
-
236
- self._update_base_metadata(mapping, lora_index_to_id, max_loras,
237
- vocab_size, extra_vocab_size,
238
- long_lora_context)
239
- if mapping.is_prefill:
240
- # Update metadata required for prefill-related operators.
241
- self._update_prefill_metada(self.token_lora_indices)
242
- self.is_prefill = True
243
- else:
244
- self.is_prefill = False
245
-
246
- def _update_base_metadata(
247
- self,
248
- mapping: "LoRAMapping",
249
- lora_index_to_id: List[Optional[int]],
250
- max_loras: int,
251
- vocab_size: int,
252
- extra_vocab_size: int,
253
- long_lora_context: Optional["LongContextLoRAContext"] = None,
254
- ):
255
- (
256
- base_indices,
257
- sampler_indices,
258
- sampler_indices_padded,
259
- embeddings_indices,
260
- long_lora_offsets_tensor,
261
- indices_len,
262
- ) = convert_mapping(
263
- mapping,
264
- lora_index_to_id,
265
- max_loras,
266
- vocab_size,
267
- extra_vocab_size,
268
- self.device,
269
- long_lora_context,
270
- )
271
- self._token_lora_indices[:base_indices.shape[0]].copy_(base_indices)
272
- self._sampler_indices[:sampler_indices.shape[0]].copy_(sampler_indices)
273
- self._sampler_indices_padded[:sampler_indices_padded.shape[0]].copy_(
274
- sampler_indices_padded)
275
- self._embeddings_indices[:embeddings_indices.
276
- shape[0], :embeddings_indices.shape[1]].copy_(
277
- embeddings_indices)
278
- if long_lora_offsets_tensor is not None:
279
- self._long_lora_indices[:long_lora_offsets_tensor.shape[0]].copy_(
280
- long_lora_offsets_tensor)
281
- else:
282
- self._long_lora_indices.zero_()
283
- self.indices_len[:] = indices_len
284
-
285
- def _update_prefill_metada(self, token_lora_tensor: torch.Tensor) -> None:
286
-
287
- (b_seq_start_tensor, seq_length_tensor, lora_indices_tensor,
288
- batch_size, max_length, token_nums,
289
- no_lora) = compute_meta(token_lora_tensor)
290
-
291
- self._seq_start_locs[:b_seq_start_tensor.shape[0]].copy_(
292
- b_seq_start_tensor)
293
- self._seq_lengths[:seq_length_tensor.shape[0]].copy_(seq_length_tensor)
294
- self._lora_indices_per_batch[:lora_indices_tensor.shape[0]].copy_(
295
- lora_indices_tensor)
296
- self.batch_size = batch_size
297
- self.max_length = max_length
298
- self.token_nums = token_nums
299
- self.no_lora = no_lora
300
-
301
- @property
302
- def prefill_metadata(
303
- self
304
- ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, int, int, int]:
305
- """
306
- This property provides a convenient way to access the necessary
307
- metadata for prefill-related kernel computations.
308
- 1. seq_start_locs: Tensor of sequence start positions.
309
- 2. seq_lengths: Tensor of sequence lengths.
310
- 3. lora_indices_per_batch: Tensor of lora indices, and an index of
311
- -1 means no lora should be applied.
312
- 4. batch_size: Batch size after clustering identical lora indices.
313
- 5. max_length: The maximum sequence length in the batch.
314
- 6. token_nums: The token numbers in the batch.
315
- """
316
- return (self._seq_start_locs[:self.batch_size],
317
- self._seq_lengths[:self.batch_size],
318
- self._lora_indices_per_batch[:self.batch_size],
319
- self.batch_size, self.max_length, self.token_nums)
320
-
321
- @property
322
- def token_lora_indices(self) -> torch.Tensor:
323
- """
324
- This property provides the lora indices corresponding to each token
325
- in the batch. An index of -1 means no lora should be applied.
326
- """
327
- token_lora_len = self.indices_len[0]
328
- return self._token_lora_indices[:token_lora_len]
329
-
330
- @property
331
- def sampler_indices(self) -> torch.Tensor:
332
- """
333
- This property is used to access the lora indices specifically for
334
- LogitsProcessorWithLoRA.
335
- """
336
- sampler_indices_len = self.indices_len[1]
337
- return self._sampler_indices[:sampler_indices_len]
338
-
339
- @property
340
- def sampler_indices_padded(self) -> torch.Tensor:
341
- """
342
- This property provides access to padded sampler indices.
343
- """
344
- indices_padded_len = self.indices_len[2]
345
- return self._sampler_indices_padded[:indices_padded_len]
346
-
347
- @property
348
- def embeddings_indices(self) -> torch.Tensor:
349
- """
350
- This property provides access to the indices used for lora embeddings,
351
- specifically for VocabParallelEmbeddingWithLoRA.
352
- """
353
- embeddings_indices_len = self.indices_len[3]
354
- return self._embeddings_indices[:, :embeddings_indices_len]
355
-
356
- @property
357
- def long_lora_indices(self) -> torch.Tensor:
358
- """
359
- This property provides access to the indices used for long context
360
- lora, specifically for LinearScalingRotaryEmbeddingWithLora.
361
- """
362
- long_lora_len = self.indices_len[4]
363
- return self._long_lora_indices[:long_lora_len]
364
-
365
- def _shrink_prefill(
366
- self,
367
- y: torch.Tensor,
368
- x: torch.Tensor,
369
- w_t_all: torch.Tensor,
370
- scale: float,
371
- ):
372
- #No LoRA request, so return directly
373
- if self.no_lora:
374
- return
375
- sgmv_shrink(
376
- x,
377
- w_t_all,
378
- y,
379
- *self.prefill_metadata,
380
- scale,
381
- )
382
-
383
- def _shrink_decode(
384
- self,
385
- y: torch.Tensor,
386
- x: torch.Tensor,
387
- w_t_all: torch.Tensor,
388
- scale: float,
389
- ):
390
- bgmv_shrink(x, w_t_all, y, self.token_lora_indices, scale)
391
-
392
- def _expand_prefill(
393
- self,
394
- y: torch.Tensor,
395
- x: torch.Tensor,
396
- w_t_all: torch.Tensor,
397
- add_input: bool,
398
- ):
399
- #No LoRA request, so return directly
400
- if self.no_lora:
401
- return
402
- sgmv_expand(
403
- x,
404
- w_t_all,
405
- y,
406
- *self.prefill_metadata,
407
- add_input,
408
- )
409
-
410
- def _expand_decode(
411
- self,
412
- y: torch.Tensor,
413
- x: torch.Tensor,
414
- w_t_all: torch.Tensor,
415
- add_input: bool,
416
- ):
417
- bgmv_expand(x, w_t_all, y, self.token_lora_indices, add_input)
418
-
419
- def _expand_slice_prefill(
420
- self,
421
- y: torch.Tensor,
422
- x: torch.Tensor,
423
- w_t_all: torch.Tensor,
424
- y_offset: Optional[int],
425
- y_slice_size: Optional[int],
426
- add_input: bool,
427
- ):
428
- #No LoRA request, so return directly
429
- if self.no_lora:
430
- return
431
- sgmv_expand_slice(
432
- x,
433
- w_t_all,
434
- y,
435
- *self.prefill_metadata,
436
- y_offset,
437
- y_slice_size,
438
- add_input,
439
- )
440
-
441
- def _expand_slice_decode(
442
- self,
443
- y: torch.Tensor,
444
- x: torch.Tensor,
445
- w_t_all: torch.Tensor,
446
- y_offset: Optional[int],
447
- y_slice_size: Optional[int],
448
- add_input: bool,
449
- ):
450
- bgmv_expand_slice(x, w_t_all, y, self.token_lora_indices, y_offset,
451
- y_slice_size, add_input)
452
-
453
- def _apply_expand(self,
454
- y: torch.Tensor,
455
- x: torch.Tensor,
456
- w_t_all: torch.Tensor,
457
- y_offset: Optional[int],
458
- y_slice_size: Optional[int],
459
- add_input: bool = True):
460
- """
461
- Perform the ` y[:,y_offset:y_offset+y_slice_size]+=x@w_t_all`
462
- computation, which is suitable for the
463
- GEMM of lora'b.
464
- """
465
-
466
- expand_slice_fun: Callable = (self._expand_slice_prefill
467
- if self.is_prefill else
468
- self._expand_slice_decode)
469
- expand_slice_fun(y, x, w_t_all, y_offset, y_slice_size, add_input)
470
-
471
- def _apply_bias(
472
- self,
473
- indices: torch.Tensor,
474
- output: torch.Tensor,
475
- output_slices: Tuple[int, ...],
476
- lora_bias_stacked: Tuple[Optional[torch.Tensor], ...],
477
- ):
478
- """Applies bias to output
479
-
480
- Input shapes:
481
- lora_bias_stacked: 3 element tuple of (num_loras, output_dim)
482
- indices: (batch_size)
483
- output: (batch_size, q_slice_size + 2*kv_slice_size)
484
- output_slices: n-1 element tuple of (slice_size...),
485
- where n is number of slices
486
- """
487
- org_output = output
488
- output = output.view(-1, output.shape[-1])
489
- indices = indices.view(-1)
490
-
491
- offset_left = 0
492
- for slice_idx, slice in enumerate(output_slices):
493
- bias = lora_bias_stacked[slice_idx]
494
- if bias is not None:
495
- bias = bias.view(-1, bias.shape[-1])
496
- bias = bias[indices]
497
- bias[indices == -1] = 0
498
- output[:, offset_left:offset_left + slice] += bias
499
- offset_left += slice
500
-
501
- return output.view_as(org_output)
502
-
503
- def _apply_shrink(
504
- self,
505
- y: torch.Tensor,
506
- x: torch.Tensor,
507
- w_t_all: torch.Tensor,
508
- scale: float,
509
- ):
510
- """
511
- Perform the ` y+=x@w_t_all` computation, which is suitable for the
512
- GEMM of lora'a.
513
- When `is_prefill is` true, it indicates that it is currently the
514
- prefill stage, and the `_shrink_prefill` function should be called.
515
- Otherwise, it is the decode stage, and the _shrink_decode function
516
- should be called.
517
- """
518
- y_org = y
519
- y = y.view(-1, y.shape[-1])
520
- shrink_fun: Callable = (self._shrink_prefill
521
- if self.is_prefill else self._shrink_decode)
522
- shrink_fun(y, x, w_t_all, scale)
523
- y = y.view_as(y_org)
524
-
525
- def add_shrink(
526
- self,
527
- y: Union[Tuple[torch.Tensor, ...], torch.Tensor],
528
- x: torch.Tensor,
529
- lora_a_stacked: Tuple[torch.Tensor, ...],
530
- scale: float,
531
- ):
532
- """
533
- Performs GEMM for multiple slices of lora_a.
534
- When `is_prefill is` true, it indicates that it is currently the
535
- prefill stage, and the `_shrink_prefill` function should be called.
536
- Otherwise, it is the decode stage, and the _shrink_decode function
537
- should be called.
538
-
539
- Semantics:
540
- for i in range(len(lora_a_stacked)):
541
- y[i] += (x @ lora_a_stacked[i]) * scale
542
-
543
- Args:
544
- y (Union[Tuple[torch.Tensor, ...], torch.Tensor]): Output tensors
545
- x (torch.Tensor): Input tensor
546
- lora_a_stacked (Tuple[torch.Tensor, ...]): lora_a's weights
547
- scale (float): Scaling factor for the operation
548
- """
549
-
550
- x = x.view(-1, x.shape[-1])
551
- # TODO fuse these kernels
552
- for slice_idx in range(len(lora_a_stacked)):
553
- self._apply_shrink(y[slice_idx], x, lora_a_stacked[slice_idx],
554
- scale)
555
-
556
- def add_expand(
557
- self,
558
- y: torch.Tensor,
559
- x: Union[Tuple[torch.Tensor, ...], torch.Tensor],
560
- lora_b_stacked: Tuple[torch.Tensor, ...],
561
- lora_bias_stacked: Optional[Tuple[torch.Tensor, ...]],
562
- output_slices: Tuple[int, ...],
563
- offset_start: int = 0,
564
- add_input=True,
565
- ) -> None:
566
- """
567
- Performs GEMM and bias addition for multiple slices of lora_b.
568
-
569
- Semantics:
570
- for i in range(len(lora_b_stacked)):
571
- slice = output_slices[i]
572
- y[:, offset:offset+slice] += x[i] @ lora_b_stacked[i] +
573
- lora_bias_stacked[i]
574
- offset += slice
575
-
576
- Args:
577
- y (torch.Tensor): Output tensor.
578
- x (Union[Tuple[torch.Tensor, ...], torch.Tensor]): Input tensors
579
- lora_b_stacked (Tuple[torch.Tensor, ...]): lora_b's weight
580
- lora_bias_stacked (Optional[Tuple[torch.Tensor, ...]]):
581
- bias's weight
582
- output_slices (Tuple[int, ...]): Every slice's size
583
- add_input (bool): Defaults to True.
584
- """
585
- y_org = y
586
- y = y.view(-1, y.shape[-1])
587
- offset_left = offset_start
588
- if lora_bias_stacked is not None:
589
- self._apply_bias(self.token_lora_indices, y, output_slices,
590
- lora_bias_stacked)
591
- for slice_idx in range(len(lora_b_stacked)):
592
- self._apply_expand(
593
- y,
594
- x[slice_idx],
595
- lora_b_stacked[slice_idx],
596
- offset_left,
597
- output_slices[slice_idx],
598
- add_input=add_input,
599
- )
600
- offset_left += output_slices[slice_idx]
601
- y = y.view_as(y_org)
602
-
603
- def add_lora_embedding(
604
- self,
605
- y: torch.Tensor,
606
- x: torch.Tensor,
607
- lora_b_stacked: torch.Tensor,
608
- add_input: bool = True,
609
- ):
610
- """
611
- Applies lora specifically for VocabParallelEmbeddingWithLoRA.
612
-
613
- Semantics:
614
- y += x @ lora_b_stacked
615
-
616
- Args:
617
- y (torch.Tensor): Output tensor.
618
- x (torch.Tensor): Input tensor.
619
- lora_b_stacked (torch.Tensor): lora_b's weights.
620
- add_input (bool): Default to True.
621
-
622
- """
623
-
624
- # Embedding layer only need expand op
625
- expand_fun: Callable = (self._expand_prefill
626
- if self.is_prefill else self._expand_decode)
627
- expand_fun(y, x, lora_b_stacked, add_input)
628
-
629
- def add_lora_linear(
630
- self,
631
- y: torch.Tensor,
632
- x: torch.Tensor,
633
- lora_a_stacked: Tuple[torch.Tensor, ...],
634
- lora_b_stacked: Tuple[torch.Tensor, ...],
635
- lora_bias_stacked: Optional[Tuple[torch.Tensor, ...]],
636
- scale: float,
637
- output_slices: Tuple[int, ...],
638
- *,
639
- buffer: Optional[Tuple[torch.Tensor, ...]] = None) -> None:
640
- """
641
- Applicable to linear-related lora.
642
-
643
- Semantics:
644
- for i in range(len(lora_a_stacked)):
645
- y[i] += (
646
- x[i].unsqueeze(0)
647
- @ lora_a_stacked[indices[i], layer_idx, :, :]
648
- @ lora_b_stacked[indices[i], layer_idx, :, :]
649
- * scale
650
- ).squeeze(0)+lora_bias_stacked[i]
651
-
652
- Args:
653
- y (torch.Tensor): Output tensor. Will be changed in-place.
654
- x (torch.Tensor): Input tensor
655
- lora_a_stacked (Tuple[torch.Tensor, ...]): lora_a's weight.
656
- lora_b_stacked (Tuple[torch.Tensor, ...]): lora_b's weight.
657
- lora_bias_stacked (Optional[Tuple[torch.Tensor, ...]]): lora's bias.
658
- scale (float): Scaling factor.
659
- output_slices (Tuple[int, ...]): Every slice's size.
660
- buffer (Optional[Tuple[torch.Tensor, ...]]): Defaults to None.
661
- """
662
-
663
- assert len(lora_a_stacked) == len(lora_b_stacked) == len(output_slices)
664
- if lora_bias_stacked is not None:
665
- assert len(lora_bias_stacked) == len(output_slices)
666
- y = self._apply_bias(self.token_lora_indices, y, output_slices,
667
- lora_bias_stacked)
668
-
669
- if buffer is None:
670
- r = lora_b_stacked[0].size(-1)
671
- # We set the buffer to be float32 by default ,refer to:
672
- # https://github.com/triton-lang/triton/issues/1387
673
- buffer = tuple(
674
- torch.zeros(
675
- (x.size(0), r), dtype=torch.float32, device=x.device)
676
- for _ in range(len(output_slices)))
677
- self.add_shrink(buffer, x, lora_a_stacked, scale)
678
- self.add_expand(y,
679
- buffer,
680
- lora_b_stacked,
681
- None,
682
- output_slices,
683
- add_input=True)
684
-
685
- def add_lora_logits(self,
686
- y: torch.Tensor,
687
- x: torch.Tensor,
688
- lora_a_stacked: torch.Tensor,
689
- lora_b_stacked: torch.Tensor,
690
- scale,
691
- *,
692
- buffer: Optional[torch.Tensor] = None) -> None:
693
- """
694
- Applies lora specifically for LogitsProcessorWithLoRA.
695
-
696
- Semantics:
697
- buffer = (x @ lora_a_stacked) * scale
698
- y += buffer @ lora_b_stacked
699
-
700
- Args:
701
- y (torch.Tensor): Output tensor.
702
- x (torch.Tensor): Input tensor.
703
- lora_a_stacked (torch.Tensor): lora_a's weights.
704
- lora_b_stacked (torch.Tensor):lora_b's weights.
705
- scale (float): Scaling factor.
706
- buffer (Optional[torch.Tensor]):Default to None.
707
- """
708
- y_org = y
709
- y = y.view(-1, y.shape[-1])
710
- x = x.view(-1, x.shape[-1])
711
- r = lora_b_stacked.size(-1)
712
- if buffer is None:
713
- # We set the buffer to be float32 by default ,refer to:
714
- # https://github.com/triton-lang/triton/issues/1387
715
- buffer = torch.zeros((x.size(0), r),
716
- dtype=torch.float32,
717
- device=x.device)
718
- # LogitsProcessorWithLoRA always using bgmv.
719
- bgmv_shrink(x, lora_a_stacked, buffer, self.sampler_indices, scale)
720
- bgmv_expand(buffer,
721
- lora_b_stacked,
722
- y,
723
- self.sampler_indices,
724
- add_inputs=True)
725
- y = y.view_as(y_org)
vllm/lora/punica_wrapper/__init__.py
new
+7
@@ -0,0 +1,7 @@
1
+from vllm.lora.punica_wrapper.punica_base import PunicaWrapperBase
2
+from vllm.lora.punica_wrapper.punica_selector import get_punica_wrapper
3
+
4
+__all__ = [
5
+ "PunicaWrapperBase",
6
+ "get_punica_wrapper",
7
+]
vllm/lora/punica_wrapper/punica_base.py
new
+480
@@ -0,0 +1,480 @@
1
+"""
2
+Based on:
3
+Chen, L., Ye, Z., Wu, Y., Zhuo, D., Ceze, L., & Krishnamurthy, A. (2023).
4
+Punica: Multi-Tenant LoRA Serving.
5
+https://arxiv.org/abs/2310.18547
6
+"""
7
+
8
+from abc import ABC, abstractmethod
9
+from typing import TYPE_CHECKING, List, Optional, Tuple, Union
10
+
11
+import torch
12
+
13
+from .utils import compute_meta, convert_mapping
14
+
15
+if TYPE_CHECKING:
16
+ # avoid circuit import
17
+ from vllm.lora.layers import LoRAMapping
18
+ from vllm.lora.models import LongContextLoRAContext
19
+
20
+
21
+class PunicaWrapperABC(ABC):
22
+ """
23
+ PunicaWrapper ABC.
24
+ """
25
+
26
+ @abstractmethod
27
+ def update_metadata(
28
+ self,
29
+ mapping: "LoRAMapping",
30
+ lora_index_to_id: List[Optional[int]],
31
+ max_loras: int,
32
+ vocab_size: int,
33
+ extra_vocab_size: int,
34
+ long_lora_context: Optional["LongContextLoRAContext"] = None,
35
+ **kwargs,
36
+ ) -> None:
37
+ """
38
+ Update the lora-related metadata
39
+ """
40
+ raise NotImplementedError
41
+
42
+ @abstractmethod
43
+ def add_shrink(
44
+ self,
45
+ y: Union[Tuple[torch.Tensor, ...], torch.Tensor],
46
+ x: torch.Tensor,
47
+ lora_a_stacked: Tuple[torch.Tensor, ...],
48
+ scale: float,
49
+ **kwargs,
50
+ ) -> None:
51
+ """
52
+ Performs GEMM for multiple slices of lora_a.
53
+ """
54
+
55
+ raise NotImplementedError
56
+
57
+ @abstractmethod
58
+ def add_expand(
59
+ self,
60
+ y: torch.Tensor,
61
+ x: Union[Tuple[torch.Tensor, ...], torch.Tensor],
62
+ lora_b_stacked: Tuple[torch.Tensor, ...],
63
+ lora_bias_stacked: Optional[Tuple[torch.Tensor, ...]],
64
+ output_slices: Tuple[int, ...],
65
+ offset_start: int = 0,
66
+ add_input=True,
67
+ **kwargs,
68
+ ) -> None:
69
+ """
70
+ Performs GEMM and bias addition for multiple slices of lora_b.
71
+ """
72
+ raise NotImplementedError
73
+
74
+ @abstractmethod
75
+ def add_lora_embedding(
76
+ self,
77
+ y: torch.Tensor,
78
+ x: torch.Tensor,
79
+ lora_b_stacked: torch.Tensor,
80
+ add_input: bool = True,
81
+ **kwargs,
82
+ ) -> None:
83
+ """
84
+ Applies lora specifically for VocabParallelEmbeddingWithLoRA,
85
+ and this layer only requires the expand operation.
86
+ """
87
+ raise NotImplementedError
88
+
89
+ @abstractmethod
90
+ def add_lora_linear(self,
91
+ y: torch.Tensor,
92
+ x: torch.Tensor,
93
+ lora_a_stacked: Tuple[torch.Tensor, ...],
94
+ lora_b_stacked: Tuple[torch.Tensor, ...],
95
+ lora_bias_stacked: Optional[Tuple[torch.Tensor, ...]],
96
+ scale: float,
97
+ output_slices: Tuple[int, ...],
98
+ *,
99
+ buffer: Optional[Tuple[torch.Tensor, ...]] = None,
100
+ **kwargs) -> None:
101
+ """
102
+ Applicable to linear-related lora.
103
+ """
104
+
105
+ raise NotImplementedError
106
+
107
+ @abstractmethod
108
+ def add_lora_logits(self,
109
+ y: torch.Tensor,
110
+ x: torch.Tensor,
111
+ lora_a_stacked: torch.Tensor,
112
+ lora_b_stacked: torch.Tensor,
113
+ scale,
114
+ *,
115
+ buffer: Optional[torch.Tensor] = None,
116
+ **kwargs) -> None:
117
+ """
118
+ Applies lora specifically for LogitsProcessorWithLoRA.
119
+ """
120
+ raise NotImplementedError
121
+
122
+
123
+class PunicaWrapperBase(PunicaWrapperABC):
124
+ """
125
+ PunicaWrapperBase is designed to manage and provide metadata for the punica
126
+ kernel. The main function is to maintain the state information for
127
+ Multi-LoRA, and to provide the interface for the punica.
128
+ """
129
+
130
+ def __init__(self, max_num_batched_tokens: int, max_batches: int,
131
+ device: Union[torch.device, str], **kwargs):
132
+ self._token_lora_indices = torch.empty(max_num_batched_tokens,
133
+ dtype=torch.long,
134
+ device=device)
135
+ self._sampler_indices = torch.empty(max_num_batched_tokens,
136
+ dtype=torch.long,
137
+ device=device)
138
+ self._sampler_indices_padded = torch.empty(max_num_batched_tokens,
139
+ dtype=torch.long,
140
+ device=device)
141
+ self._embeddings_indices = torch.empty(2,
142
+ max_num_batched_tokens,
143
+ dtype=torch.long,
144
+ device=device)
145
+ self._long_lora_indices = torch.empty(max_num_batched_tokens,
146
+ dtype=torch.long,
147
+ device=device)
148
+
149
+ # 5 is the number of indicies tensors.
150
+ # base_indices, sampler_indices, sampler_indices_padded,
151
+ # embeddings_indices,long_lora_indices
152
+ self.indices_len: List[Optional[int]] = [None] * 5
153
+ # these attributes are the information required for sgmv kernel
154
+ self._seq_start_locs = torch.empty(max_batches,
155
+ dtype=torch.long,
156
+ device=device)
157
+ self._seq_lengths = torch.empty(max_batches,
158
+ dtype=torch.long,
159
+ device=device)
160
+ self._lora_indices_per_batch = torch.empty(max_batches,
161
+ dtype=torch.long,
162
+ device=device)
163
+ self.device: torch.device = device
164
+ self.max_length: int = 0
165
+ self.token_nums: int = 0
166
+ self.batch_size: int = -1
167
+ self.is_prefill = False
168
+ self.no_lora = False
169
+
170
+ def _update_base_metadata(
171
+ self,
172
+ mapping: "LoRAMapping",
173
+ lora_index_to_id: List[Optional[int]],
174
+ max_loras: int,
175
+ vocab_size: int,
176
+ extra_vocab_size: int,
177
+ long_lora_context: Optional["LongContextLoRAContext"] = None,
178
+ ):
179
+ (
180
+ base_indices,
181
+ sampler_indices,
182
+ sampler_indices_padded,
183
+ embeddings_indices,
184
+ long_lora_offsets_tensor,
185
+ indices_len,
186
+ ) = convert_mapping(
187
+ mapping,
188
+ lora_index_to_id,
189
+ max_loras,
190
+ vocab_size,
191
+ extra_vocab_size,
192
+ self.device,
193
+ long_lora_context,
194
+ )
195
+ self._token_lora_indices[:base_indices.shape[0]].copy_(base_indices)
196
+ self._sampler_indices[:sampler_indices.shape[0]].copy_(sampler_indices)
197
+ self._sampler_indices_padded[:sampler_indices_padded.shape[0]].copy_(
198
+ sampler_indices_padded)
199
+ self._embeddings_indices[:embeddings_indices.
200
+ shape[0], :embeddings_indices.shape[1]].copy_(
201
+ embeddings_indices)
202
+ if long_lora_offsets_tensor is not None:
203
+ self._long_lora_indices[:long_lora_offsets_tensor.shape[0]].copy_(
204
+ long_lora_offsets_tensor)
205
+ else:
206
+ self._long_lora_indices.zero_()
207
+ self.indices_len[:] = indices_len
208
+
209
+ def _update_prefill_metada(self, token_lora_tensor: torch.Tensor) -> None:
210
+
211
+ (b_seq_start_tensor, seq_length_tensor, lora_indices_tensor,
212
+ batch_size, max_length, token_nums,
213
+ no_lora) = compute_meta(token_lora_tensor)
214
+
215
+ self._seq_start_locs[:b_seq_start_tensor.shape[0]].copy_(
216
+ b_seq_start_tensor)
217
+ self._seq_lengths[:seq_length_tensor.shape[0]].copy_(seq_length_tensor)
218
+ self._lora_indices_per_batch[:lora_indices_tensor.shape[0]].copy_(
219
+ lora_indices_tensor)
220
+ self.batch_size = batch_size
221
+ self.max_length = max_length
222
+ self.token_nums = token_nums
223
+ self.no_lora = no_lora
224
+
225
+ def _apply_bias(
226
+ self,
227
+ indices: torch.Tensor,
228
+ output: torch.Tensor,
229
+ output_slices: Tuple[int, ...],
230
+ lora_bias_stacked: Tuple[Optional[torch.Tensor], ...],
231
+ ):
232
+ """Applies bias to output
233
+
234
+ Input shapes:
235
+ lora_bias_stacked: 3 element tuple of (num_loras, output_dim)
236
+ indices: (batch_size)
237
+ output: (batch_size, q_slice_size + 2*kv_slice_size)
238
+ output_slices: n-1 element tuple of (slice_size...),
239
+ where n is number of slices
240
+ """
241
+ org_output = output
242
+ output = output.view(-1, output.shape[-1])
243
+ indices = indices.view(-1)
244
+
245
+ offset_left = 0
246
+ for slice_idx, slice in enumerate(output_slices):
247
+ bias = lora_bias_stacked[slice_idx]
248
+ if bias is not None:
249
+ bias = bias.view(-1, bias.shape[-1])
250
+ bias = bias[indices]
251
+ bias[indices == -1] = 0
252
+ output[:, offset_left:offset_left + slice] += bias
253
+ offset_left += slice
254
+
255
+ return output.view_as(org_output)
256
+
257
+ @property
258
+ def prefill_metadata(
259
+ self
260
+ ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, int, int, int]:
261
+ """
262
+ This property provides a convenient way to access the necessary
263
+ metadata for prefill-related kernel computations.
264
+ 1. seq_start_locs: Tensor of sequence start positions.
265
+ 2. seq_lengths: Tensor of sequence lengths.
266
+ 3. lora_indices_per_batch: Tensor of lora indices, and an index of
267
+ -1 means no lora should be applied.
268
+ 4. batch_size: Batch size after clustering identical lora indices.
269
+ 5. max_length: The maximum sequence length in the batch.
270
+ 6. token_nums: The token numbers in the batch.
271
+ """
272
+ return (self._seq_start_locs[:self.batch_size],
273
+ self._seq_lengths[:self.batch_size],
274
+ self._lora_indices_per_batch[:self.batch_size],
275
+ self.batch_size, self.max_length, self.token_nums)
276
+
277
+ @property
278
+ def token_lora_indices(self) -> torch.Tensor:
279
+ """
280
+ This property provides the lora indices corresponding to each token
281
+ in the batch. An index of -1 means no lora should be applied.
282
+ """
283
+ token_lora_len = self.indices_len[0]
284
+ return self._token_lora_indices[:token_lora_len]
285
+
286
+ @property
287
+ def sampler_indices(self) -> torch.Tensor:
288
+ """
289
+ This property is used to access the lora indices specifically for
290
+ LogitsProcessorWithLoRA.
291
+ """
292
+ sampler_indices_len = self.indices_len[1]
293
+ return self._sampler_indices[:sampler_indices_len]
294
+
295
+ @property
296
+ def sampler_indices_padded(self) -> torch.Tensor:
297
+ """
298
+ This property provides access to padded sampler indices.
299
+ """
300
+ indices_padded_len = self.indices_len[2]
301
+ return self._sampler_indices_padded[:indices_padded_len]
302
+
303
+ @property
304
+ def embeddings_indices(self) -> torch.Tensor:
305
+ """
306
+ This property provides access to the indices used for lora embeddings,
307
+ specifically for VocabParallelEmbeddingWithLoRA.
308
+ """
309
+ embeddings_indices_len = self.indices_len[3]
310
+ return self._embeddings_indices[:, :embeddings_indices_len]
311
+
312
+ @property
313
+ def long_lora_indices(self) -> torch.Tensor:
314
+ """
315
+ This property provides access to the indices used for long context
316
+ lora, specifically for LinearScalingRotaryEmbeddingWithLora.
317
+ """
318
+ long_lora_len = self.indices_len[4]
319
+ return self._long_lora_indices[:long_lora_len]
320
+
321
+ def update_metadata(
322
+ self,
323
+ mapping: "LoRAMapping",
324
+ lora_index_to_id: List[Optional[int]],
325
+ max_loras: int,
326
+ vocab_size: int,
327
+ extra_vocab_size: int,
328
+ long_lora_context: Optional["LongContextLoRAContext"] = None,
329
+ **kwargs):
330
+
331
+ self._update_base_metadata(mapping, lora_index_to_id, max_loras,
332
+ vocab_size, extra_vocab_size,
333
+ long_lora_context)
334
+ if mapping.is_prefill:
335
+ # Update metadata required for prefill-related operators.
336
+ self._update_prefill_metada(self.token_lora_indices)
337
+ self.is_prefill = True
338
+ else:
339
+ self.is_prefill = False
340
+
341
+ @abstractmethod
342
+ def add_shrink(self, y: Union[Tuple[torch.Tensor, ...], torch.Tensor],
343
+ x: torch.Tensor, lora_a_stacked: Tuple[torch.Tensor, ...],
344
+ scale: float, **kwargs) -> None:
345
+ """
346
+ Performs GEMM for multiple slices of lora_a.
347
+
348
+ Semantics:
349
+ for i in range(len(lora_a_stacked)):
350
+ y[i] += (x @ lora_a_stacked[i]) * scale
351
+
352
+ Args:
353
+ y (Union[Tuple[torch.Tensor, ...], torch.Tensor]): Output tensors
354
+ x (torch.Tensor): Input tensor
355
+ lora_a_stacked (Tuple[torch.Tensor, ...]): lora_a's weights
356
+ scale (float): Scaling factor for the operation
357
+
358
+ """
359
+ # TODO: implement it based on torch ops
360
+ raise NotImplementedError
361
+
362
+ @abstractmethod
363
+ def add_expand(self,
364
+ y: torch.Tensor,
365
+ x: Union[Tuple[torch.Tensor, ...], torch.Tensor],
366
+ lora_b_stacked: Tuple[torch.Tensor, ...],
367
+ lora_bias_stacked: Optional[Tuple[torch.Tensor, ...]],
368
+ output_slices: Tuple[int, ...],
369
+ offset_start: int = 0,
370
+ add_input=True,
371
+ **kwargs) -> None:
372
+ """
373
+ Performs GEMM and bias addition for multiple slices of lora_b.
374
+
375
+ Semantics:
376
+ for i in range(len(lora_b_stacked)):
377
+ slice = output_slices[i]
378
+ y[:, offset:offset+slice] += x[i] @ lora_b_stacked[i] +
379
+ lora_bias_stacked[i]
380
+ offset += slice
381
+
382
+ Args:
383
+ y (torch.Tensor): Output tensor.
384
+ x (Union[Tuple[torch.Tensor, ...], torch.Tensor]): Input tensors
385
+ lora_b_stacked (Tuple[torch.Tensor, ...]): lora_b's weight
386
+ lora_bias_stacked (Optional[Tuple[torch.Tensor, ...]]):
387
+ bias's weight
388
+ output_slices (Tuple[int, ...]): Every slice's size
389
+ add_input (bool): Defaults to True.
390
+
391
+ """
392
+ # TODO: implement it based on torch ops
393
+ raise NotImplementedError
394
+
395
+ @abstractmethod
396
+ def add_lora_embedding(self,
397
+ y: torch.Tensor,
398
+ x: torch.Tensor,
399
+ lora_b_stacked: torch.Tensor,
400
+ add_input: bool = True,
401
+ **kwargs) -> None:
402
+ """
403
+ Applies lora specifically for VocabParallelEmbeddingWithLoRA.
404
+ and this layer only requires the expand operation.
405
+ Semantics:
406
+ y += x @ lora_b_stacked
407
+
408
+ Args:
409
+ y (torch.Tensor): Output tensor.
410
+ x (torch.Tensor): Input tensor.
411
+ lora_b_stacked (torch.Tensor): lora_b's weights.
412
+ add_input (bool): Default to True.
413
+ """
414
+ # TODO: implement it based on torch ops
415
+ raise NotImplementedError
416
+
417
+ @abstractmethod
418
+ def add_lora_linear(self,
419
+ y: torch.Tensor,
420
+ x: torch.Tensor,
421
+ lora_a_stacked: Tuple[torch.Tensor, ...],
422
+ lora_b_stacked: Tuple[torch.Tensor, ...],
423
+ lora_bias_stacked: Optional[Tuple[torch.Tensor, ...]],
424
+ scale: float,
425
+ output_slices: Tuple[int, ...],
426
+ *,
427
+ buffer: Optional[Tuple[torch.Tensor, ...]] = None,
428
+ **kwargs) -> None:
429
+ """
430
+ Applicable to linear-related lora.
431
+
432
+ Semantics:
433
+ for i in range(len(lora_a_stacked)):
434
+ y[i] += (
435
+ x[i].unsqueeze(0)
436
+ @ lora_a_stacked[indices[i], layer_idx, :, :]
437
+ @ lora_b_stacked[indices[i], layer_idx, :, :]
438
+ * scale
439
+ ).squeeze(0)+lora_bias_stacked[i]
440
+
441
+ Args:
442
+ y (torch.Tensor): Output tensor. Will be changed in-place.
443
+ x (torch.Tensor): Input tensor
444
+ lora_a_stacked (Tuple[torch.Tensor, ...]): lora_a's weight.
445
+ lora_b_stacked (Tuple[torch.Tensor, ...]): lora_b's weight.
446
+ lora_bias_stacked (Optional[Tuple[torch.Tensor, ...]]): lora's bias.
447
+ scale (float): Scaling factor.
448
+ output_slices (Tuple[int, ...]): Every slice's size.
449
+ buffer (Optional[Tuple[torch.Tensor, ...]]): Defaults to None.
450
+ """
451
+ # TODO: implement it based on torch ops
452
+ raise NotImplementedError
453
+
454
+ @abstractmethod
455
+ def add_lora_logits(self,
456
+ y: torch.Tensor,
457
+ x: torch.Tensor,
458
+ lora_a_stacked: torch.Tensor,
459
+ lora_b_stacked: torch.Tensor,
460
+ scale,
461
+ *,
462
+ buffer: Optional[torch.Tensor] = None,
463
+ **kwargs) -> None:
464
+ """
465
+ Applies lora specifically for LogitsProcessorWithLoRA.
466
+
467
+ Semantics:
468
+ buffer = (x @ lora_a_stacked) * scale
469
+ y += buffer @ lora_b_stacked
470
+
471
+ Args:
472
+ y (torch.Tensor): Output tensor.
473
+ x (torch.Tensor): Input tensor.
474
+ lora_a_stacked (torch.Tensor): lora_a's weights.
475
+ lora_b_stacked (torch.Tensor):lora_b's weights.
476
+ scale (float): Scaling factor.
477
+ buffer (Optional[torch.Tensor]):Default to None.
478
+ """
479
+ # TODO: implement it based on torch ops
480
+ raise NotImplementedError
vllm/lora/punica_wrapper/punica_gpu.py
new
+358
@@ -0,0 +1,358 @@
1
+"""
2
+Based on:
3
+Chen, L., Ye, Z., Wu, Y., Zhuo, D., Ceze, L., & Krishnamurthy, A. (2023).
4
+Punica: Multi-Tenant LoRA Serving.
5
+https://arxiv.org/abs/2310.18547
6
+"""
7
+
8
+from typing import Callable, Optional, Tuple, Union, final
9
+
10
+import torch
11
+
12
+from vllm.triton_utils import HAS_TRITON
13
+
14
+if HAS_TRITON:
15
+ from vllm.lora.ops.bgmv_expand import bgmv_expand
16
+ from vllm.lora.ops.bgmv_expand_slice import bgmv_expand_slice
17
+ from vllm.lora.ops.bgmv_shrink import bgmv_shrink
18
+ from vllm.lora.ops.sgmv_expand import sgmv_expand
19
+ from vllm.lora.ops.sgmv_expand_slice import sgmv_expand_slice
20
+ from vllm.lora.ops.sgmv_shrink import sgmv_shrink
21
+
22
+from .punica_base import PunicaWrapperBase
23
+
24
+
25
+@final
26
+class PunicaWrapperGPU(PunicaWrapperBase):
27
+ """
28
+ PunicaWrapperGPU is designed to manage and provide metadata for the punica
29
+ kernel. The main function is to maintain the state information for
30
+ Multi-LoRA, and to provide the interface for the punica triton kernel.
31
+ """
32
+
33
+ def __init__(self, max_num_batched_tokens: int, max_batches: int,
34
+ device: Union[torch.device, str], **kwargs):
35
+ PunicaWrapperBase.__init__(self, max_num_batched_tokens, max_batches,
36
+ device)
37
+
38
+ def _shrink_prefill(
39
+ self,
40
+ y: torch.Tensor,
41
+ x: torch.Tensor,
42
+ w_t_all: torch.Tensor,
43
+ scale: float,
44
+ ):
45
+ #No LoRA request, so return directly
46
+ if self.no_lora:
47
+ return
48
+ sgmv_shrink(
49
+ x,
50
+ w_t_all,
51
+ y,
52
+ *self.prefill_metadata,
53
+ scale,
54
+ )
55
+
56
+ def _shrink_decode(
57
+ self,
58
+ y: torch.Tensor,
59
+ x: torch.Tensor,
60
+ w_t_all: torch.Tensor,
61
+ scale: float,
62
+ ):
63
+ bgmv_shrink(x, w_t_all, y, self.token_lora_indices, scale)
64
+
65
+ def _expand_prefill(
66
+ self,
67
+ y: torch.Tensor,
68
+ x: torch.Tensor,
69
+ w_t_all: torch.Tensor,
70
+ add_input: bool,
71
+ ):
72
+ #No LoRA request, so return directly
73
+ if self.no_lora:
74
+ return
75
+ sgmv_expand(
76
+ x,
77
+ w_t_all,
78
+ y,
79
+ *self.prefill_metadata,
80
+ add_input,
81
+ )
82
+
83
+ def _expand_decode(
84
+ self,
85
+ y: torch.Tensor,
86
+ x: torch.Tensor,
87
+ w_t_all: torch.Tensor,
88
+ add_input: bool,
89
+ ):
90
+ bgmv_expand(x, w_t_all, y, self.token_lora_indices, add_input)
91
+
92
+ def _expand_slice_prefill(
93
+ self,
94
+ y: torch.Tensor,
95
+ x: torch.Tensor,
96
+ w_t_all: torch.Tensor,
97
+ y_offset: Optional[int],
98
+ y_slice_size: Optional[int],
99
+ add_input: bool,
100
+ ):
101
+ #No LoRA request, so return directly
102
+ if self.no_lora:
103
+ return
104
+ sgmv_expand_slice(
105
+ x,
106
+ w_t_all,
107
+ y,
108
+ *self.prefill_metadata,
109
+ y_offset,
110
+ y_slice_size,
111
+ add_input,
112
+ )
113
+
114
+ def _expand_slice_decode(
115
+ self,
116
+ y: torch.Tensor,
117
+ x: torch.Tensor,
118
+ w_t_all: torch.Tensor,
119
+ y_offset: Optional[int],
120
+ y_slice_size: Optional[int],
121
+ add_input: bool,
122
+ ):
123
+ bgmv_expand_slice(x, w_t_all, y, self.token_lora_indices, y_offset,
124
+ y_slice_size, add_input)
125
+
126
+ def _apply_expand(
127
+ self,
128
+ y: torch.Tensor,
129
+ x: torch.Tensor,
130
+ w_t_all: torch.Tensor,
131
+ y_offset: Optional[int],
132
+ y_slice_size: Optional[int],
133
+ add_input: bool = True,
134
+ ):
135
+ """
136
+ Perform the ` y[:,y_offset:y_offset+y_slice_size]+=x@w_t_all`
137
+ computation, which is suitable for the
138
+ GEMM of lora'b.
139
+ """
140
+
141
+ expand_slice_fun: Callable = (self._expand_slice_prefill
142
+ if self.is_prefill else
143
+ self._expand_slice_decode)
144
+ expand_slice_fun(y, x, w_t_all, y_offset, y_slice_size, add_input)
145
+
146
+ def _apply_shrink(self, y: torch.Tensor, x: torch.Tensor,
147
+ w_t_all: torch.Tensor, scale: float):
148
+ """
149
+ Perform the ` y+=x@w_t_all` computation, which is suitable for the
150
+ GEMM of lora'a.
151
+ When `is_prefill is` true, it indicates that it is currently the
152
+ prefill stage, and the `_shrink_prefill` function should be called.
153
+ Otherwise, it is the decode stage, and the _shrink_decode function
154
+ should be called.
155
+ """
156
+ y_org = y
157
+ y = y.view(-1, y.shape[-1])
158
+ shrink_fun: Callable = (self._shrink_prefill
159
+ if self.is_prefill else self._shrink_decode)
160
+ shrink_fun(y, x, w_t_all, scale)
161
+ y = y.view_as(y_org)
162
+
163
+ def add_shrink(self, y: Union[Tuple[torch.Tensor, ...], torch.Tensor],
164
+ x: torch.Tensor, lora_a_stacked: Tuple[torch.Tensor, ...],
165
+ scale: float, **kwargs):
166
+ """
167
+ Performs GEMM for multiple slices of lora_a.
168
+ When `is_prefill is` true, it indicates that it is currently the
169
+ prefill stage, and the `_shrink_prefill` function should be called.
170
+ Otherwise, it is the decode stage, and the _shrink_decode function
171
+ should be called.
172
+
173
+ Semantics:
174
+ for i in range(len(lora_a_stacked)):
175
+ y[i] += (x @ lora_a_stacked[i]) * scale
176
+
177
+ Args:
178
+ y (Union[Tuple[torch.Tensor, ...], torch.Tensor]): Output tensors
179
+ x (torch.Tensor): Input tensor
180
+ lora_a_stacked (Tuple[torch.Tensor, ...]): lora_a's weights
181
+ scale (float): Scaling factor for the operation
182
+ """
183
+
184
+ x = x.view(-1, x.shape[-1])
185
+ # TODO fuse these kernels
186
+ for slice_idx in range(len(lora_a_stacked)):
187
+ self._apply_shrink(y[slice_idx], x, lora_a_stacked[slice_idx],
188
+ scale)
189
+
190
+ def add_expand(self,
191
+ y: torch.Tensor,
192
+ x: Union[Tuple[torch.Tensor, ...], torch.Tensor],
193
+ lora_b_stacked: Tuple[torch.Tensor, ...],
194
+ lora_bias_stacked: Optional[Tuple[torch.Tensor, ...]],
195
+ output_slices: Tuple[int, ...],
196
+ offset_start: int = 0,
197
+ add_input=True,
198
+ **kwargs) -> None:
199
+ """
200
+ Performs GEMM and bias addition for multiple slices of lora_b.
201
+
202
+ Semantics:
203
+ for i in range(len(lora_b_stacked)):
204
+ slice = output_slices[i]
205
+ y[:, offset:offset+slice] += x[i] @ lora_b_stacked[i] +
206
+ lora_bias_stacked[i]
207
+ offset += slice
208
+
209
+ Args:
210
+ y (torch.Tensor): Output tensor.
211
+ x (Union[Tuple[torch.Tensor, ...], torch.Tensor]): Input tensors
212
+ lora_b_stacked (Tuple[torch.Tensor, ...]): lora_b's weight
213
+ lora_bias_stacked (Optional[Tuple[torch.Tensor, ...]]):
214
+ bias's weight
215
+ output_slices (Tuple[int, ...]): Every slice's size
216
+ add_input (bool): Defaults to True.
217
+ """
218
+ y_org = y
219
+ y = y.view(-1, y.shape[-1])
220
+ offset_left = offset_start
221
+ if lora_bias_stacked is not None:
222
+ self._apply_bias(self.token_lora_indices, y, output_slices,
223
+ lora_bias_stacked)
224
+ for slice_idx in range(len(lora_b_stacked)):
225
+ self._apply_expand(
226
+ y,
227
+ x[slice_idx],
228
+ lora_b_stacked[slice_idx],
229
+ offset_left,
230
+ output_slices[slice_idx],
231
+ add_input=add_input,
232
+ )
233
+ offset_left += output_slices[slice_idx]
234
+ y = y.view_as(y_org)
235
+
236
+ def add_lora_embedding(self,
237
+ y: torch.Tensor,
238
+ x: torch.Tensor,
239
+ lora_b_stacked: torch.Tensor,
240
+ add_input: bool = True,
241
+ **kwargs) -> None:
242
+ """
243
+ Applies lora specifically for VocabParallelEmbeddingWithLoRA.
244
+
245
+ Semantics:
246
+ y += x @ lora_b_stacked
247
+
248
+ Args:
249
+ y (torch.Tensor): Output tensor.
250
+ x (torch.Tensor): Input tensor.
251
+ lora_b_stacked (torch.Tensor): lora_b's weights.
252
+ add_input (bool): Default to True.
253
+ """
254
+
255
+ # Embedding layer only need expand op
256
+ expand_fun: Callable = (self._expand_prefill
257
+ if self.is_prefill else self._expand_decode)
258
+ expand_fun(y, x, lora_b_stacked, add_input)
259
+
260
+ def add_lora_linear(self,
261
+ y: torch.Tensor,
262
+ x: torch.Tensor,
263
+ lora_a_stacked: Tuple[torch.Tensor, ...],
264
+ lora_b_stacked: Tuple[torch.Tensor, ...],
265
+ lora_bias_stacked: Optional[Tuple[torch.Tensor, ...]],
266
+ scale: float,
267
+ output_slices: Tuple[int, ...],
268
+ *,
269
+ buffer: Optional[Tuple[torch.Tensor, ...]] = None,
270
+ **kwargs) -> None:
271
+ """
272
+ Applicable to linear-related lora.
273
+
274
+ Semantics:
275
+ for i in range(len(lora_a_stacked)):
276
+ y[i] += (
277
+ x[i].unsqueeze(0)
278
+ @ lora_a_stacked[indices[i], layer_idx, :, :]
279
+ @ lora_b_stacked[indices[i], layer_idx, :, :]
280
+ * scale
281
+ ).squeeze(0)+lora_bias_stacked[i]
282
+
283
+ Args:
284
+ y (torch.Tensor): Output tensor. Will be changed in-place.
285
+ x (torch.Tensor): Input tensor
286
+ lora_a_stacked (Tuple[torch.Tensor, ...]): lora_a's weight.
287
+ lora_b_stacked (Tuple[torch.Tensor, ...]): lora_b's weight.
288
+ lora_bias_stacked (Optional[Tuple[torch.Tensor, ...]]): lora's bias.
289
+ scale (float): Scaling factor.
290
+ output_slices (Tuple[int, ...]): Every slice's size.
291
+ buffer (Optional[Tuple[torch.Tensor, ...]]): Defaults to None.
292
+ """
293
+
294
+ assert len(lora_a_stacked) == len(lora_b_stacked) == len(output_slices)
295
+ if lora_bias_stacked is not None:
296
+ assert len(lora_bias_stacked) == len(output_slices)
297
+ y = self._apply_bias(self.token_lora_indices, y, output_slices,
298
+ lora_bias_stacked)
299
+
300
+ if buffer is None:
301
+ r = lora_b_stacked[0].size(-1)
302
+ # We set the buffer to be float32 by default ,refer to:
303
+ # https://github.com/triton-lang/triton/issues/1387
304
+ buffer = tuple(
305
+ torch.zeros(
306
+ (x.size(0), r), dtype=torch.float32, device=x.device)
307
+ for _ in range(len(output_slices)))
308
+ self.add_shrink(buffer, x, lora_a_stacked, scale, **kwargs)
309
+ self.add_expand(y,
310
+ buffer,
311
+ lora_b_stacked,
312
+ None,
313
+ output_slices,
314
+ add_input=True,
315
+ **kwargs)
316
+
317
+ def add_lora_logits(self,
318
+ y: torch.Tensor,
319
+ x: torch.Tensor,
320
+ lora_a_stacked: torch.Tensor,
321
+ lora_b_stacked: torch.Tensor,
322
+ scale,
323
+ *,
324
+ buffer: Optional[torch.Tensor] = None,
325
+ **kwargs) -> None:
326
+ """
327
+ Applies lora specifically for LogitsProcessorWithLoRA.
328
+
329
+ Semantics:
330
+ buffer = (x @ lora_a_stacked) * scale
331
+ y += buffer @ lora_b_stacked
332
+
333
+ Args:
334
+ y (torch.Tensor): Output tensor.
335
+ x (torch.Tensor): Input tensor.
336
+ lora_a_stacked (torch.Tensor): lora_a's weights.
337
+ lora_b_stacked (torch.Tensor):lora_b's weights.
338
+ scale (float): Scaling factor.
339
+ buffer (Optional[torch.Tensor]):Default to None.
340
+ """
341
+ y_org = y
342
+ y = y.view(-1, y.shape[-1])
343
+ x = x.view(-1, x.shape[-1])
344
+ r = lora_b_stacked.size(-1)
345
+ if buffer is None:
346
+ # We set the buffer to be float32 by default ,refer to:
347
+ # https://github.com/triton-lang/triton/issues/1387
348
+ buffer = torch.zeros((x.size(0), r),
349
+ dtype=torch.float32,
350
+ device=x.device)
351
+ # LogitsProcessorWithLoRA always using bgmv.
352
+ bgmv_shrink(x, lora_a_stacked, buffer, self.sampler_indices, scale)
353
+ bgmv_expand(buffer,
354
+ lora_b_stacked,
355
+ y,
356
+ self.sampler_indices,
357
+ add_inputs=True)
358
+ y = y.view_as(y_org)
vllm/lora/punica_wrapper/punica_selector.py
new
+14
@@ -0,0 +1,14 @@
1
+from vllm.platforms import current_platform
2
+from vllm.utils import print_info_once
3
+
4
+from .punica_base import PunicaWrapperBase
5
+
6
+
7
+def get_punica_wrapper(*args, **kwargs) -> PunicaWrapperBase:
8
+ if current_platform.is_cuda_alike():
9
+ # Lazy import to avoid ImportError
10
+ from vllm.lora.punica_wrapper.punica_gpu import PunicaWrapperGPU
11
+ print_info_once("Using PunicaWrapperGPU.")
12
+ return PunicaWrapperGPU(*args, **kwargs)
13
+ else:
14
+ raise NotImplementedError
vllm/lora/punica_wrapper/utils.py
new
+159
@@ -0,0 +1,159 @@
1
+from typing import TYPE_CHECKING, List, Optional, Tuple, Union
2
+
3
+import torch
4
+
5
+if TYPE_CHECKING:
6
+ # avoid circuit import
7
+ from vllm.lora.layers import LoRAMapping
8
+ from vllm.lora.models import LongContextLoRAContext
9
+
10
+
11
+def compute_meta(
12
+ token_lora_tensor: torch.Tensor
13
+) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, int, int, int, bool]:
14
+ """
15
+ Get the information required for the sgmv kernel. With the features:
16
+ 1. If consecutive requests in the batch use the same LoRA, this function
17
+ will combine them into a single request, improving sgmv kernel inference
18
+ performance.
19
+ 2. At the beginning of each prefill stage inference, recalculations are
20
+ needed based on the input, but only once.
21
+ """
22
+
23
+ lora_indices_tensor, seq_length_tensor = torch.unique_consecutive(
24
+ token_lora_tensor, return_counts=True)
25
+ cum_result = torch.cumsum(seq_length_tensor, dim=0)
26
+ b_seq_start_tensor = torch.zeros_like(seq_length_tensor)
27
+ b_seq_start_tensor[1:].copy_(cum_result[:-1])
28
+ max_length = seq_length_tensor.max().item()
29
+ token_nums = seq_length_tensor.sum().item()
30
+ batch_size = lora_indices_tensor.size(0)
31
+ no_lora = False
32
+ # -1 means no lora should be applied. Use `no_lora` to determine whether
33
+ # the current step requires LoRA. If LoRA is not needed, the prefill stage
34
+ # does not need to launch the triton kernel, which can improve performance
35
+ if batch_size == 1 and lora_indices_tensor == -1:
36
+ no_lora = True
37
+ return (b_seq_start_tensor, seq_length_tensor, lora_indices_tensor,
38
+ batch_size, max_length, token_nums, no_lora)
39
+
40
+
41
+# TODO see if this can be vectorized
42
+def convert_mapping(
43
+ mapping: "LoRAMapping",
44
+ lora_index_to_id: List[Optional[int]],
45
+ max_loras: int,
46
+ vocab_size: int,
47
+ extra_vocab_size: int,
48
+ device: torch.device,
49
+ long_lora_context: Optional["LongContextLoRAContext"] = None,
50
+) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor,
51
+ Optional[torch.Tensor], List[int]]:
52
+ """Converts LoRAMapping to index tensors.
53
+
54
+ Args:
55
+ mapping: LoRAMapping mapping rows in a batch to LoRA ids.
56
+ lora_index_to_id: List mapping LoRA ids to LoRA indices.
57
+ max_loras: Maximum number of LoRAs.
58
+ vocab_size: Model vocab size.
59
+ extra_vocab_size: Extra vocab size each LoRA can have.
60
+ long_lora_context: Passed if there are long context lora in a batch.
61
+
62
+ Returns:
63
+ A tuple of tensors:
64
+ base_indices: Tensor of shape [batch_size] mapping batch rows to
65
+ LoRA indices.
66
+ sampler_indices: Tensor of shape [batch_size] mapping requests to
67
+ LoRA indices for sampler. For generation, this will be the
68
+ same as base_indicies. For prefill, this will map requests
69
+ to LoRA indices.
70
+ sampler_indices_padded: Tensor of shape [batch_size] mapping
71
+ requests to LoRA indices for sampler with padding.
72
+ Same as sampler_indicies, but -1 is replaced with
73
+ max_loras.
74
+ embeddings_indices: Tensor of shape [2, batch_size] mapping
75
+ requests to embedding indices. First row is for embeddings
76
+ added by the LoRAs, second row is for the LoRA.lora_a
77
+ embeddings.
78
+ long_lora_indices: Tensor of shape [batch_size] mapping
79
+ requests to RoPE offsets and rot dims for long LoRAs.
80
+ None if long context lora doesn't exist.
81
+ indices_len: List of lengths of the above tensors. It contains
82
+ (base_indices, sampler_indices, sampler_indices_padded,
83
+ embeddings_indices, long_lora_indices).
84
+ """
85
+ index_mapping_indices: List[int] = list(mapping.index_mapping).copy()
86
+ embedding_indices = index_mapping_indices.copy()
87
+ lora_indices = index_mapping_indices.copy()
88
+ long_lora_offsets: Optional[torch.Tensor] = None
89
+ if long_lora_context:
90
+ long_lora_offsets = torch.zeros(len(index_mapping_indices),
91
+ device=device,
92
+ dtype=torch.long)
93
+ prompt_mapping: List[int] = [
94
+ lora_index_to_id.index(x) if x > 0 else -1
95
+ for x in mapping.prompt_mapping
96
+ ]
97
+ lora_idx = None
98
+ for i in range(len(index_mapping_indices)):
99
+ # TODO index can be slow. optimize
100
+ lora_idx = (lora_index_to_id.index(index_mapping_indices[i])
101
+ if index_mapping_indices[i] > 0 else -1)
102
+ embedding_indices[i] = lora_idx if index_mapping_indices[i] > 0 else 0
103
+ lora_indices[i] = lora_idx
104
+ if long_lora_context:
105
+ assert long_lora_offsets is not None
106
+ lora_offset: int = long_lora_context.offsets_by_lora_id.get(
107
+ index_mapping_indices[i], 0)
108
+ long_lora_offsets[i] = lora_offset
109
+
110
+ indices_list: List[Union[List[int], torch.Tensor]] = [
111
+ index_mapping_indices,
112
+ lora_indices,
113
+ embedding_indices,
114
+ ]
115
+ if long_lora_context:
116
+ assert long_lora_offsets is not None
117
+ indices_list.append(long_lora_offsets)
118
+ indices = torch.tensor(indices_list, dtype=torch.long, device=device)
119
+ prompt_mapping_tensor = torch.tensor(prompt_mapping,
120
+ dtype=torch.long,
121
+ device=device)
122
+ embeddings_indices = torch.stack([
123
+ indices[2] * extra_vocab_size,
124
+ indices[2] * (vocab_size + extra_vocab_size),
125
+ ])
126
+ embeddings_indices[embeddings_indices == -1] = max_loras - 1
127
+ base_indices = indices[1]
128
+ sampler_indices = prompt_mapping_tensor
129
+ sampler_indices_padded = sampler_indices.clone()
130
+ sampler_indices_padded[sampler_indices_padded == -1] = max_loras - 1
131
+ sampler_indices_padded = torch.arange(
132
+ 0, len(sampler_indices_padded), device=device, dtype=torch.long) + (
133
+ sampler_indices_padded * len(sampler_indices_padded))
134
+ long_lora_indices = None
135
+ long_lora_indices_len: Optional[int] = None
136
+ if long_lora_context:
137
+ long_lora_indices = indices[3]
138
+ long_lora_indices_len = long_lora_indices.shape[-1]
139
+ # Contain length of indices tensors. Used to index into each tensor.
140
+ indices_len = [
141
+ base_indices.shape[-1],
142
+ sampler_indices.shape[-1],
143
+ sampler_indices_padded.shape[-1],
144
+ embeddings_indices.shape[-1],
145
+ ]
146
+ if long_lora_indices_len is not None:
147
+ indices_len.append(long_lora_indices_len)
148
+ else:
149
+ # If long_lora doesn't exist,append None
150
+ indices_len.append(None)
151
+
152
+ return (
153
+ base_indices,
154
+ sampler_indices,
155
+ sampler_indices_padded,
156
+ embeddings_indices,
157
+ long_lora_indices,
158
+ indices_len,
159
+ )