A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 03:15 UTC
[Kernel][LoRA]Punica prefill kernels fusion (#11234)
Signed-off-by: Jee Jee Li <pandaleefree@gmail.com> Signed-off-by: Abatom <abzhonghua@gmail.com> Co-authored-by: Zhonghua Deng <abatom@163.com>
Jee Jee Li committed
Jan 7, 2025 at 12:01 UTC
b278557935d78b337fb5e82a32b02f75678b4101
11 files changed
+707
-764
.buildkite/test-pipeline.yaml
+2
-1
@@ -242,7 +242,7 @@ steps:
242
source_file_dependencies:
243
- vllm/lora
244
- tests/lora
245
- command: pytest -v -s lora --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --ignore=lora/test_long_context.py --ignore=lora/test_chatglm3_tp.py --ignore=lora/test_llama_tp.py
245
+ command: pytest -v -s lora --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --ignore=lora/test_long_context.py --ignore=lora/test_chatglm3_tp.py --ignore=lora/test_llama_tp.py --ignore=lora/test_minicpmv_tp.py
246
parallelism: 4
247
248
- label: "PyTorch Fullgraph Smoke Test" # 9min
@@ -535,6 +535,7 @@ steps:
535
# requires multi-GPU testing for validation.
536
- pytest -v -s -x lora/test_chatglm3_tp.py
537
- pytest -v -s -x lora/test_llama_tp.py
538
+ - pytest -v -s -x lora/test_minicpmv_tp.py
539
540
541
- label: Weight Loading Multiple GPU Test # 33min
tests/lora/test_minicpmv.py
deleted
-77
@@ -1,77 +0,0 @@
1
-from typing import List
2
-
3
-import pytest
4
-
5
-import vllm
6
-from vllm.assets.image import ImageAsset
7
-from vllm.lora.request import LoRARequest
8
-from vllm.platforms import current_platform
9
-
10
-MODEL_PATH = "openbmb/MiniCPM-Llama3-V-2_5"
11
-
12
-PROMPT_TEMPLATE = (
13
- "<|begin_of_text|><|start_header_id|>user<|end_header_id|>\n\n"
14
- "(<image>./</image>)\nWhat is in the image?<|eot_id|>"
15
- "<|start_header_id|>assistant<|end_header_id|>\n\n")
16
-
17
-IMAGE_ASSETS = [
18
- ImageAsset("stop_sign"),
19
- ImageAsset("cherry_blossom"),
20
-]
21
-
22
-# After fine-tuning with LoRA, all generated content should start begin `A`.
23
-EXPECTED_OUTPUT = [
24
- "A red and white stop sign with a Chinese archway in the background featuring red lanterns and gold accents.", # noqa: E501
25
- "A pink cherry blossom tree with a blue sky in the background.",
26
-]
27
-
28
-
29
-def do_sample(llm: vllm.LLM, lora_path: str, lora_id: int) -> List[str]:
30
- sampling_params = vllm.SamplingParams(
31
- temperature=0,
32
- max_tokens=5,
33
- stop_token_ids=[128001, 128009], # eos_id, eot_id
34
- )
35
-
36
- inputs = [{
37
- "prompt": PROMPT_TEMPLATE,
38
- "multi_modal_data": {
39
- "image": asset.pil_image
40
- },
41
- } for asset in IMAGE_ASSETS]
42
-
43
- outputs = llm.generate(
44
- inputs,
45
- sampling_params,
46
- lora_request=LoRARequest(str(lora_id), lora_id, lora_path)
47
- if lora_id else None,
48
- )
49
- # Print the outputs.
50
- generated_texts: List[str] = []
51
- for output in outputs:
52
- prompt = output.prompt
53
- generated_text = output.outputs[0].text.strip()
54
- generated_texts.append(generated_text)
55
- print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
56
- return generated_texts
57
-
58
-
59
-@pytest.mark.xfail(
60
- current_platform.is_rocm(),
61
- reason="MiniCPM-V dependency xformers incompatible with ROCm")
62
-def test_minicpmv_lora(minicpmv_lora_files):
63
- llm = vllm.LLM(
64
- MODEL_PATH,
65
- max_num_seqs=2,
66
- enable_lora=True,
67
- max_loras=4,
68
- max_lora_rank=64,
69
- trust_remote_code=True,
70
- enable_chunked_prefill=True,
71
- )
72
- output1 = do_sample(llm, minicpmv_lora_files, lora_id=1)
73
- for i in range(len(EXPECTED_OUTPUT)):
74
- assert EXPECTED_OUTPUT[i].startswith(output1[i])
75
- output2 = do_sample(llm, minicpmv_lora_files, lora_id=2)
76
- for i in range(len(EXPECTED_OUTPUT)):
77
- assert EXPECTED_OUTPUT[i].startswith(output2[i])
tests/lora/test_minicpmv_tp.py
+44
-19
@@ -3,10 +3,10 @@ from typing import List
3
import pytest
4
5
import vllm
6
+from tests.utils import fork_new_process_for_each_test
7
from vllm.assets.image import ImageAsset
8
from vllm.lora.request import LoRARequest
8
-
9
-from ..utils import multi_gpu_test
9
+from vllm.platforms import current_platform
10
11
MODEL_PATH = "openbmb/MiniCPM-Llama3-V-2_5"
12
@@ -17,13 +17,11 @@ PROMPT_TEMPLATE = (
17
18
IMAGE_ASSETS = [
19
ImageAsset("stop_sign"),
20
- ImageAsset("cherry_blossom"),
20
]
21
22
# After fine-tuning with LoRA, all generated content should start begin `A`.
23
EXPECTED_OUTPUT = [
24
"A red and white stop sign with a Chinese archway in the background featuring red lanterns and gold accents.", # noqa: E501
26
- "A pink cherry blossom tree with a blue sky in the background.",
25
]
26
27
@@ -50,48 +48,75 @@ def do_sample(llm: vllm.LLM, lora_path: str, lora_id: int) -> List[str]:
48
# Print the outputs.
49
generated_texts: List[str] = []
50
for output in outputs:
53
- prompt = output.prompt
51
generated_text = output.outputs[0].text.strip()
52
generated_texts.append(generated_text)
56
- print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
53
+ print(f"Generated text: {generated_text!r}")
54
return generated_texts
55
56
60
-@multi_gpu_test(num_gpus=2)
61
-@pytest.mark.parametrize("fully_sharded", [True, False])
62
-def test_minicpmv_tp2(minicpmv_lora_files, fully_sharded):
57
+@pytest.mark.xfail(
58
+ current_platform.is_rocm(),
59
+ reason="MiniCPM-V dependency xformers incompatible with ROCm")
60
+@fork_new_process_for_each_test
61
+def test_minicpmv_lora(minicpmv_lora_files):
62
+ llm = vllm.LLM(
63
+ MODEL_PATH,
64
+ max_num_seqs=2,
65
+ enable_lora=True,
66
+ max_loras=2,
67
+ max_lora_rank=8,
68
+ enforce_eager=True,
69
+ trust_remote_code=True,
70
+ enable_chunked_prefill=True,
71
+ )
72
+ output1 = do_sample(llm, minicpmv_lora_files, lora_id=1)
73
+ for i in range(len(EXPECTED_OUTPUT)):
74
+ assert EXPECTED_OUTPUT[i].startswith(output1[i])
75
+ output2 = do_sample(llm, minicpmv_lora_files, lora_id=2)
76
+ for i in range(len(EXPECTED_OUTPUT)):
77
+ assert EXPECTED_OUTPUT[i].startswith(output2[i])
78
+
79
+
80
+@pytest.mark.xfail(
81
+ current_platform.is_rocm(),
82
+ reason="MiniCPM-V dependency xformers incompatible with ROCm")
83
+@fork_new_process_for_each_test
84
+def test_minicpmv_tp4_wo_fully_sharded_loras(minicpmv_lora_files):
85
llm = vllm.LLM(
86
MODEL_PATH,
87
enable_lora=True,
88
max_num_seqs=2,
89
max_loras=4,
90
max_lora_rank=64,
69
- tensor_parallel_size=2,
91
+ tensor_parallel_size=4,
92
trust_remote_code=True,
71
- fully_sharded_loras=fully_sharded,
93
+ enforce_eager=True,
94
enable_chunked_prefill=True,
95
)
74
-
96
output_tp = do_sample(llm, minicpmv_lora_files, lora_id=1)
76
-
97
for i in range(len(EXPECTED_OUTPUT)):
98
assert EXPECTED_OUTPUT[i].startswith(output_tp[i])
99
100
81
-@multi_gpu_test(num_gpus=4)
82
-@pytest.mark.parametrize("fully_sharded", [True, False])
83
-def test_minicpmv_tp4(minicpmv_lora_files, fully_sharded):
101
+@pytest.mark.xfail(
102
+ current_platform.is_rocm(),
103
+ reason="MiniCPM-V dependency xformers incompatible with ROCm")
104
+@fork_new_process_for_each_test
105
+def test_minicpmv_tp4_fully_sharded_loras(minicpmv_lora_files):
106
llm = vllm.LLM(
107
MODEL_PATH,
108
enable_lora=True,
109
max_num_seqs=2,
88
- max_loras=4,
89
- max_lora_rank=64,
110
+ max_loras=2,
111
+ max_lora_rank=8,
112
tensor_parallel_size=4,
113
trust_remote_code=True,
92
- fully_sharded_loras=fully_sharded,
114
+ fully_sharded_loras=True,
115
enable_chunked_prefill=True,
116
)
117
output_tp = do_sample(llm, minicpmv_lora_files, lora_id=1)
118
for i in range(len(EXPECTED_OUTPUT)):
119
assert EXPECTED_OUTPUT[i].startswith(output_tp[i])
120
+ output_tp = do_sample(llm, minicpmv_lora_files, lora_id=2)
121
+ for i in range(len(EXPECTED_OUTPUT)):
122
+ assert EXPECTED_OUTPUT[i].startswith(output_tp[i])
tests/lora/test_punica_sizes.py
+80
-86
@@ -4,6 +4,8 @@ hidden_sizes included in the LoRA models currently supported by vLLM. It tests
4
whether the corresponding Triton kernel can run normally when tensor parallelism
5
is set to [1, 2, 4, 8, 16, 32, 64].
6
"""
7
+from threading import Lock
8
+
9
import pytest
10
import torch
11
@@ -11,12 +13,13 @@ from vllm.lora.ops.bgmv_expand import bgmv_expand
13
from vllm.lora.ops.bgmv_expand_slice import bgmv_expand_slice
14
from vllm.lora.ops.bgmv_shrink import bgmv_shrink
15
from vllm.lora.ops.sgmv_expand import sgmv_expand
14
-from vllm.lora.ops.sgmv_expand_slice import sgmv_expand_slice
16
from vllm.lora.ops.sgmv_shrink import sgmv_shrink
17
+from vllm.lora.ops.utils import _LORA_A_PTR_DICT, _LORA_B_PTR_DICT
18
from vllm.platforms import current_platform
19
18
-from .utils import (generate_data, generate_data_for_expand_nslices,
19
- ref_torch_groupgemm)
20
+from .utils import (assert_close, generate_data,
21
+ generate_data_for_expand_nslices,
22
+ generate_data_for_nslices, ref_torch_groupgemm)
23
24
HIDDEN_SIZES = [
25
128,
@@ -112,14 +115,7 @@ SCALES = [0.5]
115
SEED = [0]
116
CUDA_DEVICES = [f"cuda:{0}"]
117
115
-
116
-def assert_close(a, b):
117
- rtol, atol = {
118
- torch.float16: (6e-2, 6e-2),
119
- torch.bfloat16: (6e-2, 6e-2),
120
- torch.float32: (1e-2, 1e-2),
121
- }[a.dtype]
122
- torch.testing.assert_close(a, b, rtol=rtol, atol=atol)
118
+_dict_lock = Lock()
119
120
121
@pytest.mark.parametrize("batches", BATCHES)
@@ -127,6 +123,7 @@ def assert_close(a, b):
123
@pytest.mark.parametrize("rank", MAX_RANKS)
124
@pytest.mark.parametrize("hidden_size", HIDDEN_SIZES)
125
@pytest.mark.parametrize("scaling", SCALES)
126
+@pytest.mark.parametrize("nslices", [1, 2, 3])
127
@pytest.mark.parametrize("dtype", DTYPES)
128
@pytest.mark.parametrize("op_type", ["shrink", "expand"])
129
@pytest.mark.parametrize("seed", SEED)
@@ -137,6 +134,7 @@ def test_punica_sgmv(
134
rank: int,
135
hidden_size: int,
136
scaling: float,
137
+ nslices: int,
138
dtype: torch.dtype,
139
op_type: str,
140
seed: int,
@@ -148,19 +146,20 @@ def test_punica_sgmv(
146
seq_length = 128
147
(
148
inputs_tensor,
151
- lora_weights,
149
+ lora_weights_lst,
150
our_out_tensor,
151
ref_out_tensor,
152
b_seq_start_loc,
153
lora_indices_tensor,
154
seq_len_tensor,
155
indices,
158
- ) = generate_data(
156
+ ) = generate_data_for_nslices(
157
batches,
158
hidden_size,
159
num_loras,
160
rank,
161
seq_length,
162
+ nslices,
163
dtype,
164
op_type,
165
device,
@@ -172,43 +171,64 @@ def test_punica_sgmv(
171
else:
172
max_seq_length = max_seq_length.item()
173
if op_type == "shrink":
175
- sgmv_shrink(
176
- inputs_tensor,
177
- lora_weights,
178
- our_out_tensor,
179
- b_seq_start_loc,
180
- seq_len_tensor,
181
- lora_indices_tensor,
182
- batches,
183
- max_seq_length,
184
- token_nums,
185
- scaling,
186
- )
174
+ # Preventing cache error pointer.
175
+ with _dict_lock:
176
+ _LORA_A_PTR_DICT.clear()
177
+ sgmv_shrink(
178
+ inputs_tensor,
179
+ lora_weights_lst,
180
+ our_out_tensor,
181
+ b_seq_start_loc,
182
+ seq_len_tensor,
183
+ lora_indices_tensor,
184
+ batches,
185
+ max_seq_length,
186
+ token_nums,
187
+ scaling,
188
+ )
189
+ for index in range(nslices):
190
+ ref_torch_groupgemm(
191
+ ref_out_tensor[index],
192
+ inputs_tensor,
193
+ lora_weights_lst[index],
194
+ lora_indices_tensor,
195
+ seq_len_tensor,
196
+ batches,
197
+ scaling,
198
+ op_type,
199
+ )
200
else:
188
- sgmv_expand(
189
- inputs_tensor,
190
- lora_weights,
191
- our_out_tensor,
192
- b_seq_start_loc,
193
- seq_len_tensor,
194
- lora_indices_tensor,
195
- batches,
196
- max_seq_length,
197
- token_nums,
198
- add_inputs=True,
199
- )
200
- ref_torch_groupgemm(
201
- ref_out_tensor,
202
- inputs_tensor,
203
- lora_weights,
204
- lora_indices_tensor,
205
- seq_len_tensor,
206
- batches,
207
- scaling if op_type == "shrink" else 1.0,
208
- op_type,
209
- )
210
- if op_type == "shrink":
211
- ref_out_tensor = ref_out_tensor.to(torch.float32)
201
+ with _dict_lock:
202
+ _LORA_B_PTR_DICT.clear()
203
+ sgmv_expand(
204
+ inputs_tensor,
205
+ lora_weights_lst,
206
+ our_out_tensor,
207
+ b_seq_start_loc,
208
+ seq_len_tensor,
209
+ lora_indices_tensor,
210
+ batches,
211
+ max_seq_length,
212
+ token_nums,
213
+ offset_start=0,
214
+ add_inputs=True,
215
+ )
216
+
217
+ slice_offset = 0
218
+ for index in range(nslices):
219
+ lora_weights = lora_weights_lst[index]
220
+ ref_torch_groupgemm(
221
+ ref_out_tensor[:, slice_offset:slice_offset + hidden_size],
222
+ inputs_tensor[index],
223
+ lora_weights,
224
+ lora_indices_tensor,
225
+ seq_len_tensor,
226
+ batches,
227
+ 1.0,
228
+ op_type,
229
+ )
230
+ slice_offset += hidden_size
231
+
232
assert_close(our_out_tensor, ref_out_tensor)
233
234
@@ -292,25 +312,22 @@ def test_punica_bgmv(
312
@pytest.mark.parametrize("hidden_size", HIDDEN_SIZES)
313
@pytest.mark.parametrize("nslices", [2, 3])
314
@pytest.mark.parametrize("dtype", DTYPES)
295
-@pytest.mark.parametrize("op_type", ["sgmv", "bgmv"])
315
@pytest.mark.parametrize("seed", SEED)
316
@pytest.mark.parametrize("device", CUDA_DEVICES)
298
-def test_punica_expand_nslices(
317
+def test_punica_bgmv_expand_nslices(
318
batches: int,
319
num_loras: int,
320
rank: int,
321
hidden_size: int,
322
nslices: int,
323
dtype: torch.dtype,
305
- op_type: str,
324
seed: int,
325
device: str,
326
):
309
-
327
torch.set_default_device(device)
328
current_platform.seed_everything(seed)
329
313
- seq_length = 128 if op_type == "sgmv" else 1
330
+ seq_length = 1
331
(
332
inputs_tensor,
333
lora_weights_lst,
@@ -330,41 +347,18 @@ def test_punica_expand_nslices(
347
nslices,
348
device,
349
)
333
- max_seq_length = seq_len_tensor.max()
334
- token_nums = seq_len_tensor.sum().item()
335
- if isinstance(max_seq_length, tuple):
336
- max_seq_length = max_seq_length[0].item()
337
- else:
338
- max_seq_length = max_seq_length.item()
350
slice_offset = 0
351
for index in range(nslices):
352
lora_weights = lora_weights_lst[index]
342
- if op_type == "sgmv":
343
- sgmv_expand_slice(
344
- inputs_tensor,
345
- lora_weights,
346
- our_outputs,
347
- b_seq_start_loc,
348
- seq_len_tensor,
349
- lora_indices_tensor,
350
- batches,
351
- max_seq_length,
352
- token_nums,
353
- slice_offset,
354
- hidden_size,
355
- add_inputs=True,
356
- )
357
- else:
358
-
359
- bgmv_expand_slice(
360
- inputs_tensor,
361
- lora_weights,
362
- our_outputs,
363
- indices,
364
- slice_offset,
365
- slice_size=hidden_size,
366
- add_inputs=True,
367
- )
353
+ bgmv_expand_slice(
354
+ inputs_tensor,
355
+ lora_weights,
356
+ our_outputs,
357
+ indices,
358
+ slice_offset,
359
+ slice_size=hidden_size,
360
+ add_inputs=True,
361
+ )
362
ref_torch_groupgemm(
363
ref_outputs[:, slice_offset:slice_offset + hidden_size],
364
inputs_tensor,
tests/lora/test_punica_variation.py
+81
-87
@@ -3,6 +3,8 @@ This script is mainly used to test whether trtion kernels can run normally
3
under different conditions, including various batches, numbers of LoRA , and
4
maximum ranks.
5
"""
6
+from threading import Lock
7
+
8
import pytest
9
import torch
10
@@ -11,12 +13,13 @@ import vllm.lora.ops.bgmv_expand
13
import vllm.lora.ops.bgmv_expand_slice
14
import vllm.lora.ops.bgmv_shrink
15
import vllm.lora.ops.sgmv_expand
14
-import vllm.lora.ops.sgmv_expand_slice
16
import vllm.lora.ops.sgmv_shrink # noqa: F401
17
+from vllm.lora.ops.utils import _LORA_A_PTR_DICT, _LORA_B_PTR_DICT
18
from vllm.platforms import current_platform
19
18
-from .utils import (generate_data, generate_data_for_expand_nslices,
19
- ref_torch_groupgemm)
20
+from .utils import (assert_close, generate_data,
21
+ generate_data_for_expand_nslices,
22
+ generate_data_for_nslices, ref_torch_groupgemm)
23
24
HIDDEN_SIZES = [4097]
25
@@ -28,31 +31,23 @@ SCALES = [0.5]
31
SEED = [0]
32
CUDA_DEVICES = [f"cuda:{0}"]
33
31
-
32
-def assert_close(a, b):
33
- rtol, atol = {
34
- torch.float16: (6e-2, 6e-2),
35
- torch.bfloat16: (6e-2, 6e-2),
36
- torch.float32: (1e-2, 1e-2),
37
- }[a.dtype]
38
- torch.testing.assert_close(a, b, rtol=rtol, atol=atol)
39
-
40
-
34
# Unlike test_punica_sizes.py, we directly utilize custom op for
35
# testing, which verifies the correct registration of these ops.
36
bgmv_expand = torch.ops.vllm.bgmv_expand
37
bgmv_expand_slice = torch.ops.vllm.bgmv_expand_slice
38
bgmv_shrink = torch.ops.vllm.bgmv_shrink
39
sgmv_expand = torch.ops.vllm.sgmv_expand
47
-sgmv_expand_slice = torch.ops.vllm.sgmv_expand_slice
40
sgmv_shrink = torch.ops.vllm.sgmv_shrink
41
42
+_dict_lock = Lock()
43
+
44
45
@pytest.mark.parametrize("batches", BATCHES)
46
@pytest.mark.parametrize("num_loras", NUM_LORA)
47
@pytest.mark.parametrize("rank", MAX_RANKS)
48
@pytest.mark.parametrize("hidden_size", HIDDEN_SIZES)
49
@pytest.mark.parametrize("scaling", SCALES)
50
+@pytest.mark.parametrize("nslices", [1, 2, 3])
51
@pytest.mark.parametrize("dtype", DTYPES)
52
@pytest.mark.parametrize("op_type", ["shrink", "expand"])
53
@pytest.mark.parametrize("seed", SEED)
@@ -63,6 +58,7 @@ def test_punica_sgmv(
58
rank: int,
59
hidden_size: int,
60
scaling: float,
61
+ nslices: int,
62
dtype: torch.dtype,
63
op_type: str,
64
seed: int,
@@ -74,19 +70,20 @@ def test_punica_sgmv(
70
seq_length = 128
71
(
72
inputs_tensor,
77
- lora_weights,
73
+ lora_weights_lst,
74
our_out_tensor,
75
ref_out_tensor,
76
b_seq_start_loc,
77
lora_indices_tensor,
78
seq_len_tensor,
79
indices,
84
- ) = generate_data(
80
+ ) = generate_data_for_nslices(
81
batches,
82
hidden_size,
83
num_loras,
84
rank,
85
seq_length,
86
+ nslices,
87
dtype,
88
op_type,
89
device,
@@ -98,43 +95,64 @@ def test_punica_sgmv(
95
else:
96
max_seq_length = max_seq_length.item()
97
if op_type == "shrink":
101
- sgmv_shrink(
102
- inputs_tensor,
103
- lora_weights,
104
- our_out_tensor,
105
- b_seq_start_loc,
106
- seq_len_tensor,
107
- lora_indices_tensor,
108
- batches,
109
- max_seq_length,
110
- token_nums,
111
- scaling,
112
- )
98
+ # Preventing cache error pointer.
99
+ with _dict_lock:
100
+ _LORA_A_PTR_DICT.clear()
101
+ sgmv_shrink(
102
+ inputs_tensor,
103
+ lora_weights_lst,
104
+ our_out_tensor,
105
+ b_seq_start_loc,
106
+ seq_len_tensor,
107
+ lora_indices_tensor,
108
+ batches,
109
+ max_seq_length,
110
+ token_nums,
111
+ scaling,
112
+ )
113
+ for index in range(nslices):
114
+ ref_torch_groupgemm(
115
+ ref_out_tensor[index],
116
+ inputs_tensor,
117
+ lora_weights_lst[index],
118
+ lora_indices_tensor,
119
+ seq_len_tensor,
120
+ batches,
121
+ scaling,
122
+ op_type,
123
+ )
124
else:
114
- sgmv_expand(
115
- inputs_tensor,
116
- lora_weights,
117
- our_out_tensor,
118
- b_seq_start_loc,
119
- seq_len_tensor,
120
- lora_indices_tensor,
121
- batches,
122
- max_seq_length,
123
- token_nums,
124
- add_inputs=True,
125
- )
126
- ref_torch_groupgemm(
127
- ref_out_tensor,
128
- inputs_tensor,
129
- lora_weights,
130
- lora_indices_tensor,
131
- seq_len_tensor,
132
- batches,
133
- scaling if op_type == "shrink" else 1.0,
134
- op_type,
135
- )
136
- if op_type == "shrink":
137
- ref_out_tensor = ref_out_tensor.to(torch.float32)
125
+ with _dict_lock:
126
+ _LORA_B_PTR_DICT.clear()
127
+ sgmv_expand(
128
+ inputs_tensor,
129
+ lora_weights_lst,
130
+ our_out_tensor,
131
+ b_seq_start_loc,
132
+ seq_len_tensor,
133
+ lora_indices_tensor,
134
+ batches,
135
+ max_seq_length,
136
+ token_nums,
137
+ offset_start=0,
138
+ add_inputs=True,
139
+ )
140
+
141
+ slice_offset = 0
142
+ for index in range(nslices):
143
+ lora_weights = lora_weights_lst[index]
144
+ ref_torch_groupgemm(
145
+ ref_out_tensor[:, slice_offset:slice_offset + hidden_size],
146
+ inputs_tensor[index],
147
+ lora_weights,
148
+ lora_indices_tensor,
149
+ seq_len_tensor,
150
+ batches,
151
+ 1.0,
152
+ op_type,
153
+ )
154
+ slice_offset += hidden_size
155
+
156
assert_close(our_out_tensor, ref_out_tensor)
157
158
@@ -220,24 +238,22 @@ def test_punica_bgmv(
238
@pytest.mark.parametrize("hidden_size", HIDDEN_SIZES)
239
@pytest.mark.parametrize("nslices", [2, 3])
240
@pytest.mark.parametrize("dtype", DTYPES)
223
-@pytest.mark.parametrize("op_type", ["sgmv", "bgmv"])
241
@pytest.mark.parametrize("seed", SEED)
242
@pytest.mark.parametrize("device", CUDA_DEVICES)
226
-def test_punica_expand_nslices(
243
+def test_punica_bgmv_expand_nslices(
244
batches: int,
245
num_loras: int,
246
rank: int,
247
hidden_size: int,
248
nslices: int,
249
dtype: torch.dtype,
233
- op_type: str,
250
seed: int,
251
device: str,
252
):
253
torch.set_default_device(device)
254
current_platform.seed_everything(seed)
255
240
- seq_length = 128 if op_type == "sgmv" else 1
256
+ seq_length = 1
257
(
258
inputs_tensor,
259
lora_weights_lst,
@@ -257,40 +273,18 @@ def test_punica_expand_nslices(
273
nslices,
274
device,
275
)
260
- max_seq_length = seq_len_tensor.max()
261
- token_nums = seq_len_tensor.sum().item()
262
- if isinstance(max_seq_length, tuple):
263
- max_seq_length = max_seq_length[0].item()
264
- else:
265
- max_seq_length = max_seq_length.item()
276
slice_offset = 0
277
for index in range(nslices):
278
lora_weights = lora_weights_lst[index]
269
- if op_type == "sgmv":
270
- sgmv_expand_slice(
271
- inputs_tensor,
272
- lora_weights,
273
- our_outputs,
274
- b_seq_start_loc,
275
- seq_len_tensor,
276
- lora_indices_tensor,
277
- batches,
278
- max_seq_length,
279
- token_nums,
280
- slice_offset,
281
- hidden_size,
282
- add_inputs=True,
283
- )
284
- else:
285
- bgmv_expand_slice(
286
- inputs_tensor,
287
- lora_weights,
288
- our_outputs,
289
- indices,
290
- slice_offset,
291
- slice_size=hidden_size,
292
- add_inputs=True,
293
- )
279
+ bgmv_expand_slice(
280
+ inputs_tensor,
281
+ lora_weights,
282
+ our_outputs,
283
+ indices,
284
+ slice_offset,
285
+ slice_size=hidden_size,
286
+ add_inputs=True,
287
+ )
288
ref_torch_groupgemm(
289
ref_outputs[:, slice_offset:slice_offset + hidden_size],
290
inputs_tensor,
tests/lora/utils.py
+123
-21
@@ -18,11 +18,13 @@ class DummyLoRAManager:
18
def get_module_lora(self, module_name: str) -> LoRALayerWeights:
19
return self._loras[module_name]
20
21
- def init_random_lora(self,
22
- module_name: str,
23
- weight: torch.Tensor,
24
- rank: int = 8,
25
- generate_embeddings_tensor: int = 0):
21
+ def init_random_lora(
22
+ self,
23
+ module_name: str,
24
+ weight: torch.Tensor,
25
+ rank: int = 8,
26
+ generate_embeddings_tensor: int = 0,
27
+ ):
28
lora = LoRALayerWeights(
29
module_name,
30
rank=rank,
@@ -35,21 +37,25 @@ class DummyLoRAManager:
37
device=self._device),
38
)
39
if generate_embeddings_tensor:
38
- lora.embeddings_tensor = torch.rand(5,
39
- generate_embeddings_tensor,
40
- dtype=weight.dtype,
41
- device=self._device)
40
+ lora.embeddings_tensor = torch.rand(
41
+ 5,
42
+ generate_embeddings_tensor,
43
+ dtype=weight.dtype,
44
+ device=self._device,
45
+ )
46
self.set_module_lora(module_name, lora)
47
48
return lora
49
46
- def init_lora(self,
47
- module_name: str,
48
- input_dim: int,
49
- output_dim: int,
50
- rank=8,
51
- noop=False,
52
- embeddings_tensor=None):
50
+ def init_lora(
51
+ self,
52
+ module_name: str,
53
+ input_dim: int,
54
+ output_dim: int,
55
+ rank=8,
56
+ noop=False,
57
+ embeddings_tensor=None,
58
+ ):
59
lora = LoRALayerWeights(
60
module_name,
61
rank=rank,
@@ -125,8 +131,16 @@ def ref_torch_groupgemm(
131
return
132
133
128
-def generate_data(batches, hidden_size, lora_nums, max_rank, seq_length, dtype,
129
- op_type, device):
134
+def generate_data(
135
+ batches,
136
+ hidden_size,
137
+ lora_nums,
138
+ max_rank,
139
+ seq_length,
140
+ dtype,
141
+ op_type,
142
+ device,
143
+):
144
seq_len_tensor = torch.randint(seq_length, seq_length + 1,
145
(batches, )).to(device)
146
b_seq_start_loc = torch.cumsum(
@@ -187,8 +201,16 @@ def generate_data(batches, hidden_size, lora_nums, max_rank, seq_length, dtype,
201
)
202
203
190
-def generate_data_for_expand_nslices(batches, hidden_size, lora_nums, max_rank,
191
- seq_length, dtype, nslices, device):
204
+def generate_data_for_expand_nslices(
205
+ batches,
206
+ hidden_size,
207
+ lora_nums,
208
+ max_rank,
209
+ seq_length,
210
+ dtype,
211
+ nslices,
212
+ device,
213
+):
214
seq_len_tensor = torch.randint(seq_length, seq_length + 1,
215
(batches, )).to(device)
216
b_seq_start_loc = torch.cumsum(
@@ -221,7 +243,87 @@ def generate_data_for_expand_nslices(batches, hidden_size, lora_nums, max_rank,
243
for b_id in range(batches):
244
lora_index = lora_indices_tensor[b_id]
245
indices[current_offset:current_offset +
224
- seq_len_tensor[b_id]] = lora_index.item()
246
+ seq_len_tensor[b_id]] = (lora_index.item())
247
+ current_offset += seq_len_tensor[b_id].item()
248
+
249
+ lora_indices_tensor = lora_indices_tensor.to(device)
250
+ return (
251
+ inputs_tensor,
252
+ lora_weights_lst,
253
+ our_out_tensor,
254
+ ref_out_tensor,
255
+ b_seq_start_loc,
256
+ lora_indices_tensor,
257
+ seq_len_tensor,
258
+ indices,
259
+ )
260
+
261
+
262
+def generate_data_for_nslices(
263
+ batches,
264
+ hidden_size,
265
+ lora_nums,
266
+ max_rank,
267
+ seq_length,
268
+ nslices,
269
+ dtype,
270
+ op_type,
271
+ device,
272
+):
273
+ seq_len_tensor = torch.randint(seq_length, seq_length + 1,
274
+ (batches, )).to(device)
275
+ b_seq_start_loc = torch.cumsum(
276
+ torch.tensor([0] + seq_len_tensor[:-1].tolist(), dtype=torch.long),
277
+ dim=0,
278
+ ).to(device)
279
+ total_tokens = seq_len_tensor.sum()
280
+
281
+ lora_weights_lst = []
282
+ if op_type == "shrink":
283
+
284
+ inputs_tensor = torch.rand((total_tokens, hidden_size),
285
+ dtype=dtype).to(device)
286
+
287
+ for _ in range(nslices):
288
+ if op_type == "shrink":
289
+ lora_weights_lst.append(
290
+ torch.rand(
291
+ (lora_nums, max_rank, hidden_size), # col-major
292
+ dtype=dtype,
293
+ ).to(device))
294
+ # NOTE shrink kernel using torch.float32 as output type
295
+ # shrink op need atomic_add, so output is initinized by 0
296
+ our_out_tensor = torch.zeros(
297
+ (nslices, total_tokens, max_rank),
298
+ dtype=torch.float32,
299
+ ).to(device)
300
+ else:
301
+ inputs_tensor = torch.rand(
302
+ (nslices, total_tokens, max_rank),
303
+ dtype=dtype,
304
+ ).to(device)
305
+ for _ in range(nslices):
306
+ lora_weights_lst.append(
307
+ torch.rand(
308
+ (lora_nums, hidden_size, max_rank), # col-major
309
+ dtype=dtype,
310
+ ).to(device))
311
+ # expand op needs to complete y+=a@lora_b, so output is
312
+ # initinized randomly
313
+ our_out_tensor = torch.rand((total_tokens, hidden_size * nslices),
314
+ dtype=dtype).to(device)
315
+
316
+ # Ensure the same input.
317
+ ref_out_tensor = our_out_tensor.clone()
318
+ lora_indices_tensor = torch.randint(0,
319
+ lora_nums - 1 if lora_nums > 1 else 1,
320
+ (batches, ))
321
+ indices = torch.zeros((total_tokens), dtype=torch.long).to(device)
322
+ current_offset = 0
323
+ for b_id in range(batches):
324
+ lora_index = lora_indices_tensor[b_id]
325
+ indices[current_offset:current_offset +
326
+ seq_len_tensor[b_id]] = (lora_index.item())
327
current_offset += seq_len_tensor[b_id].item()
328
329
lora_indices_tensor = lora_indices_tensor.to(device)
vllm/lora/ops/sgmv_expand.py
+128
-77
@@ -1,66 +1,109 @@
1
"""
2
Based on:
3
-Chen, L., Ye, Z., Wu, Y., Zhuo, D., Ceze, L., & Krishnamurthy, A. (2023).
4
-Punica: Multi-Tenant LoRA Serving.
3
+Chen, L., Ye, Z., Wu, Y., Zhuo, D., Ceze, L., & Krishnamurthy, A. (2023).
4
+Punica: Multi-Tenant LoRA Serving.
5
https://arxiv.org/abs/2310.18547
6
"""
7
8
+from typing import List
9
+
10
import torch
11
import triton
12
import triton.language as tl
13
14
from vllm.utils import direct_register_custom_op
15
16
+from .utils import _get_lora_b_ptr
17
+
18
19
@triton.jit
20
def _sgmv_expand_kernel(
17
- input_ptr,
18
- lora_ptr,
19
- out_ptr,
20
- N,
21
- K,
22
- b_seq_start_loc,
23
- seq_lens,
24
- lora_indices,
25
- xm_stride,
26
- xk_stride, # 1
27
- l0_stride, # hidden_size*max_rank
28
- lora_k_stride,
29
- lora_n_stride,
30
- cm_stride,
31
- cn_stride,
32
- BLOCK_M: tl.constexpr,
33
- BLOCK_N: tl.constexpr,
34
- BLOCK_K: tl.constexpr,
35
- EVEN_K: tl.constexpr,
36
- ADD_INPUTS: tl.constexpr,
37
- CAST_TYPE: tl.constexpr,
38
-):
21
+ input_ptr,
22
+ lora_ptr,
23
+ out_ptr,
24
+ N,
25
+ K,
26
+ b_seq_start_loc,
27
+ seq_lens,
28
+ lora_indices,
29
+ slice_start_loc,
30
+ input_d0_stride,
31
+ input_d1_stride,
32
+ input_d2_stride, # 1
33
+ ls_d0_ptr,
34
+ ls_d1_ptr,
35
+ ls_d2_ptr, # 1
36
+ output_d0_stride,
37
+ output_d1_stride, # 1
38
+ output_hs_ptr,
39
+ BLOCK_M: tl.constexpr,
40
+ BLOCK_N: tl.constexpr,
41
+ BLOCK_K: tl.constexpr,
42
+ EVEN_K: tl.constexpr,
43
+ ADD_INPUTS: tl.constexpr,
44
+ CAST_TYPE: tl.constexpr,
45
+ SLICE_NUM: tl.constexpr,
46
+ SAME_STRIDE: tl.constexpr):
47
"""
40
- The sgmv's expand triton kernel is based on GroupGEMM.
48
+
49
+ Similar to the 'sgmv_expand' operator, but with an added parameter
50
+ 'slice_offset'. The reason for not reusing the 'sgmv_expand' operator
51
+ might be that in the future, we could implement a fusion operator to
52
+ achieve the current functionality instead of having to call it multiple
53
+ times.
54
"""
55
pid = tl.program_id(axis=0)
56
cur_batch = tl.program_id(axis=1)
57
+ slice_id = tl.program_id(axis=2)
58
cta_n_num = tl.cdiv(N, BLOCK_N)
59
+ # When the output dimensions of each slice are the same,cur_n=N, otherwise
60
+ # cur_n=tl.load(output_hs_ptr + slice_id), this situation exists in GQA's
61
+ # qkv linear.
62
+ curr_N = N if SAME_STRIDE else tl.load(output_hs_ptr + slice_id)
63
pid_m = pid // cta_n_num
64
pid_n = pid % cta_n_num
65
M = tl.load(seq_lens + cur_batch)
66
if pid_m * BLOCK_M > M:
67
return
68
+ if pid_n * BLOCK_N > curr_N:
69
+ return
70
lora_index = tl.load(lora_indices + cur_batch)
71
if lora_index == -1:
72
return
73
+
74
cur_seq_start = tl.load(b_seq_start_loc + cur_batch)
75
offset_m = tl.arange(0, BLOCK_M) + pid_m * BLOCK_M
76
offset_n = tl.arange(0, BLOCK_N) + pid_n * BLOCK_N
77
offset_k = tl.arange(0, BLOCK_K)
78
ram = tl.max_contiguous(tl.multiple_of(offset_m % M, BLOCK_M), BLOCK_M)
58
- rbn = tl.max_contiguous(tl.multiple_of(offset_n % N, BLOCK_N), BLOCK_N)
79
+ rbn = tl.max_contiguous(tl.multiple_of(offset_n % curr_N, BLOCK_N),
80
+ BLOCK_N)
81
+ # ls_d*_ptr can be either an integer or a pointer
82
+ if SAME_STRIDE:
83
+ # integer
84
+ cur_lora_d0_stride = ls_d0_ptr
85
+ cur_lora_d1_stride = ls_d1_ptr
86
+ cur_lora_d2_stride = ls_d2_ptr
87
+ else:
88
+ # pointer
89
+ cur_lora_d0_stride = tl.load(ls_d0_ptr + slice_id)
90
+ cur_lora_d1_stride = tl.load(ls_d1_ptr + slice_id)
91
+ cur_lora_d2_stride = tl.load(ls_d2_ptr + slice_id)
92
+ if SLICE_NUM == 1:
93
+ cur_input_ptr = input_ptr
94
+ cur_lora_ptr = lora_ptr
95
60
- a_ptr = (input_ptr + cur_seq_start * xm_stride + ram[:, None] * xm_stride +
61
- offset_k[None, :] * xk_stride, )
62
- b_ptr = (lora_ptr + l0_stride * lora_index +
63
- offset_k[:, None] * lora_n_stride + rbn[None, :] * lora_k_stride)
96
+ else:
97
+ cur_input_ptr = input_ptr + slice_id * input_d0_stride
98
+ cur_lora_ptr = tl.load(lora_ptr + slice_id).to(
99
+ tl.pointer_type(out_ptr.dtype.element_ty))
100
+
101
+ a_ptr = (cur_input_ptr + cur_seq_start * input_d1_stride +
102
+ ram[:, None] * input_d1_stride +
103
+ offset_k[None, :] * input_d2_stride, )
104
+ b_ptr = (cur_lora_ptr + cur_lora_d0_stride * lora_index +
105
+ offset_k[:, None] * cur_lora_d2_stride +
106
+ rbn[None, :] * cur_lora_d1_stride)
107
accumulator = tl.zeros((BLOCK_M, BLOCK_N), dtype=tl.float32)
108
for k in range(tl.cdiv(K, BLOCK_K)):
109
if EVEN_K:
@@ -74,26 +117,30 @@ def _sgmv_expand_kernel(
117
mask=offset_k[:, None] < K - k * BLOCK_K,
118
other=0)
119
if CAST_TYPE:
77
- tiled_a = tiled_a.to(lora_ptr.dtype.element_ty)
120
+ tiled_a = tiled_a.to(cur_lora_ptr.dtype.element_ty)
121
accumulator += tl.dot(
122
tiled_a,
123
tiled_b,
124
)
82
- a_ptr += BLOCK_K * xk_stride
83
- b_ptr += BLOCK_K * lora_n_stride
84
- tiled_c = accumulator.to(lora_ptr.dtype.element_ty)
125
+ a_ptr += BLOCK_K * input_d2_stride
126
+ b_ptr += BLOCK_K * cur_lora_d2_stride
127
+
128
+ tiled_c = accumulator.to(cur_lora_ptr.dtype.element_ty)
129
+ if SLICE_NUM == 1:
130
+ cur_slice_start = slice_start_loc
131
+ else:
132
+ cur_slice_start = tl.load(slice_start_loc + slice_id)
133
+
134
offset_cm = cur_seq_start + tl.arange(0, BLOCK_M) + pid_m * BLOCK_M
86
- offset_cn = tl.arange(0, BLOCK_N) + pid_n * BLOCK_N
87
- c_ptr = (out_ptr + offset_cm[:, None] * cm_stride +
88
- offset_cn[None, :] * cn_stride)
135
+ offset_cn = tl.arange(0, BLOCK_N) + pid_n * BLOCK_N + cur_slice_start
136
+ c_ptr = (out_ptr + offset_cm[:, None] * output_d0_stride +
137
+ offset_cn[None, :] * output_d1_stride)
138
M = tl.load(seq_lens + cur_batch)
139
c_mask = (offset_cm[:, None] <
91
- (cur_seq_start + M)) & (offset_cn[None, :] < N)
140
+ (cur_seq_start + M)) & (offset_cn[None, :] <
141
+ (cur_slice_start + curr_N))
142
if ADD_INPUTS:
93
- # explicitly pass in other=None to tell triton that masked values
94
- # can be uninitialized. This is OK because the later tl.store operation
95
- # uses the same mask, eliminating the risk of garbage values propagating
96
- tiled_out = tl.load(c_ptr, mask=c_mask, other=None)
143
+ tiled_out = tl.load(c_ptr, mask=c_mask)
144
tiled_c += tiled_out
145
tl.store(c_ptr, tiled_c, mask=c_mask)
146
@@ -101,7 +148,7 @@ def _sgmv_expand_kernel(
148
@torch.inference_mode()
149
def _sgmv_expand(
150
inputs: torch.Tensor,
104
- lora_b_weights: torch.Tensor,
151
+ lora_b_weights: List[torch.Tensor],
152
output_tensor: torch.Tensor,
153
b_seq_start_loc: torch.Tensor,
154
seq_len_tensor: torch.Tensor,
@@ -109,17 +156,18 @@ def _sgmv_expand(
156
batches: int,
157
max_seq_length: int,
158
token_nums: int,
159
+ offset_start: int = 0,
160
add_inputs: bool = False,
161
) -> None:
162
"""
163
Args:
164
inputs (torch.Tensor): input tensor
117
- lora_b_weights (torch.Tensor): lora'a weight
165
+ lora_b_weights (List[torch.Tensor]): lora'b weight
166
output_tensor (torch.Tensor): output tensor
167
b_seq_start_loc (torch.Tensor): (batch_size,). The cumulative
168
sequence lengths of the sequences in the batch, used to index
169
into sequence. E.g., if the sequence length is [4, 6], it is
122
- [0, 4, 10].
170
+ [0, 4].
171
seq_len_tensor (torch.Tensor): (batch_size,). Record the sequence
172
length of the sequences in the batch.
173
lora_indices_tensor (torch.Tensor): (batch_size,). The LoRA index
@@ -130,77 +178,80 @@ def _sgmv_expand(
178
batch.
179
token_nums (int): The token numbers in the batch. Used to verify if the
180
token numbers in the inputs matches the one in the metadata.
133
- add_inputs (bool, optional): Defaults to False, adds the final lora
134
- results to the output.
181
+ offset_start (int, optional): Offset start for output_tensor.
182
+ Defaults to 0.
183
+ add_inputs (bool, optional): Whether to add the input tensor to the
184
+ output tensor. Defaults to False.
185
"""
136
-
186
assert inputs.dtype in [torch.float16, torch.bfloat16, torch.float32]
138
- assert lora_b_weights.dtype in [
139
- torch.float16,
140
- torch.bfloat16,
141
- ]
142
- assert inputs.size(0) == token_nums
143
- assert inputs.size(1) == lora_b_weights.size(-1)
187
+ for weight in lora_b_weights:
188
+ assert weight.dtype in [torch.float16, torch.bfloat16]
189
+
190
+ assert inputs.size(1) == token_nums
191
+ assert inputs.size(0) == len(lora_b_weights)
192
+
193
assert b_seq_start_loc.size(0) == batches
194
assert lora_indices_tensor.size(0) == batches
146
- assert inputs.is_contiguous()
195
assert output_tensor.is_contiguous()
148
-
149
- if lora_b_weights.ndim == 4: # shape:(lora_num,1,size,rank)
150
- assert lora_b_weights.size(1) == 1
151
- lora_b_weights = lora_b_weights.squeeze(dim=1)
152
- else:
153
- assert lora_b_weights.ndim == 3 # shape:(lora_num,size,rank)
154
-
155
- assert lora_b_weights.is_contiguous()
196
+ (slice_start_tensor, lora_ptr_tensor, lora_strides_d0_tensor,
197
+ lora_strides_d1_tensor, lora_strides_d2_tensor, hidden_sizes_tensor,
198
+ same_stride, MAX_N) = _get_lora_b_ptr(lora_b_weights, offset_start,
199
+ b_seq_start_loc.device)
200
201
# TODO tuning this config
202
+ K = lora_b_weights[0].shape[-1] # K= rank
203
159
- N, K = lora_b_weights.shape[-2:] # K= rank,N=hidden_size
160
- BLOCK_M = 32
161
- BLOCK_N = 32
204
+ BLOCK_M = 64
205
+ BLOCK_N = 128
206
BLOCK_K = 16
207
EVEN_K = K % BLOCK_K == 0
208
ADD_INPUTS = add_inputs
209
CAST_TYPE = False
166
- if inputs.dtype == torch.float32 and lora_b_weights.dtype in [
210
+
211
+ if inputs.dtype == torch.float32 and lora_b_weights[0].dtype in [
212
torch.float16,
213
torch.bfloat16,
214
]:
215
CAST_TYPE = True
216
grid = (
172
- triton.cdiv(max_seq_length, BLOCK_M) * triton.cdiv(N, BLOCK_N),
217
+ triton.cdiv(max_seq_length, BLOCK_M) * triton.cdiv(MAX_N, BLOCK_N),
218
batches,
219
+ len(lora_b_weights),
220
)
221
_sgmv_expand_kernel[grid](
222
inputs,
177
- lora_b_weights,
223
+ lora_ptr_tensor,
224
output_tensor,
179
- N,
225
+ MAX_N,
226
K,
227
b_seq_start_loc,
228
seq_len_tensor,
229
lora_indices_tensor,
230
+ slice_start_tensor,
231
inputs.stride(0),
232
inputs.stride(1),
186
- lora_b_weights.stride(0),
187
- lora_b_weights.stride(1),
188
- lora_b_weights.stride(2),
233
+ inputs.stride(2),
234
+ lora_strides_d0_tensor,
235
+ lora_strides_d1_tensor,
236
+ lora_strides_d2_tensor,
237
output_tensor.stride(0),
238
output_tensor.stride(1),
239
+ hidden_sizes_tensor,
240
BLOCK_M,
241
BLOCK_N,
242
BLOCK_K,
243
EVEN_K,
244
ADD_INPUTS,
245
CAST_TYPE,
246
+ len(lora_b_weights),
247
+ same_stride,
248
)
249
return
250
251
201
-def sgmv_expand_fake(
252
+def _sgmv_expand_fake(
253
inputs: torch.Tensor,
203
- lora_b_weights: torch.Tensor,
254
+ lora_b_weights: List[torch.Tensor],
255
output_tensor: torch.Tensor,
256
b_seq_start_loc: torch.Tensor,
257
seq_len_tensor: torch.Tensor,
@@ -208,18 +259,18 @@ def sgmv_expand_fake(
259
batches: int,
260
max_seq_length: int,
261
token_nums: int,
262
+ offset_start: int = 0,
263
add_inputs: bool = False,
264
) -> None:
265
return
266
267
268
try:
217
-
269
direct_register_custom_op(
270
op_name="sgmv_expand",
271
op_func=_sgmv_expand,
272
mutates_args=["output_tensor"],
222
- fake_impl=sgmv_expand_fake,
273
+ fake_impl=_sgmv_expand_fake,
274
)
275
sgmv_expand = torch.ops.vllm.sgmv_expand
276
vllm/lora/ops/sgmv_expand_slice.py
deleted
-241
@@ -1,241 +0,0 @@
1
-"""
2
-Based on:
3
-Chen, L., Ye, Z., Wu, Y., Zhuo, D., Ceze, L., & Krishnamurthy, A. (2023).
4
-Punica: Multi-Tenant LoRA Serving.
5
-https://arxiv.org/abs/2310.18547
6
-"""
7
-
8
-import torch
9
-import triton
10
-import triton.language as tl
11
-
12
-from vllm.utils import direct_register_custom_op
13
-
14
-
15
-@triton.jit
16
-def _sgmv_expand_slice_kernel(
17
- input_ptr,
18
- lora_ptr,
19
- out_ptr,
20
- N,
21
- K,
22
- b_seq_start_loc,
23
- seq_lens,
24
- lora_indices,
25
- xm_stride,
26
- xk_stride, # 1
27
- l0_stride, # hidden_size*max_rank
28
- lora_k_stride,
29
- lora_n_stride,
30
- cm_stride,
31
- cn_stride,
32
- slice_offset,
33
- BLOCK_M: tl.constexpr,
34
- BLOCK_N: tl.constexpr,
35
- BLOCK_K: tl.constexpr,
36
- EVEN_K: tl.constexpr,
37
- ADD_INPUTS: tl.constexpr,
38
- CAST_TYPE: tl.constexpr,
39
-):
40
- """
41
-
42
- Similar to the 'sgmv_expand' operator, but with an added parameter
43
- 'slice_offset'. The reason for not reusing the 'sgmv_expand' operator
44
- might be that in the future, we could implement a fusion operator to
45
- achieve the current functionality instead of having to call it multiple
46
- times.
47
- """
48
- pid = tl.program_id(axis=0)
49
- cur_batch = tl.program_id(axis=1)
50
- cta_n_num = tl.cdiv(N, BLOCK_N)
51
- pid_m = pid // cta_n_num
52
- pid_n = pid % cta_n_num
53
- M = tl.load(seq_lens + cur_batch)
54
- if pid_m * BLOCK_M > M:
55
- return
56
- lora_index = tl.load(lora_indices + cur_batch)
57
- if lora_index == -1:
58
- return
59
- cur_seq_start = tl.load(b_seq_start_loc + cur_batch)
60
- offset_m = tl.arange(0, BLOCK_M) + pid_m * BLOCK_M
61
- offset_n = tl.arange(0, BLOCK_N) + pid_n * BLOCK_N
62
- offset_k = tl.arange(0, BLOCK_K)
63
- ram = tl.max_contiguous(tl.multiple_of(offset_m % M, BLOCK_M), BLOCK_M)
64
- rbn = tl.max_contiguous(tl.multiple_of(offset_n % N, BLOCK_N), BLOCK_N)
65
-
66
- a_ptr = (input_ptr + cur_seq_start * xm_stride + ram[:, None] * xm_stride +
67
- offset_k[None, :] * xk_stride, )
68
- b_ptr = (lora_ptr + l0_stride * lora_index +
69
- offset_k[:, None] * lora_n_stride + rbn[None, :] * lora_k_stride)
70
- accumulator = tl.zeros((BLOCK_M, BLOCK_N), dtype=tl.float32)
71
- for k in range(tl.cdiv(K, BLOCK_K)):
72
- if EVEN_K:
73
- tiled_a = tl.load(a_ptr)
74
- tiled_b = tl.load(b_ptr)
75
- else:
76
- tiled_a = tl.load(a_ptr,
77
- mask=offset_k[None, :] < K - k * BLOCK_K,
78
- other=0)
79
- tiled_b = tl.load(b_ptr,
80
- mask=offset_k[:, None] < K - k * BLOCK_K,
81
- other=0)
82
- if CAST_TYPE:
83
- tiled_a = tiled_a.to(lora_ptr.dtype.element_ty)
84
- accumulator += tl.dot(
85
- tiled_a,
86
- tiled_b,
87
- )
88
- a_ptr += BLOCK_K * xk_stride
89
- b_ptr += BLOCK_K * lora_n_stride
90
- tiled_c = accumulator.to(lora_ptr.dtype.element_ty)
91
- offset_cm = cur_seq_start + tl.arange(0, BLOCK_M) + pid_m * BLOCK_M
92
- offset_cn = tl.arange(0, BLOCK_N) + pid_n * BLOCK_N + slice_offset
93
- c_ptr = (out_ptr + offset_cm[:, None] * cm_stride +
94
- offset_cn[None, :] * cn_stride)
95
- M = tl.load(seq_lens + cur_batch)
96
- c_mask = (offset_cm[:, None] < (cur_seq_start + M)) & (offset_cn[None, :] <
97
- (slice_offset + N))
98
- if ADD_INPUTS:
99
- # explicitly pass in other=None to tell triton that masked values
100
- # can be uninitialized. This is OK because the later tl.store operation
101
- # uses the same mask, eliminating the risk of garbage values propagating
102
- tiled_out = tl.load(c_ptr, mask=c_mask, other=None)
103
- tiled_c += tiled_out
104
- tl.store(c_ptr, tiled_c, mask=c_mask)
105
-
106
-
107
-@torch.inference_mode()
108
-def _sgmv_expand_slice(
109
- inputs: torch.Tensor,
110
- lora_b_weights: torch.Tensor,
111
- output_tensor: torch.Tensor,
112
- b_seq_start_loc: torch.Tensor,
113
- seq_len_tensor: torch.Tensor,
114
- lora_indices_tensor: torch.Tensor,
115
- batches: int,
116
- max_seq_length: int,
117
- token_nums: int,
118
- slice_offset: int,
119
- slice_size: int,
120
- add_inputs: bool = False,
121
-) -> None:
122
- """_summary_
123
-
124
- Args:
125
- inputs (torch.Tensor): input tensor
126
- lora_b_weights (torch.Tensor): lora'a weight
127
- output_tensor (torch.Tensor): output tensor
128
- b_seq_start_loc (torch.Tensor): (batch_size,). The cumulative
129
- sequence lengths of the sequences in the batch, used to index
130
- into sequence. E.g., if the sequence length is [4, 6], it is
131
- [0, 4, 10].
132
- seq_len_tensor (torch.Tensor): (batch_size,). Record the sequence
133
- length of the sequences in the batch
134
- lora_indices_tensor (torch.Tensor): (batch_size,). The LoRA index
135
- corresponding to each batch. An index of -1 means no lora should be
136
- applied.
137
- batches (int): batch size
138
- max_seq_length (int): The max sequence lengths of the sequences
139
- in the batch
140
- token_nums (int): The token numbers in the batch. Used to verify if the
141
- token numbers in the inputs matches the one in the metadata.
142
- slice_offset (int): output_tensor's offset
143
- slice_size (int): current output_tensor's size
144
- add_inputs (bool, optional): Defaults to False, adds the final lora
145
- results to the output.
146
- """
147
-
148
- assert inputs.dtype in [torch.float16, torch.bfloat16, torch.float32]
149
- assert lora_b_weights.dtype in [
150
- torch.float16,
151
- torch.bfloat16,
152
- ]
153
- assert inputs.size(0) == token_nums
154
- assert inputs.size(1) == lora_b_weights.size(-1)
155
- assert b_seq_start_loc.size(0) == batches
156
- assert lora_indices_tensor.size(0) == batches
157
- assert slice_size == lora_b_weights.size(-2)
158
- assert inputs.is_contiguous()
159
- assert output_tensor.is_contiguous()
160
-
161
- if lora_b_weights.ndim == 4: # shape:(lora_num,1,size,rank)
162
- assert lora_b_weights.size(1) == 1
163
- lora_b_weights = lora_b_weights.squeeze(dim=1)
164
- else:
165
- assert lora_b_weights.ndim == 3 # shape:(lora_num,size,rank)
166
-
167
- assert lora_b_weights.is_contiguous()
168
-
169
- # TODO tuning this config
170
- N, K = lora_b_weights.shape[-2:] # K= rank,N=hidden_size
171
-
172
- BLOCK_M = 32
173
- BLOCK_N = 32
174
- BLOCK_K = 16
175
- EVEN_K = K % BLOCK_K == 0
176
- ADD_INPUTS = add_inputs
177
- CAST_TYPE = False
178
- if inputs.dtype == torch.float32 and lora_b_weights.dtype in [
179
- torch.float16,
180
- torch.bfloat16,
181
- ]:
182
- CAST_TYPE = True
183
- grid = (
184
- triton.cdiv(max_seq_length, BLOCK_M) * triton.cdiv(N, BLOCK_N),
185
- batches,
186
- )
187
- _sgmv_expand_slice_kernel[grid](
188
- inputs,
189
- lora_b_weights,
190
- output_tensor,
191
- N,
192
- K,
193
- b_seq_start_loc,
194
- seq_len_tensor,
195
- lora_indices_tensor,
196
- inputs.stride(0),
197
- inputs.stride(1),
198
- lora_b_weights.stride(0),
199
- lora_b_weights.stride(1),
200
- lora_b_weights.stride(2),
201
- output_tensor.stride(0),
202
- output_tensor.stride(1),
203
- slice_offset,
204
- BLOCK_M,
205
- BLOCK_N,
206
- BLOCK_K,
207
- EVEN_K,
208
- ADD_INPUTS,
209
- CAST_TYPE,
210
- )
211
- return
212
-
213
-
214
-def sgmv_expand_slice_fake(
215
- inputs: torch.Tensor,
216
- lora_b_weights: torch.Tensor,
217
- output_tensor: torch.Tensor,
218
- b_seq_start_loc: torch.Tensor,
219
- seq_len_tensor: torch.Tensor,
220
- lora_indices_tensor: torch.Tensor,
221
- batches: int,
222
- max_seq_length: int,
223
- token_nums: int,
224
- slice_offset: int,
225
- slice_size: int,
226
- add_inputs: bool = False,
227
-) -> None:
228
- return
229
-
230
-
231
-try:
232
- direct_register_custom_op(
233
- op_name="sgmv_expand_slice",
234
- op_func=_sgmv_expand_slice,
235
- mutates_args=["output_tensor"],
236
- fake_impl=sgmv_expand_slice_fake,
237
- )
238
- sgmv_expand_slice = torch.ops.vllm.sgmv_expand_slice
239
-
240
-except AttributeError:
241
- sgmv_expand_slice = _sgmv_expand_slice
vllm/lora/ops/sgmv_shrink.py
+74
-55
@@ -5,48 +5,60 @@ Punica: Multi-Tenant LoRA Serving.
5
https://arxiv.org/abs/2310.18547
6
"""
7
8
+from typing import List
9
+
10
import torch
11
import triton
12
import triton.language as tl
13
14
from vllm.utils import direct_register_custom_op
15
16
+from .utils import _get_lora_a_ptr
17
+
18
19
@triton.jit
20
def _sgmv_shrink_kernel(
17
- input_ptr,
18
- lora_ptr,
19
- out_ptr,
20
- N,
21
- K,
22
- b_seq_start_loc,
23
- seq_lens,
24
- lora_indices,
25
- scaling,
26
- xm_stride, # hidden_size
27
- xk_stride, # 1
28
- l0_stride, # hidden_size*max_rank
29
- lora_k_stride,
30
- lora_n_stride,
31
- cm_stride,
32
- cn_stride,
33
- BLOCK_M: tl.constexpr,
34
- BLOCK_N: tl.constexpr,
35
- BLOCK_K: tl.constexpr,
36
- EVEN_K: tl.constexpr,
37
- SPLIT_K: tl.constexpr,
38
-):
21
+ input_ptr,
22
+ lora_ptr, #1-3
23
+ out_ptr,
24
+ N,
25
+ K,
26
+ b_seq_start_loc,
27
+ seq_lens,
28
+ lora_indices,
29
+ scaling,
30
+ input_d0_stride,
31
+ input_d1_stride, # 1
32
+ lora_d0_stride,
33
+ lora_d1_stride,
34
+ lora_d2_stride, # 1
35
+ output_d0_stride,
36
+ output_d1_stride,
37
+ output_d2_stride, # 1
38
+ BLOCK_M: tl.constexpr,
39
+ BLOCK_N: tl.constexpr,
40
+ BLOCK_K: tl.constexpr,
41
+ EVEN_K: tl.constexpr,
42
+ SPLIT_K: tl.constexpr,
43
+ SLICE_NUM: tl.constexpr):
44
"""
45
The sgmv's shrink triton kernel is based on GroupGEMM+SPLIT-K.
46
The GEMM of Multi-LoRA can be considered as GroupGEMM. Additionally,
47
introducing SPLIT-K can improve performance
48
"""
49
pid = tl.program_id(axis=0)
45
- pid_sk = tl.program_id(axis=1)
50
+ pid_mix = tl.program_id(axis=1)
51
cur_batch = tl.program_id(axis=2)
52
cta_n_num = tl.cdiv(N, BLOCK_N)
53
pid_m = pid // cta_n_num
54
pid_n = pid % cta_n_num
55
+ if SLICE_NUM == 1:
56
+ slice_id: tl.constexpr = 0
57
+ pid_sk = tl.program_id(axis=1)
58
+ else:
59
+ pid_mix = tl.program_id(axis=1)
60
+ slice_id = pid_mix // SPLIT_K
61
+ pid_sk = pid_mix % SPLIT_K
62
63
M = tl.load(seq_lens + cur_batch)
64
if pid_m * BLOCK_M > M:
@@ -61,11 +73,22 @@ def _sgmv_shrink_kernel(
73
74
ram = tl.max_contiguous(tl.multiple_of(offset_m % M, BLOCK_M), BLOCK_M)
75
rbn = tl.max_contiguous(tl.multiple_of(offset_n % N, BLOCK_N), BLOCK_N)
76
+ # input ptr
77
+ a_ptr = (input_ptr + cur_seq_start * input_d0_stride +
78
+ ram[:, None] * input_d0_stride +
79
+ offset_k[None, :] * input_d1_stride)
80
+
81
+ if SLICE_NUM == 1:
82
+ # current lora ptr
83
+ cur_lora_ptr = lora_ptr
84
+ else:
85
+ # current lora ptr
86
+ cur_lora_ptr = tl.load(lora_ptr + slice_id).to(
87
+ tl.pointer_type(input_ptr.dtype.element_ty))
88
65
- a_ptr = (input_ptr + cur_seq_start * xm_stride + ram[:, None] * xm_stride +
66
- offset_k[None, :] * xk_stride)
67
- b_ptr = (lora_ptr + l0_stride * lora_index + rbn[None, :] * lora_k_stride +
68
- offset_k[:, None] * lora_n_stride)
89
+ b_ptr = (cur_lora_ptr + lora_d0_stride * lora_index +
90
+ rbn[None, :] * lora_d1_stride +
91
+ offset_k[:, None] * lora_d2_stride)
92
93
accumulator = tl.zeros((BLOCK_M, BLOCK_N), dtype=tl.float32)
94
for k in range(0, tl.cdiv(K, BLOCK_K * SPLIT_K)):
@@ -82,13 +105,15 @@ def _sgmv_shrink_kernel(
105
other=0.0)
106
accumulator += tl.dot(tiled_a, tiled_b)
107
85
- a_ptr += BLOCK_K * SPLIT_K * xk_stride
86
- b_ptr += BLOCK_K * SPLIT_K * lora_n_stride
108
+ a_ptr += BLOCK_K * SPLIT_K * input_d1_stride
109
+ b_ptr += BLOCK_K * SPLIT_K * lora_d2_stride
110
offset_cm = cur_seq_start + tl.arange(0, BLOCK_M) + pid_m * BLOCK_M
111
112
offset_cn = tl.arange(0, BLOCK_N) + pid_n * BLOCK_N
90
- c_ptr = (out_ptr + offset_cm[:, None] * cm_stride +
91
- offset_cn[None, :] * cn_stride)
113
+ cur_out_ptr = (out_ptr if SLICE_NUM == 1 else out_ptr +
114
+ slice_id * output_d0_stride)
115
+ c_ptr = cur_out_ptr + offset_cm[:, None] * output_d1_stride + offset_cn[
116
+ None, :] * output_d2_stride
117
c_mask = (offset_cm[:, None] <
118
(cur_seq_start + M)) & (offset_cn[None, :] < N)
119
accumulator *= scaling
@@ -102,7 +127,7 @@ def _sgmv_shrink_kernel(
127
@torch.inference_mode()
128
def _sgmv_shrink(
129
inputs: torch.Tensor,
105
- lora_a_weights: torch.Tensor,
130
+ lora_a_weights: List[torch.Tensor],
131
output_tensor: torch.Tensor,
132
b_seq_start_loc: torch.Tensor,
133
seq_len_tensor: torch.Tensor,
@@ -113,10 +138,9 @@ def _sgmv_shrink(
138
scaling: float,
139
) -> None:
140
"""
116
-
141
Args:
142
inputs (torch.Tensor): input tensor
119
- lora_a_weights (torch.Tensor): lora'a weight
143
+ lora_a_weights (List[torch.Tensor]): lora'a weight
144
output_tensor (torch.Tensor): output tensor
145
b_seq_start_loc (torch.Tensor): (batch_size,). The cumulative
146
sequence lengths of the sequences in the batch, used to index
@@ -134,27 +158,21 @@ def _sgmv_shrink(
158
token numbers in the inputs matches the one in the metadata.
159
scaling (float): Scaling factor.
160
"""
137
- assert inputs.dtype == lora_a_weights.dtype
161
+ assert inputs.dtype == lora_a_weights[0].dtype
162
assert inputs.dtype in [torch.float16, torch.bfloat16]
139
- assert lora_a_weights.dtype in [
140
- torch.float16,
141
- torch.bfloat16,
142
- ]
163
+ for weight in lora_a_weights:
164
+ assert weight.dtype in [torch.float16, torch.bfloat16]
165
+
166
assert inputs.size(0) == token_nums
144
- assert inputs.size(1) == lora_a_weights.size(-1)
167
+ assert inputs.size(1) == lora_a_weights[0].size(-1)
168
assert b_seq_start_loc.size(0) == batches
169
assert lora_indices_tensor.size(0) == batches
170
assert inputs.is_contiguous()
148
-
149
- if lora_a_weights.ndim == 4: # shape:(lora_num,1,rank, size)
150
- assert lora_a_weights.size(1) == 1
151
- lora_a_weights = lora_a_weights.squeeze(dim=1)
152
- else:
153
- assert lora_a_weights.ndim == 3 # shape:(lora_num,rank, size)
154
- assert lora_a_weights.is_contiguous()
171
assert output_tensor.is_contiguous()
172
+ (lora_ptr_tensor, lora_strides_d0, lora_strides_d1,
173
+ lora_strides_d2) = _get_lora_a_ptr(lora_a_weights, b_seq_start_loc.device)
174
# TODO tuning this config
157
- N, K = lora_a_weights.shape[-2:] # K=hidden_size,N=rank
175
+ N, K = lora_a_weights[0].shape[-2:] # K=hidden_size,N=rank
176
BLOCK_M = 32
177
BLOCK_N = 16
178
BLOCK_K = 32
@@ -162,13 +180,12 @@ def _sgmv_shrink(
180
EVEN_K = K % (BLOCK_K * SPLIT_K) == 0
181
grid = (
182
triton.cdiv(max_seq_length, BLOCK_M) * triton.cdiv(N, BLOCK_N),
165
- SPLIT_K,
183
+ SPLIT_K * len(lora_a_weights),
184
batches,
185
)
168
-
186
_sgmv_shrink_kernel[grid](
187
inputs,
171
- lora_a_weights,
188
+ lora_ptr_tensor,
189
output_tensor,
190
N,
191
K,
@@ -178,23 +195,25 @@ def _sgmv_shrink(
195
scaling,
196
inputs.stride(0),
197
inputs.stride(1),
181
- lora_a_weights.stride(0),
182
- lora_a_weights.stride(1),
183
- lora_a_weights.stride(2),
198
+ lora_strides_d0,
199
+ lora_strides_d1,
200
+ lora_strides_d2,
201
output_tensor.stride(0),
202
output_tensor.stride(1),
203
+ output_tensor.stride(2),
204
BLOCK_M,
205
BLOCK_N,
206
BLOCK_K,
207
EVEN_K,
208
SPLIT_K,
209
+ len(lora_a_weights),
210
)
211
return
212
213
214
def sgmv_shrink_fake(
215
inputs: torch.Tensor,
197
- lora_a_weights: torch.Tensor,
216
+ lora_a_weights: List[torch.Tensor],
217
output_tensor: torch.Tensor,
218
b_seq_start_loc: torch.Tensor,
219
seq_len_tensor: torch.Tensor,
vllm/lora/ops/utils.py
+120
-1
@@ -1,5 +1,7 @@
1
import functools
2
-from typing import Dict
2
+from typing import Dict, List, Tuple
3
+
4
+import torch
5
6
7
@functools.lru_cache
@@ -44,3 +46,120 @@ def get_lora_op_configs(op_type: str, batch: int,
46
if not config:
47
config = _get_default_config(op_type, batch, hidden_size)
48
return config
49
+
50
+
51
+_LORA_A_PTR_DICT: Dict[Tuple[int, ...], Tuple[torch.tensor, ...]] = {}
52
+_LORA_B_PTR_DICT: Dict[Tuple[int, ...], Tuple[torch.tensor, ...]] = {}
53
+
54
+
55
+def _get_lora_a_ptr(lora_a_weights: List[torch.Tensor], device: str):
56
+ """
57
+ `_LORA_A_PTR_DICT` collects the required information during `profile_run`,
58
+ After this, it remains constant and subsequent usage is through LUT.
59
+ Refer to:
60
+ https://github.com/triton-lang/triton/blob/release/3.1.x/python/tutorials/08-grouped-gemm.py
61
+ """
62
+ key = tuple(lora_weight.data_ptr() for lora_weight in lora_a_weights)
63
+
64
+ if values := _LORA_A_PTR_DICT.get(key):
65
+ return values
66
+
67
+ lora_strides_d0 = []
68
+ lora_strides_d1 = []
69
+ lora_strides_d2 = []
70
+ tensor_ptrs = []
71
+ for lora_a_weight in lora_a_weights:
72
+ if lora_a_weight.ndim == 4: # shape:(lora_num,1,size,rank)
73
+ assert lora_a_weight.size(1) == 1
74
+ lora_a_weight = lora_a_weight.squeeze(dim=1)
75
+ else:
76
+ assert lora_a_weight.ndim == 3 # shape:(lora_num,size,rank)
77
+ assert lora_a_weight.is_contiguous()
78
+ tensor_ptrs.append(lora_a_weight.data_ptr())
79
+ lora_strides_d0.append(lora_a_weight.stride(0))
80
+ lora_strides_d1.append(lora_a_weight.stride(1))
81
+ lora_strides_d2.append(lora_a_weight.stride(2))
82
+ if len(lora_a_weights) > 1:
83
+ lora_ptr_tensor = torch.tensor(tensor_ptrs, device=device)
84
+ else:
85
+ lora_ptr_tensor = lora_a_weights[0]
86
+
87
+ if (len(set(lora_strides_d0)) > 1 or len(set(lora_strides_d1)) > 1
88
+ or len(set(lora_strides_d2)) > 1):
89
+ raise ValueError("All LoRA weights must have the same stride.")
90
+
91
+ _LORA_A_PTR_DICT[key] = (
92
+ lora_ptr_tensor,
93
+ lora_strides_d0[0],
94
+ lora_strides_d1[0],
95
+ lora_strides_d2[0],
96
+ )
97
+ return _LORA_A_PTR_DICT.get(key)
98
+
99
+
100
+def _get_lora_b_ptr(lora_weights: List[torch.Tensor], offset_start: int,
101
+ device: str):
102
+ """
103
+ `_LORA_B_PTR_DICT` collects the required information during `profile_run`,
104
+ After this, it remains constant and subsequent usage is through LUT.
105
+ Refer to:
106
+ https://github.com/triton-lang/triton/blob/release/3.1.x/python/tutorials/08-grouped-gemm.py
107
+
108
+ """
109
+
110
+ key = tuple(lora_weight.data_ptr() for lora_weight in lora_weights)
111
+ if values := _LORA_B_PTR_DICT.get(key):
112
+ return values
113
+ slice_offset_lst = []
114
+ tensor_ptrs = []
115
+ lora_strides_d0 = []
116
+ lora_strides_d1 = []
117
+ lora_strides_d2 = []
118
+ hidden_sizes = []
119
+ slice_offset = offset_start
120
+ for lora_b_weight in lora_weights:
121
+ if lora_b_weight.ndim == 4: # shape:(lora_num,1,size,rank)
122
+ assert lora_b_weight.size(1) == 1
123
+ lora_b_weight = lora_b_weight.squeeze(dim=1)
124
+ else:
125
+ assert lora_b_weight.ndim == 3 # shape:(lora_num,size,rank)
126
+ assert lora_b_weight.is_contiguous()
127
+ tensor_ptrs.append(lora_b_weight.data_ptr())
128
+ lora_strides_d0.append(lora_b_weight.stride(0))
129
+ lora_strides_d1.append(lora_b_weight.stride(1))
130
+ lora_strides_d2.append(lora_b_weight.stride(2))
131
+ slice_offset_lst.append(slice_offset)
132
+ slice_offset += lora_b_weight.size(1)
133
+ hidden_sizes.append(lora_b_weight.size(1))
134
+
135
+ if len(lora_weights) > 1:
136
+ # note these are device tensors
137
+ lora_ptr_tensor = torch.tensor(tensor_ptrs, device=device)
138
+ slice_start_tensor = torch.tensor(slice_offset_lst, device=device)
139
+ else:
140
+ slice_start_tensor = slice_offset_lst[0]
141
+ lora_ptr_tensor = lora_b_weight[0]
142
+
143
+ # If each lora has the same stride, there's no need to use a
144
+ # tensor for storage.
145
+ if (len(set(lora_strides_d0)) == 1 and len(set(lora_strides_d1)) == 1 and
146
+ len(set(lora_strides_d2)) == 1) and len(set(hidden_sizes)) == 1:
147
+ lora_strides_d0_tensor = lora_strides_d0[0]
148
+ lora_strides_d1_tensor = lora_strides_d1[0]
149
+ lora_strides_d2_tensor = lora_strides_d2[0]
150
+ hidden_sizes_tensor = hidden_sizes[0]
151
+ same_stride = True
152
+
153
+ else:
154
+ lora_strides_d0_tensor = torch.tensor(lora_strides_d0, device=device)
155
+ lora_strides_d1_tensor = torch.tensor(lora_strides_d1, device=device)
156
+ lora_strides_d2_tensor = torch.tensor(lora_strides_d2, device=device)
157
+ hidden_sizes_tensor = torch.tensor(hidden_sizes, device=device)
158
+ same_stride = False
159
+ # MAX_N is the maximum hidden size among all the lora_b weights
160
+ MAX_N = max(hidden_sizes)
161
+ _LORA_B_PTR_DICT[key] = (slice_start_tensor, lora_ptr_tensor,
162
+ lora_strides_d0_tensor, lora_strides_d1_tensor,
163
+ lora_strides_d2_tensor, hidden_sizes_tensor,
164
+ same_stride, MAX_N)
165
+ return _LORA_B_PTR_DICT.get(key)
vllm/lora/punica_wrapper/punica_gpu.py
+55
-99
@@ -5,7 +5,7 @@ Punica: Multi-Tenant LoRA Serving.
5
https://arxiv.org/abs/2310.18547
6
"""
7
8
-from typing import Callable, Optional, Tuple, Union, final
8
+from typing import Optional, Tuple, Union, final
9
10
import torch
11
@@ -16,7 +16,6 @@ if HAS_TRITON:
16
from vllm.lora.ops.bgmv_expand_slice import bgmv_expand_slice
17
from vllm.lora.ops.bgmv_shrink import bgmv_shrink
18
from vllm.lora.ops.sgmv_expand import sgmv_expand
19
- from vllm.lora.ops.sgmv_expand_slice import sgmv_expand_slice
19
from vllm.lora.ops.sgmv_shrink import sgmv_shrink
20
21
from .punica_base import PunicaWrapperBase
@@ -35,11 +34,11 @@ class PunicaWrapperGPU(PunicaWrapperBase):
34
PunicaWrapperBase.__init__(self, max_num_batched_tokens, max_batches,
35
device)
36
38
- def _shrink_prefill(
37
+ def _apply_shrink_prefill(
38
self,
39
y: torch.Tensor,
40
x: torch.Tensor,
42
- w_t_all: torch.Tensor,
41
+ w_t_all: Tuple[torch.Tensor, ...],
42
scale: float,
43
):
44
#No LoRA request, so return directly
@@ -53,7 +52,7 @@ class PunicaWrapperGPU(PunicaWrapperBase):
52
scale,
53
)
54
56
- def _shrink_decode(
55
+ def _apply_shrink_decode(
56
self,
57
y: torch.Tensor,
58
x: torch.Tensor,
@@ -62,56 +61,28 @@ class PunicaWrapperGPU(PunicaWrapperBase):
61
):
62
bgmv_shrink(x, w_t_all, y, self.token_lora_indices, scale)
63
65
- def _expand_prefill(
64
+ def _apply_expand_prefill(
65
self,
66
y: torch.Tensor,
67
x: torch.Tensor,
68
w_t_all: torch.Tensor,
69
+ offset_start: int,
70
add_inputs: bool,
71
):
72
#No LoRA request, so return directly
73
if self.no_lora:
74
return
75
- sgmv_expand(
76
- x,
77
- w_t_all,
78
- y,
79
- *self.prefill_metadata,
80
- add_inputs,
81
- )
82
-
83
- def _expand_decode(
84
- self,
85
- y: torch.Tensor,
86
- x: torch.Tensor,
87
- w_t_all: torch.Tensor,
88
- add_inputs: bool,
89
- ):
90
- bgmv_expand(x, w_t_all, y, self.token_lora_indices, add_inputs)
75
92
- def _expand_slice_prefill(
93
- self,
94
- y: torch.Tensor,
95
- x: torch.Tensor,
96
- w_t_all: torch.Tensor,
97
- y_offset: Optional[int],
98
- y_slice_size: Optional[int],
99
- add_inputs: bool,
100
- ):
101
- #No LoRA request, so return directly
102
- if self.no_lora:
103
- return
104
- sgmv_expand_slice(
76
+ sgmv_expand(
77
x,
78
w_t_all,
79
y,
80
*self.prefill_metadata,
109
- y_offset,
110
- y_slice_size,
111
- add_inputs,
81
+ offset_start=offset_start,
82
+ add_inputs=add_inputs,
83
)
84
114
- def _expand_slice_decode(
85
+ def _apply_expand_decode(
86
self,
87
y: torch.Tensor,
88
x: torch.Tensor,
@@ -123,43 +94,6 @@ class PunicaWrapperGPU(PunicaWrapperBase):
94
bgmv_expand_slice(x, w_t_all, y, self.token_lora_indices, y_offset,
95
y_slice_size, add_inputs)
96
126
- def _apply_expand(
127
- self,
128
- y: torch.Tensor,
129
- x: torch.Tensor,
130
- w_t_all: torch.Tensor,
131
- y_offset: Optional[int],
132
- y_slice_size: Optional[int],
133
- add_inputs: bool = True,
134
- ):
135
- """
136
- Perform the ` y[:,y_offset:y_offset+y_slice_size]+=x@w_t_all`
137
- computation, which is suitable for the
138
- GEMM of lora'b.
139
- """
140
-
141
- expand_slice_fun: Callable = (self._expand_slice_prefill
142
- if self.is_prefill else
143
- self._expand_slice_decode)
144
- expand_slice_fun(y, x, w_t_all, y_offset, y_slice_size, add_inputs)
145
-
146
- def _apply_shrink(self, y: torch.Tensor, x: torch.Tensor,
147
- w_t_all: torch.Tensor, scale: float):
148
- """
149
- Perform the ` y+=x@w_t_all` computation, which is suitable for the
150
- GEMM of lora'a.
151
- When `is_prefill is` true, it indicates that it is currently the
152
- prefill stage, and the `_shrink_prefill` function should be called.
153
- Otherwise, it is the decode stage, and the _shrink_decode function
154
- should be called.
155
- """
156
- y_org = y
157
- y = y.view(-1, y.shape[-1])
158
- shrink_fun: Callable = (self._shrink_prefill
159
- if self.is_prefill else self._shrink_decode)
160
- shrink_fun(y, x, w_t_all, scale)
161
- y = y.view_as(y_org)
162
-
97
def add_shrink(self, y: Union[Tuple[torch.Tensor, ...], torch.Tensor],
98
x: torch.Tensor, lora_a_stacked: Tuple[torch.Tensor, ...],
99
scale: float, **kwargs):
@@ -182,10 +116,15 @@ class PunicaWrapperGPU(PunicaWrapperBase):
116
"""
117
118
x = x.view(-1, x.shape[-1])
185
- # TODO fuse these kernels
186
- for slice_idx in range(len(lora_a_stacked)):
187
- self._apply_shrink(y[slice_idx], x, lora_a_stacked[slice_idx],
188
- scale)
119
+
120
+ if self.is_prefill:
121
+ # NOTE fused kernel
122
+ self._apply_shrink_prefill(y, x, lora_a_stacked, scale)
123
+ else:
124
+ # TODO fuse these kernels
125
+ for slice_idx in range(len(lora_a_stacked)):
126
+ self._apply_shrink_decode(y[slice_idx], x,
127
+ lora_a_stacked[slice_idx], scale)
128
129
def add_expand(self,
130
y: torch.Tensor,
@@ -217,20 +156,28 @@ class PunicaWrapperGPU(PunicaWrapperBase):
156
"""
157
y_org = y
158
y = y.view(-1, y.shape[-1])
220
- offset_left = offset_start
159
if lora_bias_stacked is not None:
160
self._apply_bias(self.token_lora_indices, y, output_slices,
161
lora_bias_stacked)
224
- for slice_idx in range(len(lora_b_stacked)):
225
- self._apply_expand(
226
- y,
227
- x[slice_idx],
228
- lora_b_stacked[slice_idx],
229
- offset_left,
230
- output_slices[slice_idx],
231
- add_inputs=add_inputs,
232
- )
233
- offset_left += output_slices[slice_idx]
162
+ if self.is_prefill:
163
+ # NOTE fused kernel
164
+ self._apply_expand_prefill(y,
165
+ x,
166
+ lora_b_stacked,
167
+ offset_start,
168
+ add_inputs=True)
169
+ else:
170
+ # TODO fuse these kernels
171
+ for slice_idx in range(len(lora_b_stacked)):
172
+ self._apply_expand_decode(
173
+ y,
174
+ x[slice_idx],
175
+ lora_b_stacked[slice_idx],
176
+ offset_start,
177
+ output_slices[slice_idx],
178
+ add_inputs=add_inputs,
179
+ )
180
+ offset_start += output_slices[slice_idx]
181
y = y.view_as(y_org)
182
183
def add_lora_embedding(self,
@@ -252,10 +199,18 @@ class PunicaWrapperGPU(PunicaWrapperBase):
199
add_inputs (bool): Default to True.
200
"""
201
255
- # Embedding layer only need expand op
256
- expand_fun: Callable = (self._expand_prefill
257
- if self.is_prefill else self._expand_decode)
258
- expand_fun(y, x, lora_b_stacked, add_inputs)
202
+ if self.is_prefill:
203
+ sgmv_expand(
204
+ x.unsqueeze(dim=0),
205
+ [lora_b_stacked],
206
+ y,
207
+ *self.prefill_metadata,
208
+ offset_start=0,
209
+ add_inputs=add_inputs,
210
+ )
211
+ else:
212
+ bgmv_expand(x, lora_b_stacked, y, self.token_lora_indices,
213
+ add_inputs)
214
215
def add_lora_linear(self,
216
y: torch.Tensor,
@@ -301,10 +256,11 @@ class PunicaWrapperGPU(PunicaWrapperBase):
256
r = lora_b_stacked[0].size(-1)
257
# We set the buffer to be float32 by default ,refer to:
258
# https://github.com/triton-lang/triton/issues/1387
304
- buffer = tuple(
305
- torch.zeros(
306
- (x.size(0), r), dtype=torch.float32, device=x.device)
307
- for _ in range(len(output_slices)))
259
+ buffer = torch.zeros(
260
+ (len(output_slices), x.size(0), r),
261
+ dtype=torch.float32,
262
+ device=x.device,
263
+ )
264
self.add_shrink(buffer, x, lora_a_stacked, scale, **kwargs)
265
self.add_expand(y,
266
buffer,