@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / b278557935
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 03:15 UTC

[Kernel][LoRA]Punica prefill kernels fusion (#11234)

Signed-off-by: Jee Jee Li <pandaleefree@gmail.com> Signed-off-by: Abatom <abzhonghua@gmail.com> Co-authored-by: Zhonghua Deng <abatom@163.com>

Jee Jee Li committed Jan 7, 2025 at 12:01 UTC b278557935d78b337fb5e82a32b02f75678b4101
11 files changed +707 -764
.buildkite/test-pipeline.yaml
+2 -1
@@ -242,7 +242,7 @@ steps:
242 source_file_dependencies:
243 - vllm/lora
244 - tests/lora
245 - command: pytest -v -s lora --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --ignore=lora/test_long_context.py --ignore=lora/test_chatglm3_tp.py --ignore=lora/test_llama_tp.py
245 + command: pytest -v -s lora --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --ignore=lora/test_long_context.py --ignore=lora/test_chatglm3_tp.py --ignore=lora/test_llama_tp.py --ignore=lora/test_minicpmv_tp.py
246 parallelism: 4
247
248 - label: "PyTorch Fullgraph Smoke Test" # 9min
@@ -535,6 +535,7 @@ steps:
535 # requires multi-GPU testing for validation.
536 - pytest -v -s -x lora/test_chatglm3_tp.py
537 - pytest -v -s -x lora/test_llama_tp.py
538 + - pytest -v -s -x lora/test_minicpmv_tp.py
539
540
541 - label: Weight Loading Multiple GPU Test # 33min
tests/lora/test_minicpmv.py deleted
-77
@@ -1,77 +0,0 @@
1 -from typing import List
2 -
3 -import pytest
4 -
5 -import vllm
6 -from vllm.assets.image import ImageAsset
7 -from vllm.lora.request import LoRARequest
8 -from vllm.platforms import current_platform
9 -
10 -MODEL_PATH = "openbmb/MiniCPM-Llama3-V-2_5"
11 -
12 -PROMPT_TEMPLATE = (
13 - "<|begin_of_text|><|start_header_id|>user<|end_header_id|>\n\n"
14 - "(<image>./</image>)\nWhat is in the image?<|eot_id|>"
15 - "<|start_header_id|>assistant<|end_header_id|>\n\n")
16 -
17 -IMAGE_ASSETS = [
18 - ImageAsset("stop_sign"),
19 - ImageAsset("cherry_blossom"),
20 -]
21 -
22 -# After fine-tuning with LoRA, all generated content should start begin `A`.
23 -EXPECTED_OUTPUT = [
24 - "A red and white stop sign with a Chinese archway in the background featuring red lanterns and gold accents.", # noqa: E501
25 - "A pink cherry blossom tree with a blue sky in the background.",
26 -]
27 -
28 -
29 -def do_sample(llm: vllm.LLM, lora_path: str, lora_id: int) -> List[str]:
30 - sampling_params = vllm.SamplingParams(
31 - temperature=0,
32 - max_tokens=5,
33 - stop_token_ids=[128001, 128009], # eos_id, eot_id
34 - )
35 -
36 - inputs = [{
37 - "prompt": PROMPT_TEMPLATE,
38 - "multi_modal_data": {
39 - "image": asset.pil_image
40 - },
41 - } for asset in IMAGE_ASSETS]
42 -
43 - outputs = llm.generate(
44 - inputs,
45 - sampling_params,
46 - lora_request=LoRARequest(str(lora_id), lora_id, lora_path)
47 - if lora_id else None,
48 - )
49 - # Print the outputs.
50 - generated_texts: List[str] = []
51 - for output in outputs:
52 - prompt = output.prompt
53 - generated_text = output.outputs[0].text.strip()
54 - generated_texts.append(generated_text)
55 - print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
56 - return generated_texts
57 -
58 -
59 -@pytest.mark.xfail(
60 - current_platform.is_rocm(),
61 - reason="MiniCPM-V dependency xformers incompatible with ROCm")
62 -def test_minicpmv_lora(minicpmv_lora_files):
63 - llm = vllm.LLM(
64 - MODEL_PATH,
65 - max_num_seqs=2,
66 - enable_lora=True,
67 - max_loras=4,
68 - max_lora_rank=64,
69 - trust_remote_code=True,
70 - enable_chunked_prefill=True,
71 - )
72 - output1 = do_sample(llm, minicpmv_lora_files, lora_id=1)
73 - for i in range(len(EXPECTED_OUTPUT)):
74 - assert EXPECTED_OUTPUT[i].startswith(output1[i])
75 - output2 = do_sample(llm, minicpmv_lora_files, lora_id=2)
76 - for i in range(len(EXPECTED_OUTPUT)):
77 - assert EXPECTED_OUTPUT[i].startswith(output2[i])
tests/lora/test_minicpmv_tp.py
+44 -19
@@ -3,10 +3,10 @@ from typing import List
3 import pytest
4
5 import vllm
6 +from tests.utils import fork_new_process_for_each_test
7 from vllm.assets.image import ImageAsset
8 from vllm.lora.request import LoRARequest
8 -
9 -from ..utils import multi_gpu_test
9 +from vllm.platforms import current_platform
10
11 MODEL_PATH = "openbmb/MiniCPM-Llama3-V-2_5"
12
@@ -17,13 +17,11 @@ PROMPT_TEMPLATE = (
17
18 IMAGE_ASSETS = [
19 ImageAsset("stop_sign"),
20 - ImageAsset("cherry_blossom"),
20 ]
21
22 # After fine-tuning with LoRA, all generated content should start begin `A`.
23 EXPECTED_OUTPUT = [
24 "A red and white stop sign with a Chinese archway in the background featuring red lanterns and gold accents.", # noqa: E501
26 - "A pink cherry blossom tree with a blue sky in the background.",
25 ]
26
27
@@ -50,48 +48,75 @@ def do_sample(llm: vllm.LLM, lora_path: str, lora_id: int) -> List[str]:
48 # Print the outputs.
49 generated_texts: List[str] = []
50 for output in outputs:
53 - prompt = output.prompt
51 generated_text = output.outputs[0].text.strip()
52 generated_texts.append(generated_text)
56 - print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
53 + print(f"Generated text: {generated_text!r}")
54 return generated_texts
55
56
60 -@multi_gpu_test(num_gpus=2)
61 -@pytest.mark.parametrize("fully_sharded", [True, False])
62 -def test_minicpmv_tp2(minicpmv_lora_files, fully_sharded):
57 +@pytest.mark.xfail(
58 + current_platform.is_rocm(),
59 + reason="MiniCPM-V dependency xformers incompatible with ROCm")
60 +@fork_new_process_for_each_test
61 +def test_minicpmv_lora(minicpmv_lora_files):
62 + llm = vllm.LLM(
63 + MODEL_PATH,
64 + max_num_seqs=2,
65 + enable_lora=True,
66 + max_loras=2,
67 + max_lora_rank=8,
68 + enforce_eager=True,
69 + trust_remote_code=True,
70 + enable_chunked_prefill=True,
71 + )
72 + output1 = do_sample(llm, minicpmv_lora_files, lora_id=1)
73 + for i in range(len(EXPECTED_OUTPUT)):
74 + assert EXPECTED_OUTPUT[i].startswith(output1[i])
75 + output2 = do_sample(llm, minicpmv_lora_files, lora_id=2)
76 + for i in range(len(EXPECTED_OUTPUT)):
77 + assert EXPECTED_OUTPUT[i].startswith(output2[i])
78 +
79 +
80 +@pytest.mark.xfail(
81 + current_platform.is_rocm(),
82 + reason="MiniCPM-V dependency xformers incompatible with ROCm")
83 +@fork_new_process_for_each_test
84 +def test_minicpmv_tp4_wo_fully_sharded_loras(minicpmv_lora_files):
85 llm = vllm.LLM(
86 MODEL_PATH,
87 enable_lora=True,
88 max_num_seqs=2,
89 max_loras=4,
90 max_lora_rank=64,
69 - tensor_parallel_size=2,
91 + tensor_parallel_size=4,
92 trust_remote_code=True,
71 - fully_sharded_loras=fully_sharded,
93 + enforce_eager=True,
94 enable_chunked_prefill=True,
95 )
74 -
96 output_tp = do_sample(llm, minicpmv_lora_files, lora_id=1)
76 -
97 for i in range(len(EXPECTED_OUTPUT)):
98 assert EXPECTED_OUTPUT[i].startswith(output_tp[i])
99
100
81 -@multi_gpu_test(num_gpus=4)
82 -@pytest.mark.parametrize("fully_sharded", [True, False])
83 -def test_minicpmv_tp4(minicpmv_lora_files, fully_sharded):
101 +@pytest.mark.xfail(
102 + current_platform.is_rocm(),
103 + reason="MiniCPM-V dependency xformers incompatible with ROCm")
104 +@fork_new_process_for_each_test
105 +def test_minicpmv_tp4_fully_sharded_loras(minicpmv_lora_files):
106 llm = vllm.LLM(
107 MODEL_PATH,
108 enable_lora=True,
109 max_num_seqs=2,
88 - max_loras=4,
89 - max_lora_rank=64,
110 + max_loras=2,
111 + max_lora_rank=8,
112 tensor_parallel_size=4,
113 trust_remote_code=True,
92 - fully_sharded_loras=fully_sharded,
114 + fully_sharded_loras=True,
115 enable_chunked_prefill=True,
116 )
117 output_tp = do_sample(llm, minicpmv_lora_files, lora_id=1)
118 for i in range(len(EXPECTED_OUTPUT)):
119 assert EXPECTED_OUTPUT[i].startswith(output_tp[i])
120 + output_tp = do_sample(llm, minicpmv_lora_files, lora_id=2)
121 + for i in range(len(EXPECTED_OUTPUT)):
122 + assert EXPECTED_OUTPUT[i].startswith(output_tp[i])
tests/lora/test_punica_sizes.py
+80 -86
@@ -4,6 +4,8 @@ hidden_sizes included in the LoRA models currently supported by vLLM. It tests
4 whether the corresponding Triton kernel can run normally when tensor parallelism
5 is set to [1, 2, 4, 8, 16, 32, 64].
6 """
7 +from threading import Lock
8 +
9 import pytest
10 import torch
11
@@ -11,12 +13,13 @@ from vllm.lora.ops.bgmv_expand import bgmv_expand
13 from vllm.lora.ops.bgmv_expand_slice import bgmv_expand_slice
14 from vllm.lora.ops.bgmv_shrink import bgmv_shrink
15 from vllm.lora.ops.sgmv_expand import sgmv_expand
14 -from vllm.lora.ops.sgmv_expand_slice import sgmv_expand_slice
16 from vllm.lora.ops.sgmv_shrink import sgmv_shrink
17 +from vllm.lora.ops.utils import _LORA_A_PTR_DICT, _LORA_B_PTR_DICT
18 from vllm.platforms import current_platform
19
18 -from .utils import (generate_data, generate_data_for_expand_nslices,
19 - ref_torch_groupgemm)
20 +from .utils import (assert_close, generate_data,
21 + generate_data_for_expand_nslices,
22 + generate_data_for_nslices, ref_torch_groupgemm)
23
24 HIDDEN_SIZES = [
25 128,
@@ -112,14 +115,7 @@ SCALES = [0.5]
115 SEED = [0]
116 CUDA_DEVICES = [f"cuda:{0}"]
117
115 -
116 -def assert_close(a, b):
117 - rtol, atol = {
118 - torch.float16: (6e-2, 6e-2),
119 - torch.bfloat16: (6e-2, 6e-2),
120 - torch.float32: (1e-2, 1e-2),
121 - }[a.dtype]
122 - torch.testing.assert_close(a, b, rtol=rtol, atol=atol)
118 +_dict_lock = Lock()
119
120
121 @pytest.mark.parametrize("batches", BATCHES)
@@ -127,6 +123,7 @@ def assert_close(a, b):
123 @pytest.mark.parametrize("rank", MAX_RANKS)
124 @pytest.mark.parametrize("hidden_size", HIDDEN_SIZES)
125 @pytest.mark.parametrize("scaling", SCALES)
126 +@pytest.mark.parametrize("nslices", [1, 2, 3])
127 @pytest.mark.parametrize("dtype", DTYPES)
128 @pytest.mark.parametrize("op_type", ["shrink", "expand"])
129 @pytest.mark.parametrize("seed", SEED)
@@ -137,6 +134,7 @@ def test_punica_sgmv(
134 rank: int,
135 hidden_size: int,
136 scaling: float,
137 + nslices: int,
138 dtype: torch.dtype,
139 op_type: str,
140 seed: int,
@@ -148,19 +146,20 @@ def test_punica_sgmv(
146 seq_length = 128
147 (
148 inputs_tensor,
151 - lora_weights,
149 + lora_weights_lst,
150 our_out_tensor,
151 ref_out_tensor,
152 b_seq_start_loc,
153 lora_indices_tensor,
154 seq_len_tensor,
155 indices,
158 - ) = generate_data(
156 + ) = generate_data_for_nslices(
157 batches,
158 hidden_size,
159 num_loras,
160 rank,
161 seq_length,
162 + nslices,
163 dtype,
164 op_type,
165 device,
@@ -172,43 +171,64 @@ def test_punica_sgmv(
171 else:
172 max_seq_length = max_seq_length.item()
173 if op_type == "shrink":
175 - sgmv_shrink(
176 - inputs_tensor,
177 - lora_weights,
178 - our_out_tensor,
179 - b_seq_start_loc,
180 - seq_len_tensor,
181 - lora_indices_tensor,
182 - batches,
183 - max_seq_length,
184 - token_nums,
185 - scaling,
186 - )
174 + # Preventing cache error pointer.
175 + with _dict_lock:
176 + _LORA_A_PTR_DICT.clear()
177 + sgmv_shrink(
178 + inputs_tensor,
179 + lora_weights_lst,
180 + our_out_tensor,
181 + b_seq_start_loc,
182 + seq_len_tensor,
183 + lora_indices_tensor,
184 + batches,
185 + max_seq_length,
186 + token_nums,
187 + scaling,
188 + )
189 + for index in range(nslices):
190 + ref_torch_groupgemm(
191 + ref_out_tensor[index],
192 + inputs_tensor,
193 + lora_weights_lst[index],
194 + lora_indices_tensor,
195 + seq_len_tensor,
196 + batches,
197 + scaling,
198 + op_type,
199 + )
200 else:
188 - sgmv_expand(
189 - inputs_tensor,
190 - lora_weights,
191 - our_out_tensor,
192 - b_seq_start_loc,
193 - seq_len_tensor,
194 - lora_indices_tensor,
195 - batches,
196 - max_seq_length,
197 - token_nums,
198 - add_inputs=True,
199 - )
200 - ref_torch_groupgemm(
201 - ref_out_tensor,
202 - inputs_tensor,
203 - lora_weights,
204 - lora_indices_tensor,
205 - seq_len_tensor,
206 - batches,
207 - scaling if op_type == "shrink" else 1.0,
208 - op_type,
209 - )
210 - if op_type == "shrink":
211 - ref_out_tensor = ref_out_tensor.to(torch.float32)
201 + with _dict_lock:
202 + _LORA_B_PTR_DICT.clear()
203 + sgmv_expand(
204 + inputs_tensor,
205 + lora_weights_lst,
206 + our_out_tensor,
207 + b_seq_start_loc,
208 + seq_len_tensor,
209 + lora_indices_tensor,
210 + batches,
211 + max_seq_length,
212 + token_nums,
213 + offset_start=0,
214 + add_inputs=True,
215 + )
216 +
217 + slice_offset = 0
218 + for index in range(nslices):
219 + lora_weights = lora_weights_lst[index]
220 + ref_torch_groupgemm(
221 + ref_out_tensor[:, slice_offset:slice_offset + hidden_size],
222 + inputs_tensor[index],
223 + lora_weights,
224 + lora_indices_tensor,
225 + seq_len_tensor,
226 + batches,
227 + 1.0,
228 + op_type,
229 + )
230 + slice_offset += hidden_size
231 +
232 assert_close(our_out_tensor, ref_out_tensor)
233
234
@@ -292,25 +312,22 @@ def test_punica_bgmv(
312 @pytest.mark.parametrize("hidden_size", HIDDEN_SIZES)
313 @pytest.mark.parametrize("nslices", [2, 3])
314 @pytest.mark.parametrize("dtype", DTYPES)
295 -@pytest.mark.parametrize("op_type", ["sgmv", "bgmv"])
315 @pytest.mark.parametrize("seed", SEED)
316 @pytest.mark.parametrize("device", CUDA_DEVICES)
298 -def test_punica_expand_nslices(
317 +def test_punica_bgmv_expand_nslices(
318 batches: int,
319 num_loras: int,
320 rank: int,
321 hidden_size: int,
322 nslices: int,
323 dtype: torch.dtype,
305 - op_type: str,
324 seed: int,
325 device: str,
326 ):
309 -
327 torch.set_default_device(device)
328 current_platform.seed_everything(seed)
329
313 - seq_length = 128 if op_type == "sgmv" else 1
330 + seq_length = 1
331 (
332 inputs_tensor,
333 lora_weights_lst,
@@ -330,41 +347,18 @@ def test_punica_expand_nslices(
347 nslices,
348 device,
349 )
333 - max_seq_length = seq_len_tensor.max()
334 - token_nums = seq_len_tensor.sum().item()
335 - if isinstance(max_seq_length, tuple):
336 - max_seq_length = max_seq_length[0].item()
337 - else:
338 - max_seq_length = max_seq_length.item()
350 slice_offset = 0
351 for index in range(nslices):
352 lora_weights = lora_weights_lst[index]
342 - if op_type == "sgmv":
343 - sgmv_expand_slice(
344 - inputs_tensor,
345 - lora_weights,
346 - our_outputs,
347 - b_seq_start_loc,
348 - seq_len_tensor,
349 - lora_indices_tensor,
350 - batches,
351 - max_seq_length,
352 - token_nums,
353 - slice_offset,
354 - hidden_size,
355 - add_inputs=True,
356 - )
357 - else:
358 -
359 - bgmv_expand_slice(
360 - inputs_tensor,
361 - lora_weights,
362 - our_outputs,
363 - indices,
364 - slice_offset,
365 - slice_size=hidden_size,
366 - add_inputs=True,
367 - )
353 + bgmv_expand_slice(
354 + inputs_tensor,
355 + lora_weights,
356 + our_outputs,
357 + indices,
358 + slice_offset,
359 + slice_size=hidden_size,
360 + add_inputs=True,
361 + )
362 ref_torch_groupgemm(
363 ref_outputs[:, slice_offset:slice_offset + hidden_size],
364 inputs_tensor,
tests/lora/test_punica_variation.py
+81 -87
@@ -3,6 +3,8 @@ This script is mainly used to test whether trtion kernels can run normally
3 under different conditions, including various batches, numbers of LoRA , and
4 maximum ranks.
5 """
6 +from threading import Lock
7 +
8 import pytest
9 import torch
10
@@ -11,12 +13,13 @@ import vllm.lora.ops.bgmv_expand
13 import vllm.lora.ops.bgmv_expand_slice
14 import vllm.lora.ops.bgmv_shrink
15 import vllm.lora.ops.sgmv_expand
14 -import vllm.lora.ops.sgmv_expand_slice
16 import vllm.lora.ops.sgmv_shrink # noqa: F401
17 +from vllm.lora.ops.utils import _LORA_A_PTR_DICT, _LORA_B_PTR_DICT
18 from vllm.platforms import current_platform
19
18 -from .utils import (generate_data, generate_data_for_expand_nslices,
19 - ref_torch_groupgemm)
20 +from .utils import (assert_close, generate_data,
21 + generate_data_for_expand_nslices,
22 + generate_data_for_nslices, ref_torch_groupgemm)
23
24 HIDDEN_SIZES = [4097]
25
@@ -28,31 +31,23 @@ SCALES = [0.5]
31 SEED = [0]
32 CUDA_DEVICES = [f"cuda:{0}"]
33
31 -
32 -def assert_close(a, b):
33 - rtol, atol = {
34 - torch.float16: (6e-2, 6e-2),
35 - torch.bfloat16: (6e-2, 6e-2),
36 - torch.float32: (1e-2, 1e-2),
37 - }[a.dtype]
38 - torch.testing.assert_close(a, b, rtol=rtol, atol=atol)
39 -
40 -
34 # Unlike test_punica_sizes.py, we directly utilize custom op for
35 # testing, which verifies the correct registration of these ops.
36 bgmv_expand = torch.ops.vllm.bgmv_expand
37 bgmv_expand_slice = torch.ops.vllm.bgmv_expand_slice
38 bgmv_shrink = torch.ops.vllm.bgmv_shrink
39 sgmv_expand = torch.ops.vllm.sgmv_expand
47 -sgmv_expand_slice = torch.ops.vllm.sgmv_expand_slice
40 sgmv_shrink = torch.ops.vllm.sgmv_shrink
41
42 +_dict_lock = Lock()
43 +
44
45 @pytest.mark.parametrize("batches", BATCHES)
46 @pytest.mark.parametrize("num_loras", NUM_LORA)
47 @pytest.mark.parametrize("rank", MAX_RANKS)
48 @pytest.mark.parametrize("hidden_size", HIDDEN_SIZES)
49 @pytest.mark.parametrize("scaling", SCALES)
50 +@pytest.mark.parametrize("nslices", [1, 2, 3])
51 @pytest.mark.parametrize("dtype", DTYPES)
52 @pytest.mark.parametrize("op_type", ["shrink", "expand"])
53 @pytest.mark.parametrize("seed", SEED)
@@ -63,6 +58,7 @@ def test_punica_sgmv(
58 rank: int,
59 hidden_size: int,
60 scaling: float,
61 + nslices: int,
62 dtype: torch.dtype,
63 op_type: str,
64 seed: int,
@@ -74,19 +70,20 @@ def test_punica_sgmv(
70 seq_length = 128
71 (
72 inputs_tensor,
77 - lora_weights,
73 + lora_weights_lst,
74 our_out_tensor,
75 ref_out_tensor,
76 b_seq_start_loc,
77 lora_indices_tensor,
78 seq_len_tensor,
79 indices,
84 - ) = generate_data(
80 + ) = generate_data_for_nslices(
81 batches,
82 hidden_size,
83 num_loras,
84 rank,
85 seq_length,
86 + nslices,
87 dtype,
88 op_type,
89 device,
@@ -98,43 +95,64 @@ def test_punica_sgmv(
95 else:
96 max_seq_length = max_seq_length.item()
97 if op_type == "shrink":
101 - sgmv_shrink(
102 - inputs_tensor,
103 - lora_weights,
104 - our_out_tensor,
105 - b_seq_start_loc,
106 - seq_len_tensor,
107 - lora_indices_tensor,
108 - batches,
109 - max_seq_length,
110 - token_nums,
111 - scaling,
112 - )
98 + # Preventing cache error pointer.
99 + with _dict_lock:
100 + _LORA_A_PTR_DICT.clear()
101 + sgmv_shrink(
102 + inputs_tensor,
103 + lora_weights_lst,
104 + our_out_tensor,
105 + b_seq_start_loc,
106 + seq_len_tensor,
107 + lora_indices_tensor,
108 + batches,
109 + max_seq_length,
110 + token_nums,
111 + scaling,
112 + )
113 + for index in range(nslices):
114 + ref_torch_groupgemm(
115 + ref_out_tensor[index],
116 + inputs_tensor,
117 + lora_weights_lst[index],
118 + lora_indices_tensor,
119 + seq_len_tensor,
120 + batches,
121 + scaling,
122 + op_type,
123 + )
124 else:
114 - sgmv_expand(
115 - inputs_tensor,
116 - lora_weights,
117 - our_out_tensor,
118 - b_seq_start_loc,
119 - seq_len_tensor,
120 - lora_indices_tensor,
121 - batches,
122 - max_seq_length,
123 - token_nums,
124 - add_inputs=True,
125 - )
126 - ref_torch_groupgemm(
127 - ref_out_tensor,
128 - inputs_tensor,
129 - lora_weights,
130 - lora_indices_tensor,
131 - seq_len_tensor,
132 - batches,
133 - scaling if op_type == "shrink" else 1.0,
134 - op_type,
135 - )
136 - if op_type == "shrink":
137 - ref_out_tensor = ref_out_tensor.to(torch.float32)
125 + with _dict_lock:
126 + _LORA_B_PTR_DICT.clear()
127 + sgmv_expand(
128 + inputs_tensor,
129 + lora_weights_lst,
130 + our_out_tensor,
131 + b_seq_start_loc,
132 + seq_len_tensor,
133 + lora_indices_tensor,
134 + batches,
135 + max_seq_length,
136 + token_nums,
137 + offset_start=0,
138 + add_inputs=True,
139 + )
140 +
141 + slice_offset = 0
142 + for index in range(nslices):
143 + lora_weights = lora_weights_lst[index]
144 + ref_torch_groupgemm(
145 + ref_out_tensor[:, slice_offset:slice_offset + hidden_size],
146 + inputs_tensor[index],
147 + lora_weights,
148 + lora_indices_tensor,
149 + seq_len_tensor,
150 + batches,
151 + 1.0,
152 + op_type,
153 + )
154 + slice_offset += hidden_size
155 +
156 assert_close(our_out_tensor, ref_out_tensor)
157
158
@@ -220,24 +238,22 @@ def test_punica_bgmv(
238 @pytest.mark.parametrize("hidden_size", HIDDEN_SIZES)
239 @pytest.mark.parametrize("nslices", [2, 3])
240 @pytest.mark.parametrize("dtype", DTYPES)
223 -@pytest.mark.parametrize("op_type", ["sgmv", "bgmv"])
241 @pytest.mark.parametrize("seed", SEED)
242 @pytest.mark.parametrize("device", CUDA_DEVICES)
226 -def test_punica_expand_nslices(
243 +def test_punica_bgmv_expand_nslices(
244 batches: int,
245 num_loras: int,
246 rank: int,
247 hidden_size: int,
248 nslices: int,
249 dtype: torch.dtype,
233 - op_type: str,
250 seed: int,
251 device: str,
252 ):
253 torch.set_default_device(device)
254 current_platform.seed_everything(seed)
255
240 - seq_length = 128 if op_type == "sgmv" else 1
256 + seq_length = 1
257 (
258 inputs_tensor,
259 lora_weights_lst,
@@ -257,40 +273,18 @@ def test_punica_expand_nslices(
273 nslices,
274 device,
275 )
260 - max_seq_length = seq_len_tensor.max()
261 - token_nums = seq_len_tensor.sum().item()
262 - if isinstance(max_seq_length, tuple):
263 - max_seq_length = max_seq_length[0].item()
264 - else:
265 - max_seq_length = max_seq_length.item()
276 slice_offset = 0
277 for index in range(nslices):
278 lora_weights = lora_weights_lst[index]
269 - if op_type == "sgmv":
270 - sgmv_expand_slice(
271 - inputs_tensor,
272 - lora_weights,
273 - our_outputs,
274 - b_seq_start_loc,
275 - seq_len_tensor,
276 - lora_indices_tensor,
277 - batches,
278 - max_seq_length,
279 - token_nums,
280 - slice_offset,
281 - hidden_size,
282 - add_inputs=True,
283 - )
284 - else:
285 - bgmv_expand_slice(
286 - inputs_tensor,
287 - lora_weights,
288 - our_outputs,
289 - indices,
290 - slice_offset,
291 - slice_size=hidden_size,
292 - add_inputs=True,
293 - )
279 + bgmv_expand_slice(
280 + inputs_tensor,
281 + lora_weights,
282 + our_outputs,
283 + indices,
284 + slice_offset,
285 + slice_size=hidden_size,
286 + add_inputs=True,
287 + )
288 ref_torch_groupgemm(
289 ref_outputs[:, slice_offset:slice_offset + hidden_size],
290 inputs_tensor,
tests/lora/utils.py
+123 -21
@@ -18,11 +18,13 @@ class DummyLoRAManager:
18 def get_module_lora(self, module_name: str) -> LoRALayerWeights:
19 return self._loras[module_name]
20
21 - def init_random_lora(self,
22 - module_name: str,
23 - weight: torch.Tensor,
24 - rank: int = 8,
25 - generate_embeddings_tensor: int = 0):
21 + def init_random_lora(
22 + self,
23 + module_name: str,
24 + weight: torch.Tensor,
25 + rank: int = 8,
26 + generate_embeddings_tensor: int = 0,
27 + ):
28 lora = LoRALayerWeights(
29 module_name,
30 rank=rank,
@@ -35,21 +37,25 @@ class DummyLoRAManager:
37 device=self._device),
38 )
39 if generate_embeddings_tensor:
38 - lora.embeddings_tensor = torch.rand(5,
39 - generate_embeddings_tensor,
40 - dtype=weight.dtype,
41 - device=self._device)
40 + lora.embeddings_tensor = torch.rand(
41 + 5,
42 + generate_embeddings_tensor,
43 + dtype=weight.dtype,
44 + device=self._device,
45 + )
46 self.set_module_lora(module_name, lora)
47
48 return lora
49
46 - def init_lora(self,
47 - module_name: str,
48 - input_dim: int,
49 - output_dim: int,
50 - rank=8,
51 - noop=False,
52 - embeddings_tensor=None):
50 + def init_lora(
51 + self,
52 + module_name: str,
53 + input_dim: int,
54 + output_dim: int,
55 + rank=8,
56 + noop=False,
57 + embeddings_tensor=None,
58 + ):
59 lora = LoRALayerWeights(
60 module_name,
61 rank=rank,
@@ -125,8 +131,16 @@ def ref_torch_groupgemm(
131 return
132
133
128 -def generate_data(batches, hidden_size, lora_nums, max_rank, seq_length, dtype,
129 - op_type, device):
134 +def generate_data(
135 + batches,
136 + hidden_size,
137 + lora_nums,
138 + max_rank,
139 + seq_length,
140 + dtype,
141 + op_type,
142 + device,
143 +):
144 seq_len_tensor = torch.randint(seq_length, seq_length + 1,
145 (batches, )).to(device)
146 b_seq_start_loc = torch.cumsum(
@@ -187,8 +201,16 @@ def generate_data(batches, hidden_size, lora_nums, max_rank, seq_length, dtype,
201 )
202
203
190 -def generate_data_for_expand_nslices(batches, hidden_size, lora_nums, max_rank,
191 - seq_length, dtype, nslices, device):
204 +def generate_data_for_expand_nslices(
205 + batches,
206 + hidden_size,
207 + lora_nums,
208 + max_rank,
209 + seq_length,
210 + dtype,
211 + nslices,
212 + device,
213 +):
214 seq_len_tensor = torch.randint(seq_length, seq_length + 1,
215 (batches, )).to(device)
216 b_seq_start_loc = torch.cumsum(
@@ -221,7 +243,87 @@ def generate_data_for_expand_nslices(batches, hidden_size, lora_nums, max_rank,
243 for b_id in range(batches):
244 lora_index = lora_indices_tensor[b_id]
245 indices[current_offset:current_offset +
224 - seq_len_tensor[b_id]] = lora_index.item()
246 + seq_len_tensor[b_id]] = (lora_index.item())
247 + current_offset += seq_len_tensor[b_id].item()
248 +
249 + lora_indices_tensor = lora_indices_tensor.to(device)
250 + return (
251 + inputs_tensor,
252 + lora_weights_lst,
253 + our_out_tensor,
254 + ref_out_tensor,
255 + b_seq_start_loc,
256 + lora_indices_tensor,
257 + seq_len_tensor,
258 + indices,
259 + )
260 +
261 +
262 +def generate_data_for_nslices(
263 + batches,
264 + hidden_size,
265 + lora_nums,
266 + max_rank,
267 + seq_length,
268 + nslices,
269 + dtype,
270 + op_type,
271 + device,
272 +):
273 + seq_len_tensor = torch.randint(seq_length, seq_length + 1,
274 + (batches, )).to(device)
275 + b_seq_start_loc = torch.cumsum(
276 + torch.tensor([0] + seq_len_tensor[:-1].tolist(), dtype=torch.long),
277 + dim=0,
278 + ).to(device)
279 + total_tokens = seq_len_tensor.sum()
280 +
281 + lora_weights_lst = []
282 + if op_type == "shrink":
283 +
284 + inputs_tensor = torch.rand((total_tokens, hidden_size),
285 + dtype=dtype).to(device)
286 +
287 + for _ in range(nslices):
288 + if op_type == "shrink":
289 + lora_weights_lst.append(
290 + torch.rand(
291 + (lora_nums, max_rank, hidden_size), # col-major
292 + dtype=dtype,
293 + ).to(device))
294 + # NOTE shrink kernel using torch.float32 as output type
295 + # shrink op need atomic_add, so output is initinized by 0
296 + our_out_tensor = torch.zeros(
297 + (nslices, total_tokens, max_rank),
298 + dtype=torch.float32,
299 + ).to(device)
300 + else:
301 + inputs_tensor = torch.rand(
302 + (nslices, total_tokens, max_rank),
303 + dtype=dtype,
304 + ).to(device)
305 + for _ in range(nslices):
306 + lora_weights_lst.append(
307 + torch.rand(
308 + (lora_nums, hidden_size, max_rank), # col-major
309 + dtype=dtype,
310 + ).to(device))
311 + # expand op needs to complete y+=a@lora_b, so output is
312 + # initinized randomly
313 + our_out_tensor = torch.rand((total_tokens, hidden_size * nslices),
314 + dtype=dtype).to(device)
315 +
316 + # Ensure the same input.
317 + ref_out_tensor = our_out_tensor.clone()
318 + lora_indices_tensor = torch.randint(0,
319 + lora_nums - 1 if lora_nums > 1 else 1,
320 + (batches, ))
321 + indices = torch.zeros((total_tokens), dtype=torch.long).to(device)
322 + current_offset = 0
323 + for b_id in range(batches):
324 + lora_index = lora_indices_tensor[b_id]
325 + indices[current_offset:current_offset +
326 + seq_len_tensor[b_id]] = (lora_index.item())
327 current_offset += seq_len_tensor[b_id].item()
328
329 lora_indices_tensor = lora_indices_tensor.to(device)
vllm/lora/ops/sgmv_expand.py
+128 -77
@@ -1,66 +1,109 @@
1 """
2 Based on:
3 -Chen, L., Ye, Z., Wu, Y., Zhuo, D., Ceze, L., & Krishnamurthy, A. (2023).
4 -Punica: Multi-Tenant LoRA Serving.
3 +Chen, L., Ye, Z., Wu, Y., Zhuo, D., Ceze, L., & Krishnamurthy, A. (2023).
4 +Punica: Multi-Tenant LoRA Serving.
5 https://arxiv.org/abs/2310.18547
6 """
7
8 +from typing import List
9 +
10 import torch
11 import triton
12 import triton.language as tl
13
14 from vllm.utils import direct_register_custom_op
15
16 +from .utils import _get_lora_b_ptr
17 +
18
19 @triton.jit
20 def _sgmv_expand_kernel(
17 - input_ptr,
18 - lora_ptr,
19 - out_ptr,
20 - N,
21 - K,
22 - b_seq_start_loc,
23 - seq_lens,
24 - lora_indices,
25 - xm_stride,
26 - xk_stride, # 1
27 - l0_stride, # hidden_size*max_rank
28 - lora_k_stride,
29 - lora_n_stride,
30 - cm_stride,
31 - cn_stride,
32 - BLOCK_M: tl.constexpr,
33 - BLOCK_N: tl.constexpr,
34 - BLOCK_K: tl.constexpr,
35 - EVEN_K: tl.constexpr,
36 - ADD_INPUTS: tl.constexpr,
37 - CAST_TYPE: tl.constexpr,
38 -):
21 + input_ptr,
22 + lora_ptr,
23 + out_ptr,
24 + N,
25 + K,
26 + b_seq_start_loc,
27 + seq_lens,
28 + lora_indices,
29 + slice_start_loc,
30 + input_d0_stride,
31 + input_d1_stride,
32 + input_d2_stride, # 1
33 + ls_d0_ptr,
34 + ls_d1_ptr,
35 + ls_d2_ptr, # 1
36 + output_d0_stride,
37 + output_d1_stride, # 1
38 + output_hs_ptr,
39 + BLOCK_M: tl.constexpr,
40 + BLOCK_N: tl.constexpr,
41 + BLOCK_K: tl.constexpr,
42 + EVEN_K: tl.constexpr,
43 + ADD_INPUTS: tl.constexpr,
44 + CAST_TYPE: tl.constexpr,
45 + SLICE_NUM: tl.constexpr,
46 + SAME_STRIDE: tl.constexpr):
47 """
40 - The sgmv's expand triton kernel is based on GroupGEMM.
48 +
49 + Similar to the 'sgmv_expand' operator, but with an added parameter
50 + 'slice_offset'. The reason for not reusing the 'sgmv_expand' operator
51 + might be that in the future, we could implement a fusion operator to
52 + achieve the current functionality instead of having to call it multiple
53 + times.
54 """
55 pid = tl.program_id(axis=0)
56 cur_batch = tl.program_id(axis=1)
57 + slice_id = tl.program_id(axis=2)
58 cta_n_num = tl.cdiv(N, BLOCK_N)
59 + # When the output dimensions of each slice are the same,cur_n=N, otherwise
60 + # cur_n=tl.load(output_hs_ptr + slice_id), this situation exists in GQA's
61 + # qkv linear.
62 + curr_N = N if SAME_STRIDE else tl.load(output_hs_ptr + slice_id)
63 pid_m = pid // cta_n_num
64 pid_n = pid % cta_n_num
65 M = tl.load(seq_lens + cur_batch)
66 if pid_m * BLOCK_M > M:
67 return
68 + if pid_n * BLOCK_N > curr_N:
69 + return
70 lora_index = tl.load(lora_indices + cur_batch)
71 if lora_index == -1:
72 return
73 +
74 cur_seq_start = tl.load(b_seq_start_loc + cur_batch)
75 offset_m = tl.arange(0, BLOCK_M) + pid_m * BLOCK_M
76 offset_n = tl.arange(0, BLOCK_N) + pid_n * BLOCK_N
77 offset_k = tl.arange(0, BLOCK_K)
78 ram = tl.max_contiguous(tl.multiple_of(offset_m % M, BLOCK_M), BLOCK_M)
58 - rbn = tl.max_contiguous(tl.multiple_of(offset_n % N, BLOCK_N), BLOCK_N)
79 + rbn = tl.max_contiguous(tl.multiple_of(offset_n % curr_N, BLOCK_N),
80 + BLOCK_N)
81 + # ls_d*_ptr can be either an integer or a pointer
82 + if SAME_STRIDE:
83 + # integer
84 + cur_lora_d0_stride = ls_d0_ptr
85 + cur_lora_d1_stride = ls_d1_ptr
86 + cur_lora_d2_stride = ls_d2_ptr
87 + else:
88 + # pointer
89 + cur_lora_d0_stride = tl.load(ls_d0_ptr + slice_id)
90 + cur_lora_d1_stride = tl.load(ls_d1_ptr + slice_id)
91 + cur_lora_d2_stride = tl.load(ls_d2_ptr + slice_id)
92 + if SLICE_NUM == 1:
93 + cur_input_ptr = input_ptr
94 + cur_lora_ptr = lora_ptr
95
60 - a_ptr = (input_ptr + cur_seq_start * xm_stride + ram[:, None] * xm_stride +
61 - offset_k[None, :] * xk_stride, )
62 - b_ptr = (lora_ptr + l0_stride * lora_index +
63 - offset_k[:, None] * lora_n_stride + rbn[None, :] * lora_k_stride)
96 + else:
97 + cur_input_ptr = input_ptr + slice_id * input_d0_stride
98 + cur_lora_ptr = tl.load(lora_ptr + slice_id).to(
99 + tl.pointer_type(out_ptr.dtype.element_ty))
100 +
101 + a_ptr = (cur_input_ptr + cur_seq_start * input_d1_stride +
102 + ram[:, None] * input_d1_stride +
103 + offset_k[None, :] * input_d2_stride, )
104 + b_ptr = (cur_lora_ptr + cur_lora_d0_stride * lora_index +
105 + offset_k[:, None] * cur_lora_d2_stride +
106 + rbn[None, :] * cur_lora_d1_stride)
107 accumulator = tl.zeros((BLOCK_M, BLOCK_N), dtype=tl.float32)
108 for k in range(tl.cdiv(K, BLOCK_K)):
109 if EVEN_K:
@@ -74,26 +117,30 @@ def _sgmv_expand_kernel(
117 mask=offset_k[:, None] < K - k * BLOCK_K,
118 other=0)
119 if CAST_TYPE:
77 - tiled_a = tiled_a.to(lora_ptr.dtype.element_ty)
120 + tiled_a = tiled_a.to(cur_lora_ptr.dtype.element_ty)
121 accumulator += tl.dot(
122 tiled_a,
123 tiled_b,
124 )
82 - a_ptr += BLOCK_K * xk_stride
83 - b_ptr += BLOCK_K * lora_n_stride
84 - tiled_c = accumulator.to(lora_ptr.dtype.element_ty)
125 + a_ptr += BLOCK_K * input_d2_stride
126 + b_ptr += BLOCK_K * cur_lora_d2_stride
127 +
128 + tiled_c = accumulator.to(cur_lora_ptr.dtype.element_ty)
129 + if SLICE_NUM == 1:
130 + cur_slice_start = slice_start_loc
131 + else:
132 + cur_slice_start = tl.load(slice_start_loc + slice_id)
133 +
134 offset_cm = cur_seq_start + tl.arange(0, BLOCK_M) + pid_m * BLOCK_M
86 - offset_cn = tl.arange(0, BLOCK_N) + pid_n * BLOCK_N
87 - c_ptr = (out_ptr + offset_cm[:, None] * cm_stride +
88 - offset_cn[None, :] * cn_stride)
135 + offset_cn = tl.arange(0, BLOCK_N) + pid_n * BLOCK_N + cur_slice_start
136 + c_ptr = (out_ptr + offset_cm[:, None] * output_d0_stride +
137 + offset_cn[None, :] * output_d1_stride)
138 M = tl.load(seq_lens + cur_batch)
139 c_mask = (offset_cm[:, None] <
91 - (cur_seq_start + M)) & (offset_cn[None, :] < N)
140 + (cur_seq_start + M)) & (offset_cn[None, :] <
141 + (cur_slice_start + curr_N))
142 if ADD_INPUTS:
93 - # explicitly pass in other=None to tell triton that masked values
94 - # can be uninitialized. This is OK because the later tl.store operation
95 - # uses the same mask, eliminating the risk of garbage values propagating
96 - tiled_out = tl.load(c_ptr, mask=c_mask, other=None)
143 + tiled_out = tl.load(c_ptr, mask=c_mask)
144 tiled_c += tiled_out
145 tl.store(c_ptr, tiled_c, mask=c_mask)
146
@@ -101,7 +148,7 @@ def _sgmv_expand_kernel(
148 @torch.inference_mode()
149 def _sgmv_expand(
150 inputs: torch.Tensor,
104 - lora_b_weights: torch.Tensor,
151 + lora_b_weights: List[torch.Tensor],
152 output_tensor: torch.Tensor,
153 b_seq_start_loc: torch.Tensor,
154 seq_len_tensor: torch.Tensor,
@@ -109,17 +156,18 @@ def _sgmv_expand(
156 batches: int,
157 max_seq_length: int,
158 token_nums: int,
159 + offset_start: int = 0,
160 add_inputs: bool = False,
161 ) -> None:
162 """
163 Args:
164 inputs (torch.Tensor): input tensor
117 - lora_b_weights (torch.Tensor): lora'a weight
165 + lora_b_weights (List[torch.Tensor]): lora'b weight
166 output_tensor (torch.Tensor): output tensor
167 b_seq_start_loc (torch.Tensor): (batch_size,). The cumulative
168 sequence lengths of the sequences in the batch, used to index
169 into sequence. E.g., if the sequence length is [4, 6], it is
122 - [0, 4, 10].
170 + [0, 4].
171 seq_len_tensor (torch.Tensor): (batch_size,). Record the sequence
172 length of the sequences in the batch.
173 lora_indices_tensor (torch.Tensor): (batch_size,). The LoRA index
@@ -130,77 +178,80 @@ def _sgmv_expand(
178 batch.
179 token_nums (int): The token numbers in the batch. Used to verify if the
180 token numbers in the inputs matches the one in the metadata.
133 - add_inputs (bool, optional): Defaults to False, adds the final lora
134 - results to the output.
181 + offset_start (int, optional): Offset start for output_tensor.
182 + Defaults to 0.
183 + add_inputs (bool, optional): Whether to add the input tensor to the
184 + output tensor. Defaults to False.
185 """
136 -
186 assert inputs.dtype in [torch.float16, torch.bfloat16, torch.float32]
138 - assert lora_b_weights.dtype in [
139 - torch.float16,
140 - torch.bfloat16,
141 - ]
142 - assert inputs.size(0) == token_nums
143 - assert inputs.size(1) == lora_b_weights.size(-1)
187 + for weight in lora_b_weights:
188 + assert weight.dtype in [torch.float16, torch.bfloat16]
189 +
190 + assert inputs.size(1) == token_nums
191 + assert inputs.size(0) == len(lora_b_weights)
192 +
193 assert b_seq_start_loc.size(0) == batches
194 assert lora_indices_tensor.size(0) == batches
146 - assert inputs.is_contiguous()
195 assert output_tensor.is_contiguous()
148 -
149 - if lora_b_weights.ndim == 4: # shape:(lora_num,1,size,rank)
150 - assert lora_b_weights.size(1) == 1
151 - lora_b_weights = lora_b_weights.squeeze(dim=1)
152 - else:
153 - assert lora_b_weights.ndim == 3 # shape:(lora_num,size,rank)
154 -
155 - assert lora_b_weights.is_contiguous()
196 + (slice_start_tensor, lora_ptr_tensor, lora_strides_d0_tensor,
197 + lora_strides_d1_tensor, lora_strides_d2_tensor, hidden_sizes_tensor,
198 + same_stride, MAX_N) = _get_lora_b_ptr(lora_b_weights, offset_start,
199 + b_seq_start_loc.device)
200
201 # TODO tuning this config
202 + K = lora_b_weights[0].shape[-1] # K= rank
203
159 - N, K = lora_b_weights.shape[-2:] # K= rank,N=hidden_size
160 - BLOCK_M = 32
161 - BLOCK_N = 32
204 + BLOCK_M = 64
205 + BLOCK_N = 128
206 BLOCK_K = 16
207 EVEN_K = K % BLOCK_K == 0
208 ADD_INPUTS = add_inputs
209 CAST_TYPE = False
166 - if inputs.dtype == torch.float32 and lora_b_weights.dtype in [
210 +
211 + if inputs.dtype == torch.float32 and lora_b_weights[0].dtype in [
212 torch.float16,
213 torch.bfloat16,
214 ]:
215 CAST_TYPE = True
216 grid = (
172 - triton.cdiv(max_seq_length, BLOCK_M) * triton.cdiv(N, BLOCK_N),
217 + triton.cdiv(max_seq_length, BLOCK_M) * triton.cdiv(MAX_N, BLOCK_N),
218 batches,
219 + len(lora_b_weights),
220 )
221 _sgmv_expand_kernel[grid](
222 inputs,
177 - lora_b_weights,
223 + lora_ptr_tensor,
224 output_tensor,
179 - N,
225 + MAX_N,
226 K,
227 b_seq_start_loc,
228 seq_len_tensor,
229 lora_indices_tensor,
230 + slice_start_tensor,
231 inputs.stride(0),
232 inputs.stride(1),
186 - lora_b_weights.stride(0),
187 - lora_b_weights.stride(1),
188 - lora_b_weights.stride(2),
233 + inputs.stride(2),
234 + lora_strides_d0_tensor,
235 + lora_strides_d1_tensor,
236 + lora_strides_d2_tensor,
237 output_tensor.stride(0),
238 output_tensor.stride(1),
239 + hidden_sizes_tensor,
240 BLOCK_M,
241 BLOCK_N,
242 BLOCK_K,
243 EVEN_K,
244 ADD_INPUTS,
245 CAST_TYPE,
246 + len(lora_b_weights),
247 + same_stride,
248 )
249 return
250
251
201 -def sgmv_expand_fake(
252 +def _sgmv_expand_fake(
253 inputs: torch.Tensor,
203 - lora_b_weights: torch.Tensor,
254 + lora_b_weights: List[torch.Tensor],
255 output_tensor: torch.Tensor,
256 b_seq_start_loc: torch.Tensor,
257 seq_len_tensor: torch.Tensor,
@@ -208,18 +259,18 @@ def sgmv_expand_fake(
259 batches: int,
260 max_seq_length: int,
261 token_nums: int,
262 + offset_start: int = 0,
263 add_inputs: bool = False,
264 ) -> None:
265 return
266
267
268 try:
217 -
269 direct_register_custom_op(
270 op_name="sgmv_expand",
271 op_func=_sgmv_expand,
272 mutates_args=["output_tensor"],
222 - fake_impl=sgmv_expand_fake,
273 + fake_impl=_sgmv_expand_fake,
274 )
275 sgmv_expand = torch.ops.vllm.sgmv_expand
276
vllm/lora/ops/sgmv_expand_slice.py deleted
-241
@@ -1,241 +0,0 @@
1 -"""
2 -Based on:
3 -Chen, L., Ye, Z., Wu, Y., Zhuo, D., Ceze, L., & Krishnamurthy, A. (2023).
4 -Punica: Multi-Tenant LoRA Serving.
5 -https://arxiv.org/abs/2310.18547
6 -"""
7 -
8 -import torch
9 -import triton
10 -import triton.language as tl
11 -
12 -from vllm.utils import direct_register_custom_op
13 -
14 -
15 -@triton.jit
16 -def _sgmv_expand_slice_kernel(
17 - input_ptr,
18 - lora_ptr,
19 - out_ptr,
20 - N,
21 - K,
22 - b_seq_start_loc,
23 - seq_lens,
24 - lora_indices,
25 - xm_stride,
26 - xk_stride, # 1
27 - l0_stride, # hidden_size*max_rank
28 - lora_k_stride,
29 - lora_n_stride,
30 - cm_stride,
31 - cn_stride,
32 - slice_offset,
33 - BLOCK_M: tl.constexpr,
34 - BLOCK_N: tl.constexpr,
35 - BLOCK_K: tl.constexpr,
36 - EVEN_K: tl.constexpr,
37 - ADD_INPUTS: tl.constexpr,
38 - CAST_TYPE: tl.constexpr,
39 -):
40 - """
41 -
42 - Similar to the 'sgmv_expand' operator, but with an added parameter
43 - 'slice_offset'. The reason for not reusing the 'sgmv_expand' operator
44 - might be that in the future, we could implement a fusion operator to
45 - achieve the current functionality instead of having to call it multiple
46 - times.
47 - """
48 - pid = tl.program_id(axis=0)
49 - cur_batch = tl.program_id(axis=1)
50 - cta_n_num = tl.cdiv(N, BLOCK_N)
51 - pid_m = pid // cta_n_num
52 - pid_n = pid % cta_n_num
53 - M = tl.load(seq_lens + cur_batch)
54 - if pid_m * BLOCK_M > M:
55 - return
56 - lora_index = tl.load(lora_indices + cur_batch)
57 - if lora_index == -1:
58 - return
59 - cur_seq_start = tl.load(b_seq_start_loc + cur_batch)
60 - offset_m = tl.arange(0, BLOCK_M) + pid_m * BLOCK_M
61 - offset_n = tl.arange(0, BLOCK_N) + pid_n * BLOCK_N
62 - offset_k = tl.arange(0, BLOCK_K)
63 - ram = tl.max_contiguous(tl.multiple_of(offset_m % M, BLOCK_M), BLOCK_M)
64 - rbn = tl.max_contiguous(tl.multiple_of(offset_n % N, BLOCK_N), BLOCK_N)
65 -
66 - a_ptr = (input_ptr + cur_seq_start * xm_stride + ram[:, None] * xm_stride +
67 - offset_k[None, :] * xk_stride, )
68 - b_ptr = (lora_ptr + l0_stride * lora_index +
69 - offset_k[:, None] * lora_n_stride + rbn[None, :] * lora_k_stride)
70 - accumulator = tl.zeros((BLOCK_M, BLOCK_N), dtype=tl.float32)
71 - for k in range(tl.cdiv(K, BLOCK_K)):
72 - if EVEN_K:
73 - tiled_a = tl.load(a_ptr)
74 - tiled_b = tl.load(b_ptr)
75 - else:
76 - tiled_a = tl.load(a_ptr,
77 - mask=offset_k[None, :] < K - k * BLOCK_K,
78 - other=0)
79 - tiled_b = tl.load(b_ptr,
80 - mask=offset_k[:, None] < K - k * BLOCK_K,
81 - other=0)
82 - if CAST_TYPE:
83 - tiled_a = tiled_a.to(lora_ptr.dtype.element_ty)
84 - accumulator += tl.dot(
85 - tiled_a,
86 - tiled_b,
87 - )
88 - a_ptr += BLOCK_K * xk_stride
89 - b_ptr += BLOCK_K * lora_n_stride
90 - tiled_c = accumulator.to(lora_ptr.dtype.element_ty)
91 - offset_cm = cur_seq_start + tl.arange(0, BLOCK_M) + pid_m * BLOCK_M
92 - offset_cn = tl.arange(0, BLOCK_N) + pid_n * BLOCK_N + slice_offset
93 - c_ptr = (out_ptr + offset_cm[:, None] * cm_stride +
94 - offset_cn[None, :] * cn_stride)
95 - M = tl.load(seq_lens + cur_batch)
96 - c_mask = (offset_cm[:, None] < (cur_seq_start + M)) & (offset_cn[None, :] <
97 - (slice_offset + N))
98 - if ADD_INPUTS:
99 - # explicitly pass in other=None to tell triton that masked values
100 - # can be uninitialized. This is OK because the later tl.store operation
101 - # uses the same mask, eliminating the risk of garbage values propagating
102 - tiled_out = tl.load(c_ptr, mask=c_mask, other=None)
103 - tiled_c += tiled_out
104 - tl.store(c_ptr, tiled_c, mask=c_mask)
105 -
106 -
107 -@torch.inference_mode()
108 -def _sgmv_expand_slice(
109 - inputs: torch.Tensor,
110 - lora_b_weights: torch.Tensor,
111 - output_tensor: torch.Tensor,
112 - b_seq_start_loc: torch.Tensor,
113 - seq_len_tensor: torch.Tensor,
114 - lora_indices_tensor: torch.Tensor,
115 - batches: int,
116 - max_seq_length: int,
117 - token_nums: int,
118 - slice_offset: int,
119 - slice_size: int,
120 - add_inputs: bool = False,
121 -) -> None:
122 - """_summary_
123 -
124 - Args:
125 - inputs (torch.Tensor): input tensor
126 - lora_b_weights (torch.Tensor): lora'a weight
127 - output_tensor (torch.Tensor): output tensor
128 - b_seq_start_loc (torch.Tensor): (batch_size,). The cumulative
129 - sequence lengths of the sequences in the batch, used to index
130 - into sequence. E.g., if the sequence length is [4, 6], it is
131 - [0, 4, 10].
132 - seq_len_tensor (torch.Tensor): (batch_size,). Record the sequence
133 - length of the sequences in the batch
134 - lora_indices_tensor (torch.Tensor): (batch_size,). The LoRA index
135 - corresponding to each batch. An index of -1 means no lora should be
136 - applied.
137 - batches (int): batch size
138 - max_seq_length (int): The max sequence lengths of the sequences
139 - in the batch
140 - token_nums (int): The token numbers in the batch. Used to verify if the
141 - token numbers in the inputs matches the one in the metadata.
142 - slice_offset (int): output_tensor's offset
143 - slice_size (int): current output_tensor's size
144 - add_inputs (bool, optional): Defaults to False, adds the final lora
145 - results to the output.
146 - """
147 -
148 - assert inputs.dtype in [torch.float16, torch.bfloat16, torch.float32]
149 - assert lora_b_weights.dtype in [
150 - torch.float16,
151 - torch.bfloat16,
152 - ]
153 - assert inputs.size(0) == token_nums
154 - assert inputs.size(1) == lora_b_weights.size(-1)
155 - assert b_seq_start_loc.size(0) == batches
156 - assert lora_indices_tensor.size(0) == batches
157 - assert slice_size == lora_b_weights.size(-2)
158 - assert inputs.is_contiguous()
159 - assert output_tensor.is_contiguous()
160 -
161 - if lora_b_weights.ndim == 4: # shape:(lora_num,1,size,rank)
162 - assert lora_b_weights.size(1) == 1
163 - lora_b_weights = lora_b_weights.squeeze(dim=1)
164 - else:
165 - assert lora_b_weights.ndim == 3 # shape:(lora_num,size,rank)
166 -
167 - assert lora_b_weights.is_contiguous()
168 -
169 - # TODO tuning this config
170 - N, K = lora_b_weights.shape[-2:] # K= rank,N=hidden_size
171 -
172 - BLOCK_M = 32
173 - BLOCK_N = 32
174 - BLOCK_K = 16
175 - EVEN_K = K % BLOCK_K == 0
176 - ADD_INPUTS = add_inputs
177 - CAST_TYPE = False
178 - if inputs.dtype == torch.float32 and lora_b_weights.dtype in [
179 - torch.float16,
180 - torch.bfloat16,
181 - ]:
182 - CAST_TYPE = True
183 - grid = (
184 - triton.cdiv(max_seq_length, BLOCK_M) * triton.cdiv(N, BLOCK_N),
185 - batches,
186 - )
187 - _sgmv_expand_slice_kernel[grid](
188 - inputs,
189 - lora_b_weights,
190 - output_tensor,
191 - N,
192 - K,
193 - b_seq_start_loc,
194 - seq_len_tensor,
195 - lora_indices_tensor,
196 - inputs.stride(0),
197 - inputs.stride(1),
198 - lora_b_weights.stride(0),
199 - lora_b_weights.stride(1),
200 - lora_b_weights.stride(2),
201 - output_tensor.stride(0),
202 - output_tensor.stride(1),
203 - slice_offset,
204 - BLOCK_M,
205 - BLOCK_N,
206 - BLOCK_K,
207 - EVEN_K,
208 - ADD_INPUTS,
209 - CAST_TYPE,
210 - )
211 - return
212 -
213 -
214 -def sgmv_expand_slice_fake(
215 - inputs: torch.Tensor,
216 - lora_b_weights: torch.Tensor,
217 - output_tensor: torch.Tensor,
218 - b_seq_start_loc: torch.Tensor,
219 - seq_len_tensor: torch.Tensor,
220 - lora_indices_tensor: torch.Tensor,
221 - batches: int,
222 - max_seq_length: int,
223 - token_nums: int,
224 - slice_offset: int,
225 - slice_size: int,
226 - add_inputs: bool = False,
227 -) -> None:
228 - return
229 -
230 -
231 -try:
232 - direct_register_custom_op(
233 - op_name="sgmv_expand_slice",
234 - op_func=_sgmv_expand_slice,
235 - mutates_args=["output_tensor"],
236 - fake_impl=sgmv_expand_slice_fake,
237 - )
238 - sgmv_expand_slice = torch.ops.vllm.sgmv_expand_slice
239 -
240 -except AttributeError:
241 - sgmv_expand_slice = _sgmv_expand_slice
vllm/lora/ops/sgmv_shrink.py
+74 -55
@@ -5,48 +5,60 @@ Punica: Multi-Tenant LoRA Serving.
5 https://arxiv.org/abs/2310.18547
6 """
7
8 +from typing import List
9 +
10 import torch
11 import triton
12 import triton.language as tl
13
14 from vllm.utils import direct_register_custom_op
15
16 +from .utils import _get_lora_a_ptr
17 +
18
19 @triton.jit
20 def _sgmv_shrink_kernel(
17 - input_ptr,
18 - lora_ptr,
19 - out_ptr,
20 - N,
21 - K,
22 - b_seq_start_loc,
23 - seq_lens,
24 - lora_indices,
25 - scaling,
26 - xm_stride, # hidden_size
27 - xk_stride, # 1
28 - l0_stride, # hidden_size*max_rank
29 - lora_k_stride,
30 - lora_n_stride,
31 - cm_stride,
32 - cn_stride,
33 - BLOCK_M: tl.constexpr,
34 - BLOCK_N: tl.constexpr,
35 - BLOCK_K: tl.constexpr,
36 - EVEN_K: tl.constexpr,
37 - SPLIT_K: tl.constexpr,
38 -):
21 + input_ptr,
22 + lora_ptr, #1-3
23 + out_ptr,
24 + N,
25 + K,
26 + b_seq_start_loc,
27 + seq_lens,
28 + lora_indices,
29 + scaling,
30 + input_d0_stride,
31 + input_d1_stride, # 1
32 + lora_d0_stride,
33 + lora_d1_stride,
34 + lora_d2_stride, # 1
35 + output_d0_stride,
36 + output_d1_stride,
37 + output_d2_stride, # 1
38 + BLOCK_M: tl.constexpr,
39 + BLOCK_N: tl.constexpr,
40 + BLOCK_K: tl.constexpr,
41 + EVEN_K: tl.constexpr,
42 + SPLIT_K: tl.constexpr,
43 + SLICE_NUM: tl.constexpr):
44 """
45 The sgmv's shrink triton kernel is based on GroupGEMM+SPLIT-K.
46 The GEMM of Multi-LoRA can be considered as GroupGEMM. Additionally,
47 introducing SPLIT-K can improve performance
48 """
49 pid = tl.program_id(axis=0)
45 - pid_sk = tl.program_id(axis=1)
50 + pid_mix = tl.program_id(axis=1)
51 cur_batch = tl.program_id(axis=2)
52 cta_n_num = tl.cdiv(N, BLOCK_N)
53 pid_m = pid // cta_n_num
54 pid_n = pid % cta_n_num
55 + if SLICE_NUM == 1:
56 + slice_id: tl.constexpr = 0
57 + pid_sk = tl.program_id(axis=1)
58 + else:
59 + pid_mix = tl.program_id(axis=1)
60 + slice_id = pid_mix // SPLIT_K
61 + pid_sk = pid_mix % SPLIT_K
62
63 M = tl.load(seq_lens + cur_batch)
64 if pid_m * BLOCK_M > M:
@@ -61,11 +73,22 @@ def _sgmv_shrink_kernel(
73
74 ram = tl.max_contiguous(tl.multiple_of(offset_m % M, BLOCK_M), BLOCK_M)
75 rbn = tl.max_contiguous(tl.multiple_of(offset_n % N, BLOCK_N), BLOCK_N)
76 + # input ptr
77 + a_ptr = (input_ptr + cur_seq_start * input_d0_stride +
78 + ram[:, None] * input_d0_stride +
79 + offset_k[None, :] * input_d1_stride)
80 +
81 + if SLICE_NUM == 1:
82 + # current lora ptr
83 + cur_lora_ptr = lora_ptr
84 + else:
85 + # current lora ptr
86 + cur_lora_ptr = tl.load(lora_ptr + slice_id).to(
87 + tl.pointer_type(input_ptr.dtype.element_ty))
88
65 - a_ptr = (input_ptr + cur_seq_start * xm_stride + ram[:, None] * xm_stride +
66 - offset_k[None, :] * xk_stride)
67 - b_ptr = (lora_ptr + l0_stride * lora_index + rbn[None, :] * lora_k_stride +
68 - offset_k[:, None] * lora_n_stride)
89 + b_ptr = (cur_lora_ptr + lora_d0_stride * lora_index +
90 + rbn[None, :] * lora_d1_stride +
91 + offset_k[:, None] * lora_d2_stride)
92
93 accumulator = tl.zeros((BLOCK_M, BLOCK_N), dtype=tl.float32)
94 for k in range(0, tl.cdiv(K, BLOCK_K * SPLIT_K)):
@@ -82,13 +105,15 @@ def _sgmv_shrink_kernel(
105 other=0.0)
106 accumulator += tl.dot(tiled_a, tiled_b)
107
85 - a_ptr += BLOCK_K * SPLIT_K * xk_stride
86 - b_ptr += BLOCK_K * SPLIT_K * lora_n_stride
108 + a_ptr += BLOCK_K * SPLIT_K * input_d1_stride
109 + b_ptr += BLOCK_K * SPLIT_K * lora_d2_stride
110 offset_cm = cur_seq_start + tl.arange(0, BLOCK_M) + pid_m * BLOCK_M
111
112 offset_cn = tl.arange(0, BLOCK_N) + pid_n * BLOCK_N
90 - c_ptr = (out_ptr + offset_cm[:, None] * cm_stride +
91 - offset_cn[None, :] * cn_stride)
113 + cur_out_ptr = (out_ptr if SLICE_NUM == 1 else out_ptr +
114 + slice_id * output_d0_stride)
115 + c_ptr = cur_out_ptr + offset_cm[:, None] * output_d1_stride + offset_cn[
116 + None, :] * output_d2_stride
117 c_mask = (offset_cm[:, None] <
118 (cur_seq_start + M)) & (offset_cn[None, :] < N)
119 accumulator *= scaling
@@ -102,7 +127,7 @@ def _sgmv_shrink_kernel(
127 @torch.inference_mode()
128 def _sgmv_shrink(
129 inputs: torch.Tensor,
105 - lora_a_weights: torch.Tensor,
130 + lora_a_weights: List[torch.Tensor],
131 output_tensor: torch.Tensor,
132 b_seq_start_loc: torch.Tensor,
133 seq_len_tensor: torch.Tensor,
@@ -113,10 +138,9 @@ def _sgmv_shrink(
138 scaling: float,
139 ) -> None:
140 """
116 -
141 Args:
142 inputs (torch.Tensor): input tensor
119 - lora_a_weights (torch.Tensor): lora'a weight
143 + lora_a_weights (List[torch.Tensor]): lora'a weight
144 output_tensor (torch.Tensor): output tensor
145 b_seq_start_loc (torch.Tensor): (batch_size,). The cumulative
146 sequence lengths of the sequences in the batch, used to index
@@ -134,27 +158,21 @@ def _sgmv_shrink(
158 token numbers in the inputs matches the one in the metadata.
159 scaling (float): Scaling factor.
160 """
137 - assert inputs.dtype == lora_a_weights.dtype
161 + assert inputs.dtype == lora_a_weights[0].dtype
162 assert inputs.dtype in [torch.float16, torch.bfloat16]
139 - assert lora_a_weights.dtype in [
140 - torch.float16,
141 - torch.bfloat16,
142 - ]
163 + for weight in lora_a_weights:
164 + assert weight.dtype in [torch.float16, torch.bfloat16]
165 +
166 assert inputs.size(0) == token_nums
144 - assert inputs.size(1) == lora_a_weights.size(-1)
167 + assert inputs.size(1) == lora_a_weights[0].size(-1)
168 assert b_seq_start_loc.size(0) == batches
169 assert lora_indices_tensor.size(0) == batches
170 assert inputs.is_contiguous()
148 -
149 - if lora_a_weights.ndim == 4: # shape:(lora_num,1,rank, size)
150 - assert lora_a_weights.size(1) == 1
151 - lora_a_weights = lora_a_weights.squeeze(dim=1)
152 - else:
153 - assert lora_a_weights.ndim == 3 # shape:(lora_num,rank, size)
154 - assert lora_a_weights.is_contiguous()
171 assert output_tensor.is_contiguous()
172 + (lora_ptr_tensor, lora_strides_d0, lora_strides_d1,
173 + lora_strides_d2) = _get_lora_a_ptr(lora_a_weights, b_seq_start_loc.device)
174 # TODO tuning this config
157 - N, K = lora_a_weights.shape[-2:] # K=hidden_size,N=rank
175 + N, K = lora_a_weights[0].shape[-2:] # K=hidden_size,N=rank
176 BLOCK_M = 32
177 BLOCK_N = 16
178 BLOCK_K = 32
@@ -162,13 +180,12 @@ def _sgmv_shrink(
180 EVEN_K = K % (BLOCK_K * SPLIT_K) == 0
181 grid = (
182 triton.cdiv(max_seq_length, BLOCK_M) * triton.cdiv(N, BLOCK_N),
165 - SPLIT_K,
183 + SPLIT_K * len(lora_a_weights),
184 batches,
185 )
168 -
186 _sgmv_shrink_kernel[grid](
187 inputs,
171 - lora_a_weights,
188 + lora_ptr_tensor,
189 output_tensor,
190 N,
191 K,
@@ -178,23 +195,25 @@ def _sgmv_shrink(
195 scaling,
196 inputs.stride(0),
197 inputs.stride(1),
181 - lora_a_weights.stride(0),
182 - lora_a_weights.stride(1),
183 - lora_a_weights.stride(2),
198 + lora_strides_d0,
199 + lora_strides_d1,
200 + lora_strides_d2,
201 output_tensor.stride(0),
202 output_tensor.stride(1),
203 + output_tensor.stride(2),
204 BLOCK_M,
205 BLOCK_N,
206 BLOCK_K,
207 EVEN_K,
208 SPLIT_K,
209 + len(lora_a_weights),
210 )
211 return
212
213
214 def sgmv_shrink_fake(
215 inputs: torch.Tensor,
197 - lora_a_weights: torch.Tensor,
216 + lora_a_weights: List[torch.Tensor],
217 output_tensor: torch.Tensor,
218 b_seq_start_loc: torch.Tensor,
219 seq_len_tensor: torch.Tensor,
vllm/lora/ops/utils.py
+120 -1
@@ -1,5 +1,7 @@
1 import functools
2 -from typing import Dict
2 +from typing import Dict, List, Tuple
3 +
4 +import torch
5
6
7 @functools.lru_cache
@@ -44,3 +46,120 @@ def get_lora_op_configs(op_type: str, batch: int,
46 if not config:
47 config = _get_default_config(op_type, batch, hidden_size)
48 return config
49 +
50 +
51 +_LORA_A_PTR_DICT: Dict[Tuple[int, ...], Tuple[torch.tensor, ...]] = {}
52 +_LORA_B_PTR_DICT: Dict[Tuple[int, ...], Tuple[torch.tensor, ...]] = {}
53 +
54 +
55 +def _get_lora_a_ptr(lora_a_weights: List[torch.Tensor], device: str):
56 + """
57 + `_LORA_A_PTR_DICT` collects the required information during `profile_run`,
58 + After this, it remains constant and subsequent usage is through LUT.
59 + Refer to:
60 + https://github.com/triton-lang/triton/blob/release/3.1.x/python/tutorials/08-grouped-gemm.py
61 + """
62 + key = tuple(lora_weight.data_ptr() for lora_weight in lora_a_weights)
63 +
64 + if values := _LORA_A_PTR_DICT.get(key):
65 + return values
66 +
67 + lora_strides_d0 = []
68 + lora_strides_d1 = []
69 + lora_strides_d2 = []
70 + tensor_ptrs = []
71 + for lora_a_weight in lora_a_weights:
72 + if lora_a_weight.ndim == 4: # shape:(lora_num,1,size,rank)
73 + assert lora_a_weight.size(1) == 1
74 + lora_a_weight = lora_a_weight.squeeze(dim=1)
75 + else:
76 + assert lora_a_weight.ndim == 3 # shape:(lora_num,size,rank)
77 + assert lora_a_weight.is_contiguous()
78 + tensor_ptrs.append(lora_a_weight.data_ptr())
79 + lora_strides_d0.append(lora_a_weight.stride(0))
80 + lora_strides_d1.append(lora_a_weight.stride(1))
81 + lora_strides_d2.append(lora_a_weight.stride(2))
82 + if len(lora_a_weights) > 1:
83 + lora_ptr_tensor = torch.tensor(tensor_ptrs, device=device)
84 + else:
85 + lora_ptr_tensor = lora_a_weights[0]
86 +
87 + if (len(set(lora_strides_d0)) > 1 or len(set(lora_strides_d1)) > 1
88 + or len(set(lora_strides_d2)) > 1):
89 + raise ValueError("All LoRA weights must have the same stride.")
90 +
91 + _LORA_A_PTR_DICT[key] = (
92 + lora_ptr_tensor,
93 + lora_strides_d0[0],
94 + lora_strides_d1[0],
95 + lora_strides_d2[0],
96 + )
97 + return _LORA_A_PTR_DICT.get(key)
98 +
99 +
100 +def _get_lora_b_ptr(lora_weights: List[torch.Tensor], offset_start: int,
101 + device: str):
102 + """
103 + `_LORA_B_PTR_DICT` collects the required information during `profile_run`,
104 + After this, it remains constant and subsequent usage is through LUT.
105 + Refer to:
106 + https://github.com/triton-lang/triton/blob/release/3.1.x/python/tutorials/08-grouped-gemm.py
107 +
108 + """
109 +
110 + key = tuple(lora_weight.data_ptr() for lora_weight in lora_weights)
111 + if values := _LORA_B_PTR_DICT.get(key):
112 + return values
113 + slice_offset_lst = []
114 + tensor_ptrs = []
115 + lora_strides_d0 = []
116 + lora_strides_d1 = []
117 + lora_strides_d2 = []
118 + hidden_sizes = []
119 + slice_offset = offset_start
120 + for lora_b_weight in lora_weights:
121 + if lora_b_weight.ndim == 4: # shape:(lora_num,1,size,rank)
122 + assert lora_b_weight.size(1) == 1
123 + lora_b_weight = lora_b_weight.squeeze(dim=1)
124 + else:
125 + assert lora_b_weight.ndim == 3 # shape:(lora_num,size,rank)
126 + assert lora_b_weight.is_contiguous()
127 + tensor_ptrs.append(lora_b_weight.data_ptr())
128 + lora_strides_d0.append(lora_b_weight.stride(0))
129 + lora_strides_d1.append(lora_b_weight.stride(1))
130 + lora_strides_d2.append(lora_b_weight.stride(2))
131 + slice_offset_lst.append(slice_offset)
132 + slice_offset += lora_b_weight.size(1)
133 + hidden_sizes.append(lora_b_weight.size(1))
134 +
135 + if len(lora_weights) > 1:
136 + # note these are device tensors
137 + lora_ptr_tensor = torch.tensor(tensor_ptrs, device=device)
138 + slice_start_tensor = torch.tensor(slice_offset_lst, device=device)
139 + else:
140 + slice_start_tensor = slice_offset_lst[0]
141 + lora_ptr_tensor = lora_b_weight[0]
142 +
143 + # If each lora has the same stride, there's no need to use a
144 + # tensor for storage.
145 + if (len(set(lora_strides_d0)) == 1 and len(set(lora_strides_d1)) == 1 and
146 + len(set(lora_strides_d2)) == 1) and len(set(hidden_sizes)) == 1:
147 + lora_strides_d0_tensor = lora_strides_d0[0]
148 + lora_strides_d1_tensor = lora_strides_d1[0]
149 + lora_strides_d2_tensor = lora_strides_d2[0]
150 + hidden_sizes_tensor = hidden_sizes[0]
151 + same_stride = True
152 +
153 + else:
154 + lora_strides_d0_tensor = torch.tensor(lora_strides_d0, device=device)
155 + lora_strides_d1_tensor = torch.tensor(lora_strides_d1, device=device)
156 + lora_strides_d2_tensor = torch.tensor(lora_strides_d2, device=device)
157 + hidden_sizes_tensor = torch.tensor(hidden_sizes, device=device)
158 + same_stride = False
159 + # MAX_N is the maximum hidden size among all the lora_b weights
160 + MAX_N = max(hidden_sizes)
161 + _LORA_B_PTR_DICT[key] = (slice_start_tensor, lora_ptr_tensor,
162 + lora_strides_d0_tensor, lora_strides_d1_tensor,
163 + lora_strides_d2_tensor, hidden_sizes_tensor,
164 + same_stride, MAX_N)
165 + return _LORA_B_PTR_DICT.get(key)
vllm/lora/punica_wrapper/punica_gpu.py
+55 -99
@@ -5,7 +5,7 @@ Punica: Multi-Tenant LoRA Serving.
5 https://arxiv.org/abs/2310.18547
6 """
7
8 -from typing import Callable, Optional, Tuple, Union, final
8 +from typing import Optional, Tuple, Union, final
9
10 import torch
11
@@ -16,7 +16,6 @@ if HAS_TRITON:
16 from vllm.lora.ops.bgmv_expand_slice import bgmv_expand_slice
17 from vllm.lora.ops.bgmv_shrink import bgmv_shrink
18 from vllm.lora.ops.sgmv_expand import sgmv_expand
19 - from vllm.lora.ops.sgmv_expand_slice import sgmv_expand_slice
19 from vllm.lora.ops.sgmv_shrink import sgmv_shrink
20
21 from .punica_base import PunicaWrapperBase
@@ -35,11 +34,11 @@ class PunicaWrapperGPU(PunicaWrapperBase):
34 PunicaWrapperBase.__init__(self, max_num_batched_tokens, max_batches,
35 device)
36
38 - def _shrink_prefill(
37 + def _apply_shrink_prefill(
38 self,
39 y: torch.Tensor,
40 x: torch.Tensor,
42 - w_t_all: torch.Tensor,
41 + w_t_all: Tuple[torch.Tensor, ...],
42 scale: float,
43 ):
44 #No LoRA request, so return directly
@@ -53,7 +52,7 @@ class PunicaWrapperGPU(PunicaWrapperBase):
52 scale,
53 )
54
56 - def _shrink_decode(
55 + def _apply_shrink_decode(
56 self,
57 y: torch.Tensor,
58 x: torch.Tensor,
@@ -62,56 +61,28 @@ class PunicaWrapperGPU(PunicaWrapperBase):
61 ):
62 bgmv_shrink(x, w_t_all, y, self.token_lora_indices, scale)
63
65 - def _expand_prefill(
64 + def _apply_expand_prefill(
65 self,
66 y: torch.Tensor,
67 x: torch.Tensor,
68 w_t_all: torch.Tensor,
69 + offset_start: int,
70 add_inputs: bool,
71 ):
72 #No LoRA request, so return directly
73 if self.no_lora:
74 return
75 - sgmv_expand(
76 - x,
77 - w_t_all,
78 - y,
79 - *self.prefill_metadata,
80 - add_inputs,
81 - )
82 -
83 - def _expand_decode(
84 - self,
85 - y: torch.Tensor,
86 - x: torch.Tensor,
87 - w_t_all: torch.Tensor,
88 - add_inputs: bool,
89 - ):
90 - bgmv_expand(x, w_t_all, y, self.token_lora_indices, add_inputs)
75
92 - def _expand_slice_prefill(
93 - self,
94 - y: torch.Tensor,
95 - x: torch.Tensor,
96 - w_t_all: torch.Tensor,
97 - y_offset: Optional[int],
98 - y_slice_size: Optional[int],
99 - add_inputs: bool,
100 - ):
101 - #No LoRA request, so return directly
102 - if self.no_lora:
103 - return
104 - sgmv_expand_slice(
76 + sgmv_expand(
77 x,
78 w_t_all,
79 y,
80 *self.prefill_metadata,
109 - y_offset,
110 - y_slice_size,
111 - add_inputs,
81 + offset_start=offset_start,
82 + add_inputs=add_inputs,
83 )
84
114 - def _expand_slice_decode(
85 + def _apply_expand_decode(
86 self,
87 y: torch.Tensor,
88 x: torch.Tensor,
@@ -123,43 +94,6 @@ class PunicaWrapperGPU(PunicaWrapperBase):
94 bgmv_expand_slice(x, w_t_all, y, self.token_lora_indices, y_offset,
95 y_slice_size, add_inputs)
96
126 - def _apply_expand(
127 - self,
128 - y: torch.Tensor,
129 - x: torch.Tensor,
130 - w_t_all: torch.Tensor,
131 - y_offset: Optional[int],
132 - y_slice_size: Optional[int],
133 - add_inputs: bool = True,
134 - ):
135 - """
136 - Perform the ` y[:,y_offset:y_offset+y_slice_size]+=x@w_t_all`
137 - computation, which is suitable for the
138 - GEMM of lora'b.
139 - """
140 -
141 - expand_slice_fun: Callable = (self._expand_slice_prefill
142 - if self.is_prefill else
143 - self._expand_slice_decode)
144 - expand_slice_fun(y, x, w_t_all, y_offset, y_slice_size, add_inputs)
145 -
146 - def _apply_shrink(self, y: torch.Tensor, x: torch.Tensor,
147 - w_t_all: torch.Tensor, scale: float):
148 - """
149 - Perform the ` y+=x@w_t_all` computation, which is suitable for the
150 - GEMM of lora'a.
151 - When `is_prefill is` true, it indicates that it is currently the
152 - prefill stage, and the `_shrink_prefill` function should be called.
153 - Otherwise, it is the decode stage, and the _shrink_decode function
154 - should be called.
155 - """
156 - y_org = y
157 - y = y.view(-1, y.shape[-1])
158 - shrink_fun: Callable = (self._shrink_prefill
159 - if self.is_prefill else self._shrink_decode)
160 - shrink_fun(y, x, w_t_all, scale)
161 - y = y.view_as(y_org)
162 -
97 def add_shrink(self, y: Union[Tuple[torch.Tensor, ...], torch.Tensor],
98 x: torch.Tensor, lora_a_stacked: Tuple[torch.Tensor, ...],
99 scale: float, **kwargs):
@@ -182,10 +116,15 @@ class PunicaWrapperGPU(PunicaWrapperBase):
116 """
117
118 x = x.view(-1, x.shape[-1])
185 - # TODO fuse these kernels
186 - for slice_idx in range(len(lora_a_stacked)):
187 - self._apply_shrink(y[slice_idx], x, lora_a_stacked[slice_idx],
188 - scale)
119 +
120 + if self.is_prefill:
121 + # NOTE fused kernel
122 + self._apply_shrink_prefill(y, x, lora_a_stacked, scale)
123 + else:
124 + # TODO fuse these kernels
125 + for slice_idx in range(len(lora_a_stacked)):
126 + self._apply_shrink_decode(y[slice_idx], x,
127 + lora_a_stacked[slice_idx], scale)
128
129 def add_expand(self,
130 y: torch.Tensor,
@@ -217,20 +156,28 @@ class PunicaWrapperGPU(PunicaWrapperBase):
156 """
157 y_org = y
158 y = y.view(-1, y.shape[-1])
220 - offset_left = offset_start
159 if lora_bias_stacked is not None:
160 self._apply_bias(self.token_lora_indices, y, output_slices,
161 lora_bias_stacked)
224 - for slice_idx in range(len(lora_b_stacked)):
225 - self._apply_expand(
226 - y,
227 - x[slice_idx],
228 - lora_b_stacked[slice_idx],
229 - offset_left,
230 - output_slices[slice_idx],
231 - add_inputs=add_inputs,
232 - )
233 - offset_left += output_slices[slice_idx]
162 + if self.is_prefill:
163 + # NOTE fused kernel
164 + self._apply_expand_prefill(y,
165 + x,
166 + lora_b_stacked,
167 + offset_start,
168 + add_inputs=True)
169 + else:
170 + # TODO fuse these kernels
171 + for slice_idx in range(len(lora_b_stacked)):
172 + self._apply_expand_decode(
173 + y,
174 + x[slice_idx],
175 + lora_b_stacked[slice_idx],
176 + offset_start,
177 + output_slices[slice_idx],
178 + add_inputs=add_inputs,
179 + )
180 + offset_start += output_slices[slice_idx]
181 y = y.view_as(y_org)
182
183 def add_lora_embedding(self,
@@ -252,10 +199,18 @@ class PunicaWrapperGPU(PunicaWrapperBase):
199 add_inputs (bool): Default to True.
200 """
201
255 - # Embedding layer only need expand op
256 - expand_fun: Callable = (self._expand_prefill
257 - if self.is_prefill else self._expand_decode)
258 - expand_fun(y, x, lora_b_stacked, add_inputs)
202 + if self.is_prefill:
203 + sgmv_expand(
204 + x.unsqueeze(dim=0),
205 + [lora_b_stacked],
206 + y,
207 + *self.prefill_metadata,
208 + offset_start=0,
209 + add_inputs=add_inputs,
210 + )
211 + else:
212 + bgmv_expand(x, lora_b_stacked, y, self.token_lora_indices,
213 + add_inputs)
214
215 def add_lora_linear(self,
216 y: torch.Tensor,
@@ -301,10 +256,11 @@ class PunicaWrapperGPU(PunicaWrapperBase):
256 r = lora_b_stacked[0].size(-1)
257 # We set the buffer to be float32 by default ,refer to:
258 # https://github.com/triton-lang/triton/issues/1387
304 - buffer = tuple(
305 - torch.zeros(
306 - (x.size(0), r), dtype=torch.float32, device=x.device)
307 - for _ in range(len(output_slices)))
259 + buffer = torch.zeros(
260 + (len(output_slices), x.size(0), r),
261 + dtype=torch.float32,
262 + device=x.device,
263 + )
264 self.add_shrink(buffer, x, lora_a_stacked, scale, **kwargs)
265 self.add_expand(y,
266 buffer,