A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 02:00 UTC
[Hardware][CPU] Multi-LoRA implementation for the CPU backend (#11100)
Signed-off-by: Akshat Tripathi <akshat@krai.ai> Signed-off-by: Oleg Mosalov <oleg@krai.ai> Signed-off-by: Jee Jee Li <pandaleefree@gmail.com> Co-authored-by: Oleg Mosalov <oleg@krai.ai> Co-authored-by: Jee Jee Li <pandaleefree@gmail.com> Co-authored-by: Isotr0py <2037008807@qq.com>
Akshat Tripathi committed
Jan 12, 2025 at 13:01 UTC
8bddb735123204872788a8ffe117321de7550e6c
25 files changed
+855
-193
.buildkite/run-cpu-test.sh
+6
@@ -75,6 +75,12 @@ function cpu_tests() {
75
--num-prompts 20 \
76
--endpoint /v1/completions \
77
--tokenizer facebook/opt-125m"
78
+
79
+ # Run multi-lora tests
80
+ docker exec cpu-test-"$BUILDKITE_BUILD_NUMBER"-"$NUMA_NODE" bash -c "
81
+ set -e
82
+ pytest -s -v \
83
+ tests/lora/test_qwen2vl.py"
84
}
85
86
# All of CPU tests are expected to be finished less than 25 mins.
docs/source/features/compatibility_matrix.md
+1
-1
@@ -359,7 +359,7 @@ Check the '✗' with links to see tracking issue for unsupported feature/hardwar
359
- ✅
360
- ✅
361
- ✅
362
- - [✗](gh-pr:4830)
362
+ - ✅
363
- ✅
364
* - <abbr title="Prompt Adapter">prmpt adptr</abbr>
365
- ✅
tests/lora/conftest.py
+19
-13
@@ -21,6 +21,7 @@ from vllm.model_executor.layers.logits_processor import LogitsProcessor
21
from vllm.model_executor.layers.sampler import Sampler
22
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
23
from vllm.model_executor.model_loader import get_model
24
+from vllm.platforms import current_platform
25
26
27
class ContextIDInfo(TypedDict):
@@ -65,13 +66,16 @@ def cleanup_fixture(should_do_global_cleanup_after_test: bool):
66
@pytest.fixture
67
def dist_init():
68
temp_file = tempfile.mkstemp()[1]
68
- init_distributed_environment(
69
- world_size=1,
70
- rank=0,
71
- distributed_init_method=f"file://{temp_file}",
72
- local_rank=0,
73
- backend="nccl",
74
- )
69
+
70
+ backend = "nccl"
71
+ if current_platform.is_cpu():
72
+ backend = "gloo"
73
+
74
+ init_distributed_environment(world_size=1,
75
+ rank=0,
76
+ distributed_init_method=f"file://{temp_file}",
77
+ local_rank=0,
78
+ backend=backend)
79
initialize_model_parallel(1, 1)
80
yield
81
cleanup_dist_env_and_memory(shutdown_ray=True)
@@ -81,13 +85,15 @@ def dist_init():
85
def dist_init_torch_only():
86
if torch.distributed.is_initialized():
87
return
88
+ backend = "nccl"
89
+ if current_platform.is_cpu():
90
+ backend = "gloo"
91
+
92
temp_file = tempfile.mkstemp()[1]
85
- torch.distributed.init_process_group(
86
- backend="nccl",
87
- world_size=1,
88
- rank=0,
89
- init_method=f"file://{temp_file}",
90
- )
93
+ torch.distributed.init_process_group(world_size=1,
94
+ rank=0,
95
+ init_method=f"file://{temp_file}",
96
+ backend=backend)
97
98
99
@pytest.fixture
tests/lora/test_layers.py
+30
-9
@@ -48,10 +48,14 @@ TOLERANCES = {
48
torch.float32: (5e-3, 5e-3),
49
torch.bfloat16: (3e-2, 2e-2),
50
}
51
-# TODO: Modify this based on platform
52
-DEVICES = [
51
+
52
+pytestmark = pytest.mark.skipif(
53
+ not (current_platform.is_cuda_alike() or current_platform.is_cpu()),
54
+ reason="Backend not supported")
55
+
56
+DEVICES = ([
57
f"cuda:{i}" for i in range(1 if torch.cuda.device_count() == 1 else 2)
54
-]
58
+] if current_platform.is_cuda_alike() else ["cpu"])
59
60
#For GPU, we will launch different triton kernels between the prefill and decode
61
# stages, so we need to verify this. prefill stage(True) or decode stage(False)
@@ -198,6 +202,10 @@ def check_punica_wrapper(punica_wrapper) -> bool:
202
from vllm.lora.punica_wrapper.punica_gpu import PunicaWrapperGPU
203
204
return type(punica_wrapper) is PunicaWrapperGPU
205
+ elif current_platform.is_cpu():
206
+ from vllm.lora.punica_wrapper.punica_cpu import PunicaWrapperCPU
207
+
208
+ return type(punica_wrapper) is PunicaWrapperCPU
209
else:
210
return False
211
@@ -211,7 +219,8 @@ def test_embeddings(dist_init, num_loras, device, vocab_size, stage) -> None:
219
# For multi-GPU testing of Triton kernel, we must explicitly set the CUDA
220
# device, see: https://github.com/triton-lang/triton/issues/2925
221
# Same below.
214
- torch.cuda.set_device(device)
222
+ if current_platform.is_cuda_alike():
223
+ torch.cuda.set_device(device)
224
225
torch.set_default_device(device)
226
max_loras = 8
@@ -313,7 +322,9 @@ def test_embeddings(dist_init, num_loras, device, vocab_size, stage) -> None:
322
def test_embeddings_with_new_embeddings(dist_init, num_loras, device,
323
vocab_size, stage) -> None:
324
316
- torch.cuda.set_device(device)
325
+ if current_platform.is_cuda_alike():
326
+ torch.cuda.set_device(device)
327
+
328
torch.set_default_device(device)
329
max_loras = 8
330
punica_wrapper = get_punica_wrapper(8192, 256, device)
@@ -450,7 +461,9 @@ def test_embeddings_with_new_embeddings(dist_init, num_loras, device,
461
def test_lm_head_logits_processor(dist_init, num_loras, device, vocab_size,
462
stage) -> None:
463
453
- torch.cuda.set_device(device)
464
+ if current_platform.is_cuda_alike():
465
+ torch.cuda.set_device(device)
466
+
467
torch.set_default_device(device)
468
max_loras = 8
469
punica_wrapper = get_punica_wrapper(8192, 256, device)
@@ -582,7 +595,9 @@ def test_lm_head_logits_processor(dist_init, num_loras, device, vocab_size,
595
def test_linear_replicated(dist_init, num_loras, device, stage,
596
bias_enabled) -> None:
597
585
- torch.cuda.set_device(device)
598
+ if current_platform.is_cuda_alike():
599
+ torch.cuda.set_device(device)
600
+
601
torch.set_default_device(device)
602
punica_wrapper = get_punica_wrapper(8192, 256, device)
603
assert check_punica_wrapper(punica_wrapper)
@@ -695,7 +710,9 @@ def test_linear_replicated(dist_init, num_loras, device, stage,
710
def test_linear_parallel(dist_init, num_loras, orientation, fully_shard,
711
device, stage, bias_enabled) -> None:
712
698
- torch.cuda.set_device(device)
713
+ if current_platform.is_cuda_alike():
714
+ torch.cuda.set_device(device)
715
+
716
torch.set_default_device(device)
717
punica_wrapper = get_punica_wrapper(8192, 256, device)
718
assert check_punica_wrapper(punica_wrapper)
@@ -818,7 +835,9 @@ def test_linear_parallel(dist_init, num_loras, orientation, fully_shard,
835
def test_column_parallel_packed(dist_init, num_loras, repeats, fully_shard,
836
device, stage, bias_enabled) -> None:
837
821
- torch.cuda.set_device(device)
838
+ if current_platform.is_cuda_alike():
839
+ torch.cuda.set_device(device)
840
+
841
torch.set_default_device(device)
842
punica_wrapper = get_punica_wrapper(8192, 256, device)
843
assert check_punica_wrapper(punica_wrapper)
@@ -971,6 +990,8 @@ def test_column_parallel_packed(dist_init, num_loras, repeats, fully_shard,
990
@pytest.mark.parametrize("rotary_dim", [None, 32])
991
@pytest.mark.parametrize("head_size", [32, 108])
992
@pytest.mark.parametrize("seq_len", [11, 1024])
993
+@pytest.mark.skipif(not current_platform.is_cuda_alike(),
994
+ reason="Only CUDA backends are supported")
995
def test_rotary_embedding_long_context(dist_init, num_loras, device,
996
scaling_factors, max_position,
997
is_neox_style, rotary_dim, head_size,
tests/lora/test_lora_manager.py
+11
-10
@@ -20,6 +20,7 @@ from vllm.lora.request import LoRARequest
20
from vllm.lora.worker_manager import (LRUCacheWorkerLoRAManager,
21
WorkerLoRAManager)
22
from vllm.model_executor.layers.linear import RowParallelLinear
23
+from vllm.platforms import current_platform
24
25
EMBEDDING_MODULES = {
26
"embed_tokens": "input_embeddings",
@@ -28,9 +29,9 @@ EMBEDDING_MODULES = {
29
30
EMBEDDING_PADDING_MODULES = ["lm_head"]
31
31
-CUDA_DEVICES = [
32
+DEVICES = ([
33
f"cuda:{i}" for i in range(1 if torch.cuda.device_count() == 1 else 2)
33
-]
34
+] if current_platform.is_cuda_alike() else ["cpu"])
35
36
37
def test_peft_helper(sql_lora_files):
@@ -83,7 +84,7 @@ def test_peft_helper(sql_lora_files):
84
PEFTHelper.from_dict(config)
85
86
86
-@pytest.mark.parametrize("device", CUDA_DEVICES)
87
+@pytest.mark.parametrize("device", DEVICES)
88
def test_from_lora_tensors(sql_lora_files, device):
89
tensors = load_file(
90
os.path.join(sql_lora_files, "adapter_model.safetensors"))
@@ -171,7 +172,7 @@ def test_replace_submodules(dist_init, dummy_model):
172
manager = LoRAModelManager(
173
model, 1, 1, 1,
174
LoRAConfig(max_lora_rank=8, max_cpu_loras=8, max_loras=8),
174
- torch.device("cuda"))
175
+ torch.device(DEVICES[0]))
176
model = manager.model
177
178
assert isinstance(model.get_submodule("dense1"),
@@ -183,7 +184,7 @@ def test_replace_submodules(dist_init, dummy_model):
184
RowParallelLinearWithLoRA)
185
186
186
-@pytest.mark.parametrize("device", CUDA_DEVICES)
187
+@pytest.mark.parametrize("device", DEVICES)
188
def test_lora_model_manager(dist_init, dummy_model, device):
189
model = dummy_model
190
model.supported_lora_modules = ["dense1", "dense2", "lm_head"]
@@ -244,7 +245,7 @@ def test_lora_model_manager(dist_init, dummy_model, device):
245
assert manager.punica_wrapper.device == device
246
247
247
-@pytest.mark.parametrize("device", CUDA_DEVICES)
248
+@pytest.mark.parametrize("device", DEVICES)
249
def test_lora_lru_cache_model_manager(dist_init, dummy_model, device):
250
model = dummy_model
251
model.supported_lora_modules = ["dense1", "dense2", "lm_head"]
@@ -336,7 +337,7 @@ def test_lora_lru_cache_model_manager(dist_init, dummy_model, device):
337
assert manager.device == device
338
339
339
-@pytest.mark.parametrize("device", CUDA_DEVICES)
340
+@pytest.mark.parametrize("device", DEVICES)
341
def test_lru_lora_model_manager(dist_init, dummy_model, device):
342
# This tests just the LRU cache functionality, everything else is
343
# tested in test_lora_model_manager
@@ -466,7 +467,7 @@ def test_lru_lora_model_manager(dist_init, dummy_model, device):
467
assert manager.device == device
468
469
469
-@pytest.mark.parametrize("device", CUDA_DEVICES)
470
+@pytest.mark.parametrize("device", DEVICES)
471
def test_lru_cache_worker_adapter_manager(llama_2_7b_model_extra_embeddings,
472
sql_lora_files, device):
473
lora_config = LoRAConfig(max_lora_rank=8, max_cpu_loras=4, max_loras=4)
@@ -545,7 +546,7 @@ def test_lru_cache_worker_adapter_manager(llama_2_7b_model_extra_embeddings,
546
device)
547
548
548
-@pytest.mark.parametrize("device", CUDA_DEVICES)
549
+@pytest.mark.parametrize("device", DEVICES)
550
def test_worker_adapter_manager(llama_2_7b_model_extra_embeddings,
551
sql_lora_files, device):
552
# Should remove every LoRA not specified in the request.
@@ -621,7 +622,7 @@ def test_worker_adapter_manager(llama_2_7b_model_extra_embeddings,
622
device)
623
624
624
-@pytest.mark.parametrize("device", CUDA_DEVICES)
625
+@pytest.mark.parametrize("device", DEVICES)
626
def test_packed_loras(dist_init, dummy_model_gate_up, device):
627
model = dummy_model_gate_up
628
model.supported_lora_modules = ["gate_up_proj"]
tests/lora/test_mixtral.py
+3
-1
@@ -5,6 +5,7 @@ import torch
5
6
import vllm
7
from vllm.lora.request import LoRARequest
8
+from vllm.platforms import current_platform
9
10
MODEL_PATH = "mistralai/Mixtral-8x7B-Instruct-v0.1"
11
@@ -31,7 +32,8 @@ def do_sample(llm: vllm.LLM, lora_path: str, lora_id: int,
32
@pytest.mark.parametrize("tp_size", [4])
33
def test_mixtral_lora(mixtral_lora_files, tp_size):
34
"""Original test, the LoRA model has the common target modules, not all"""
34
- if torch.cuda.device_count() < tp_size:
35
+ if torch.cuda.device_count(
36
+ ) < tp_size and tp_size > 1 and current_platform.is_cuda_alike():
37
pytest.skip(f"Not enough GPUs for tensor parallelism {tp_size}")
38
39
prompts = [
tests/lora/test_punica_ops_sizes.py
renamed
+75
-49
@@ -9,17 +9,16 @@ from threading import Lock
9
import pytest
10
import torch
11
12
-from vllm.lora.ops.bgmv_expand import bgmv_expand
13
-from vllm.lora.ops.bgmv_expand_slice import bgmv_expand_slice
14
-from vllm.lora.ops.bgmv_shrink import bgmv_shrink
15
-from vllm.lora.ops.sgmv_expand import sgmv_expand
16
-from vllm.lora.ops.sgmv_shrink import sgmv_shrink
17
-from vllm.lora.ops.utils import _LORA_A_PTR_DICT, _LORA_B_PTR_DICT
12
+import vllm.lora.ops.triton_ops # noqa: F401
13
+from vllm.lora.ops.torch_ops import (bgmv_expand, bgmv_expand_slice,
14
+ bgmv_shrink, sgmv_expand,
15
+ sgmv_expand_slice, sgmv_shrink)
16
+from vllm.lora.ops.triton_ops.utils import _LORA_A_PTR_DICT, _LORA_B_PTR_DICT
17
from vllm.platforms import current_platform
18
19
from .utils import (assert_close, generate_data,
20
generate_data_for_expand_nslices,
22
- generate_data_for_nslices, ref_torch_groupgemm)
21
+ generate_data_for_nslices)
22
23
HIDDEN_SIZES = [
24
128,
@@ -113,7 +112,7 @@ DTYPES = [torch.float16, torch.bfloat16]
112
MAX_RANKS = [32]
113
SCALES = [0.5]
114
SEED = [0]
116
-CUDA_DEVICES = [f"cuda:{0}"]
115
+DEVICES = [f"cuda:{0}"]
116
117
_dict_lock = Lock()
118
@@ -127,7 +126,7 @@ _dict_lock = Lock()
126
@pytest.mark.parametrize("dtype", DTYPES)
127
@pytest.mark.parametrize("op_type", ["shrink", "expand"])
128
@pytest.mark.parametrize("seed", SEED)
130
-@pytest.mark.parametrize("device", CUDA_DEVICES)
129
+@pytest.mark.parametrize("device", DEVICES)
130
def test_punica_sgmv(
131
batches: int,
132
num_loras: int,
@@ -174,7 +173,7 @@ def test_punica_sgmv(
173
# Preventing cache error pointer.
174
with _dict_lock:
175
_LORA_A_PTR_DICT.clear()
177
- sgmv_shrink(
176
+ torch.ops.vllm.sgmv_shrink(
177
inputs_tensor,
178
lora_weights_lst,
179
our_out_tensor,
@@ -187,20 +186,23 @@ def test_punica_sgmv(
186
scaling,
187
)
188
for index in range(nslices):
190
- ref_torch_groupgemm(
191
- ref_out_tensor[index],
189
+ sgmv_shrink(
190
inputs_tensor,
191
lora_weights_lst[index],
194
- lora_indices_tensor,
192
+ ref_out_tensor[index],
193
+ b_seq_start_loc,
194
seq_len_tensor,
195
+ lora_indices_tensor,
196
batches,
197
+ max_seq_length,
198
+ token_nums,
199
scaling,
198
- op_type,
200
)
201
+
202
else:
203
with _dict_lock:
204
_LORA_B_PTR_DICT.clear()
203
- sgmv_expand(
205
+ torch.ops.vllm.sgmv_expand(
206
inputs_tensor,
207
lora_weights_lst,
208
our_out_tensor,
@@ -213,21 +215,39 @@ def test_punica_sgmv(
215
offset_start=0,
216
add_inputs=True,
217
)
216
-
217
- slice_offset = 0
218
- for index in range(nslices):
219
- lora_weights = lora_weights_lst[index]
220
- ref_torch_groupgemm(
221
- ref_out_tensor[:, slice_offset:slice_offset + hidden_size],
222
- inputs_tensor[index],
223
- lora_weights,
224
- lora_indices_tensor,
218
+ if nslices == 1:
219
+ # Verify the torch's sgmv_expand op
220
+ sgmv_expand(
221
+ inputs_tensor[0],
222
+ lora_weights_lst[0],
223
+ ref_out_tensor,
224
+ b_seq_start_loc,
225
seq_len_tensor,
226
+ lora_indices_tensor,
227
batches,
227
- 1.0,
228
- op_type,
228
+ max_seq_length,
229
+ token_nums,
230
+ add_inputs=True,
231
)
230
- slice_offset += hidden_size
232
+ else:
233
+ slice_offset = 0
234
+ for index in range(nslices):
235
+ lora_weights = lora_weights_lst[index]
236
+ sgmv_expand_slice(
237
+ inputs_tensor[index],
238
+ lora_weights,
239
+ ref_out_tensor,
240
+ b_seq_start_loc,
241
+ seq_len_tensor,
242
+ lora_indices_tensor,
243
+ batches,
244
+ max_seq_length,
245
+ token_nums,
246
+ slice_offset,
247
+ hidden_size,
248
+ add_inputs=True,
249
+ )
250
+ slice_offset += hidden_size
251
252
assert_close(our_out_tensor, ref_out_tensor)
253
@@ -240,7 +260,7 @@ def test_punica_sgmv(
260
@pytest.mark.parametrize("dtype", DTYPES)
261
@pytest.mark.parametrize("op_type", ["shrink", "expand"])
262
@pytest.mark.parametrize("seed", SEED)
243
-@pytest.mark.parametrize("device", CUDA_DEVICES)
263
+@pytest.mark.parametrize("device", DEVICES)
264
def test_punica_bgmv(
265
batches: int,
266
num_loras: int,
@@ -276,31 +296,38 @@ def test_punica_bgmv(
296
device,
297
)
298
if op_type == "shrink":
279
- bgmv_shrink(
299
+ torch.ops.vllm.bgmv_shrink(
300
inputs_tensor,
301
lora_weights,
302
our_out_tensor,
303
indices,
304
scaling,
305
)
306
+
307
+ bgmv_shrink(
308
+ inputs_tensor,
309
+ lora_weights,
310
+ ref_out_tensor,
311
+ indices,
312
+ scaling,
313
+ )
314
+
315
else:
287
- bgmv_expand(
316
+ torch.ops.vllm.bgmv_expand(
317
inputs_tensor,
318
lora_weights,
319
our_out_tensor,
320
indices,
321
add_inputs=True,
322
)
294
- ref_torch_groupgemm(
295
- ref_out_tensor,
296
- inputs_tensor,
297
- lora_weights,
298
- lora_indices_tensor,
299
- seq_len_tensor,
300
- batches,
301
- scaling if op_type == "shrink" else 1.0,
302
- op_type,
303
- )
323
+ bgmv_expand(
324
+ inputs_tensor,
325
+ lora_weights,
326
+ ref_out_tensor,
327
+ indices,
328
+ add_inputs=True,
329
+ )
330
+
331
if op_type == "shrink":
332
ref_out_tensor = ref_out_tensor.to(torch.float32)
333
assert_close(our_out_tensor, ref_out_tensor)
@@ -313,7 +340,7 @@ def test_punica_bgmv(
340
@pytest.mark.parametrize("nslices", [2, 3])
341
@pytest.mark.parametrize("dtype", DTYPES)
342
@pytest.mark.parametrize("seed", SEED)
316
-@pytest.mark.parametrize("device", CUDA_DEVICES)
343
+@pytest.mark.parametrize("device", DEVICES)
344
def test_punica_bgmv_expand_nslices(
345
batches: int,
346
num_loras: int,
@@ -350,7 +377,7 @@ def test_punica_bgmv_expand_nslices(
377
slice_offset = 0
378
for index in range(nslices):
379
lora_weights = lora_weights_lst[index]
353
- bgmv_expand_slice(
380
+ torch.ops.vllm.bgmv_expand_slice(
381
inputs_tensor,
382
lora_weights,
383
our_outputs,
@@ -359,15 +386,14 @@ def test_punica_bgmv_expand_nslices(
386
slice_size=hidden_size,
387
add_inputs=True,
388
)
362
- ref_torch_groupgemm(
363
- ref_outputs[:, slice_offset:slice_offset + hidden_size],
389
+ bgmv_expand_slice(
390
inputs_tensor,
391
lora_weights,
366
- lora_indices_tensor,
367
- seq_len_tensor,
368
- batches,
369
- 1.0,
370
- op_type="expand",
392
+ ref_outputs,
393
+ indices,
394
+ slice_offset,
395
+ slice_size=hidden_size,
396
+ add_inputs=True,
397
)
398
399
slice_offset += hidden_size
tests/lora/test_punica_ops_variation.py
renamed
+75
-59
@@ -9,19 +9,18 @@ import pytest
9
import torch
10
11
# Enable custom op register
12
-import vllm.lora.ops.bgmv_expand
13
-import vllm.lora.ops.bgmv_expand_slice
14
-import vllm.lora.ops.bgmv_shrink
15
-import vllm.lora.ops.sgmv_expand
16
-import vllm.lora.ops.sgmv_shrink # noqa: F401
17
-from vllm.lora.ops.utils import _LORA_A_PTR_DICT, _LORA_B_PTR_DICT
12
+import vllm.lora.ops.triton_ops # noqa: F401
13
+from vllm.lora.ops.torch_ops import (bgmv_expand, bgmv_expand_slice,
14
+ bgmv_shrink, sgmv_expand,
15
+ sgmv_expand_slice, sgmv_shrink)
16
+from vllm.lora.ops.triton_ops.utils import _LORA_A_PTR_DICT, _LORA_B_PTR_DICT
17
from vllm.platforms import current_platform
18
19
from .utils import (assert_close, generate_data,
20
generate_data_for_expand_nslices,
22
- generate_data_for_nslices, ref_torch_groupgemm)
21
+ generate_data_for_nslices)
22
24
-HIDDEN_SIZES = [4097]
23
+HIDDEN_SIZES = [2049]
24
25
BATCHES = [1, 4, 16, 32]
26
NUM_LORA = [1, 8, 32, 128]
@@ -29,15 +28,7 @@ DTYPES = [torch.float16, torch.bfloat16]
28
MAX_RANKS = [1, 4, 8, 16, 32, 64, 128, 256]
29
SCALES = [0.5]
30
SEED = [0]
32
-CUDA_DEVICES = [f"cuda:{0}"]
33
-
34
-# Unlike test_punica_sizes.py, we directly utilize custom op for
35
-# testing, which verifies the correct registration of these ops.
36
-bgmv_expand = torch.ops.vllm.bgmv_expand
37
-bgmv_expand_slice = torch.ops.vllm.bgmv_expand_slice
38
-bgmv_shrink = torch.ops.vllm.bgmv_shrink
39
-sgmv_expand = torch.ops.vllm.sgmv_expand
40
-sgmv_shrink = torch.ops.vllm.sgmv_shrink
31
+DEVICES = [f"cuda:{0}"]
32
33
_dict_lock = Lock()
34
@@ -51,7 +42,7 @@ _dict_lock = Lock()
42
@pytest.mark.parametrize("dtype", DTYPES)
43
@pytest.mark.parametrize("op_type", ["shrink", "expand"])
44
@pytest.mark.parametrize("seed", SEED)
54
-@pytest.mark.parametrize("device", CUDA_DEVICES)
45
+@pytest.mark.parametrize("device", DEVICES)
46
def test_punica_sgmv(
47
batches: int,
48
num_loras: int,
@@ -98,7 +89,7 @@ def test_punica_sgmv(
89
# Preventing cache error pointer.
90
with _dict_lock:
91
_LORA_A_PTR_DICT.clear()
101
- sgmv_shrink(
92
+ torch.ops.vllm.sgmv_shrink(
93
inputs_tensor,
94
lora_weights_lst,
95
our_out_tensor,
@@ -111,20 +102,23 @@ def test_punica_sgmv(
102
scaling,
103
)
104
for index in range(nslices):
114
- ref_torch_groupgemm(
115
- ref_out_tensor[index],
105
+ sgmv_shrink(
106
inputs_tensor,
107
lora_weights_lst[index],
118
- lora_indices_tensor,
108
+ ref_out_tensor[index],
109
+ b_seq_start_loc,
110
seq_len_tensor,
111
+ lora_indices_tensor,
112
batches,
113
+ max_seq_length,
114
+ token_nums,
115
scaling,
122
- op_type,
116
)
117
+
118
else:
119
with _dict_lock:
120
_LORA_B_PTR_DICT.clear()
127
- sgmv_expand(
121
+ torch.ops.vllm.sgmv_expand(
122
inputs_tensor,
123
lora_weights_lst,
124
our_out_tensor,
@@ -137,21 +131,39 @@ def test_punica_sgmv(
131
offset_start=0,
132
add_inputs=True,
133
)
140
-
134
slice_offset = 0
142
- for index in range(nslices):
143
- lora_weights = lora_weights_lst[index]
144
- ref_torch_groupgemm(
145
- ref_out_tensor[:, slice_offset:slice_offset + hidden_size],
146
- inputs_tensor[index],
147
- lora_weights,
148
- lora_indices_tensor,
135
+ if nslices == 1:
136
+ # Verify the torch's sgmv_expand op
137
+ sgmv_expand(
138
+ inputs_tensor[0],
139
+ lora_weights_lst[0],
140
+ ref_out_tensor,
141
+ b_seq_start_loc,
142
seq_len_tensor,
143
+ lora_indices_tensor,
144
batches,
151
- 1.0,
152
- op_type,
145
+ max_seq_length,
146
+ token_nums,
147
+ add_inputs=True,
148
)
154
- slice_offset += hidden_size
149
+ else:
150
+ for index in range(nslices):
151
+ lora_weights = lora_weights_lst[index]
152
+ sgmv_expand_slice(
153
+ inputs_tensor[index],
154
+ lora_weights,
155
+ ref_out_tensor,
156
+ b_seq_start_loc,
157
+ seq_len_tensor,
158
+ lora_indices_tensor,
159
+ batches,
160
+ max_seq_length,
161
+ token_nums,
162
+ slice_offset,
163
+ hidden_size,
164
+ add_inputs=True,
165
+ )
166
+ slice_offset += hidden_size
167
168
assert_close(our_out_tensor, ref_out_tensor)
169
@@ -164,7 +176,7 @@ def test_punica_sgmv(
176
@pytest.mark.parametrize("dtype", DTYPES)
177
@pytest.mark.parametrize("op_type", ["shrink", "expand"])
178
@pytest.mark.parametrize("seed", SEED)
167
-@pytest.mark.parametrize("device", CUDA_DEVICES)
179
+@pytest.mark.parametrize("device", DEVICES)
180
def test_punica_bgmv(
181
batches: int,
182
num_loras: int,
@@ -176,7 +188,6 @@ def test_punica_bgmv(
188
seed: int,
189
device: str,
190
):
179
-
191
torch.set_default_device(device)
192
current_platform.seed_everything(seed)
193
@@ -201,32 +212,38 @@ def test_punica_bgmv(
212
device,
213
)
214
if op_type == "shrink":
204
- bgmv_shrink(
215
+ torch.ops.vllm.bgmv_shrink(
216
inputs_tensor,
217
lora_weights,
218
our_out_tensor,
219
indices,
220
scaling,
221
)
211
- else:
222
213
- bgmv_expand(
223
+ bgmv_shrink(
224
+ inputs_tensor,
225
+ lora_weights,
226
+ ref_out_tensor,
227
+ indices,
228
+ scaling,
229
+ )
230
+
231
+ else:
232
+ torch.ops.vllm.bgmv_expand(
233
inputs_tensor,
234
lora_weights,
235
our_out_tensor,
236
indices,
237
add_inputs=True,
238
)
220
- ref_torch_groupgemm(
221
- ref_out_tensor,
222
- inputs_tensor,
223
- lora_weights,
224
- lora_indices_tensor,
225
- seq_len_tensor,
226
- batches,
227
- scaling if op_type == "shrink" else 1.0,
228
- op_type,
229
- )
239
+ bgmv_expand(
240
+ inputs_tensor,
241
+ lora_weights,
242
+ ref_out_tensor,
243
+ indices,
244
+ add_inputs=True,
245
+ )
246
+
247
if op_type == "shrink":
248
ref_out_tensor = ref_out_tensor.to(torch.float32)
249
assert_close(our_out_tensor, ref_out_tensor)
@@ -239,7 +256,7 @@ def test_punica_bgmv(
256
@pytest.mark.parametrize("nslices", [2, 3])
257
@pytest.mark.parametrize("dtype", DTYPES)
258
@pytest.mark.parametrize("seed", SEED)
242
-@pytest.mark.parametrize("device", CUDA_DEVICES)
259
+@pytest.mark.parametrize("device", DEVICES)
260
def test_punica_bgmv_expand_nslices(
261
batches: int,
262
num_loras: int,
@@ -276,7 +293,7 @@ def test_punica_bgmv_expand_nslices(
293
slice_offset = 0
294
for index in range(nslices):
295
lora_weights = lora_weights_lst[index]
279
- bgmv_expand_slice(
296
+ torch.ops.vllm.bgmv_expand_slice(
297
inputs_tensor,
298
lora_weights,
299
our_outputs,
@@ -285,15 +302,14 @@ def test_punica_bgmv_expand_nslices(
302
slice_size=hidden_size,
303
add_inputs=True,
304
)
288
- ref_torch_groupgemm(
289
- ref_outputs[:, slice_offset:slice_offset + hidden_size],
305
+ bgmv_expand_slice(
306
inputs_tensor,
307
lora_weights,
292
- lora_indices_tensor,
293
- seq_len_tensor,
294
- batches,
295
- 1.0,
296
- op_type="expand",
308
+ ref_outputs,
309
+ indices,
310
+ slice_offset,
311
+ slice_size=hidden_size,
312
+ add_inputs=True,
313
)
314
315
slice_offset += hidden_size
tests/lora/test_quant_model.py
+2
-1
@@ -72,7 +72,8 @@ def do_sample(llm: vllm.LLM,
72
@pytest.mark.parametrize("tp_size", [1])
73
def test_quant_model_lora(tinyllama_lora_files, num_gpus_available, model,
74
tp_size):
75
- if num_gpus_available < tp_size:
75
+ if num_gpus_available < tp_size and \
76
+ tp_size > 1 and current_platform.is_cuda_alike():
77
pytest.skip(f"Not enough GPUs for tensor parallelism {tp_size}")
78
79
llm = vllm.LLM(
tests/lora/utils.py
-27
@@ -104,33 +104,6 @@ def assert_close(a, b):
104
torch.testing.assert_close(a, b, rtol=rtol, atol=atol)
105
106
107
-def ref_torch_groupgemm(
108
- out_tensor,
109
- inputs,
110
- lora_weights,
111
- lora_indices_tensor,
112
- seq_len_tensor,
113
- batches,
114
- scaling,
115
- op_type,
116
-) -> torch.Tensor:
117
- out_list = []
118
- current_offset = 0
119
- for lora_index, b_length in zip(range(batches), seq_len_tensor):
120
- input_weight = inputs[current_offset:b_length + current_offset, :]
121
- current_offset += b_length
122
- lora_weight = lora_weights[lora_indices_tensor[lora_index]]
123
- result = torch.nn.functional.linear(input_weight, lora_weight)
124
- result *= scaling
125
- out_list.append(result)
126
- cat_result = torch.cat(out_list, dim=0)
127
- if op_type == "expand":
128
- out_tensor += cat_result
129
- else:
130
- out_tensor.copy_(cat_result)
131
- return
132
-
133
-
107
def generate_data(
108
batches,
109
hidden_size,
vllm/executor/cpu_executor.py
-3
@@ -22,9 +22,6 @@ class CPUExecutor(ExecutorBase):
22
23
def _init_executor(self) -> None:
24
assert self.device_config.device_type == "cpu"
25
- # Reminder: Please update docs/source/features/compatibility_matrix.md
26
- # If the feature combo become valid
27
- assert self.lora_config is None, "cpu backend doesn't support LoRA"
25
26
#
27
# Environment variables for CPU executor
vllm/lora/ops/torch_ops/__init__.py
new
+13
@@ -0,0 +1,13 @@
1
+from vllm.lora.ops.torch_ops.lora_ops import bgmv_expand # noqa: F401
2
+from vllm.lora.ops.torch_ops.lora_ops import (bgmv_expand_slice, bgmv_shrink,
3
+ sgmv_expand, sgmv_expand_slice,
4
+ sgmv_shrink)
5
+
6
+__all__ = [
7
+ "bgmv_expand",
8
+ "bgmv_expand_slice",
9
+ "bgmv_shrink",
10
+ "sgmv_expand",
11
+ "sgmv_expand_slice",
12
+ "sgmv_shrink",
13
+]
vllm/lora/ops/torch_ops/lora_ops.py
new
+113
@@ -0,0 +1,113 @@
1
+import torch
2
+
3
+
4
+def sgmv_expand(inputs: torch.Tensor,
5
+ lora_b_weights: torch.Tensor,
6
+ output_tensor: torch.Tensor,
7
+ b_seq_start_loc: torch.Tensor,
8
+ seq_len_tensor: torch.Tensor,
9
+ lora_indices_tensor: torch.Tensor,
10
+ batches: int,
11
+ max_seq_length: int,
12
+ token_nums: int,
13
+ add_inputs: bool = False):
14
+ exploded_indices = torch.repeat_interleave(lora_indices_tensor,
15
+ seq_len_tensor)
16
+
17
+ bgmv_expand(inputs, lora_b_weights, output_tensor, exploded_indices,
18
+ add_inputs)
19
+
20
+
21
+def bgmv_expand(inputs: torch.Tensor,
22
+ lora_b_weights: torch.Tensor,
23
+ output_tensor: torch.Tensor,
24
+ lora_indices_tensor: torch.Tensor,
25
+ add_inputs: bool = True):
26
+ selected_loras = lora_b_weights[lora_indices_tensor].to(
27
+ dtype=output_tensor.dtype)
28
+ if len(selected_loras.shape) == 4:
29
+ selected_loras = selected_loras.squeeze(dim=1)
30
+ inputs = inputs.to(dtype=output_tensor.dtype)
31
+ outputs = torch.einsum("bi, boi -> bo", inputs, selected_loras)
32
+
33
+ limit = output_tensor.shape[0]
34
+ if outputs.shape[0] == 1 and output_tensor.shape[0] != 1:
35
+ limit = 1
36
+
37
+ if add_inputs:
38
+ output_tensor[:, :outputs.shape[1]] += outputs[:limit, :]
39
+ else:
40
+ output_tensor[:, :outputs.shape[1]] = outputs[:limit, :]
41
+
42
+
43
+def sgmv_shrink(
44
+ inputs: torch.Tensor,
45
+ lora_a_weights: torch.Tensor,
46
+ output_tensor: torch.Tensor,
47
+ b_seq_start_loc: torch.Tensor,
48
+ seq_len_tensor: torch.Tensor,
49
+ lora_indices_tensor: torch.Tensor,
50
+ batches: int,
51
+ max_seq_length: int,
52
+ token_nums: int,
53
+ scaling: float,
54
+):
55
+ exploded_indices = torch.repeat_interleave(lora_indices_tensor,
56
+ seq_len_tensor)
57
+
58
+ bgmv_shrink(inputs, lora_a_weights, output_tensor, exploded_indices,
59
+ scaling)
60
+
61
+
62
+def bgmv_shrink(inputs: torch.Tensor,
63
+ lora_b_weights: torch.Tensor,
64
+ output_tensor: torch.Tensor,
65
+ lora_indices_tensor: torch.Tensor,
66
+ scaling: float = 1.0):
67
+ selected_loras = lora_b_weights[lora_indices_tensor].to(
68
+ dtype=output_tensor.dtype)
69
+ if len(selected_loras.shape) == 4:
70
+ selected_loras = selected_loras.squeeze(dim=1)
71
+ inputs = inputs.to(dtype=output_tensor.dtype)
72
+ outputs = torch.einsum("bi, boi -> bo", inputs, selected_loras)
73
+
74
+ output_tensor[:, :outputs.shape[1]] = scaling * outputs[:]
75
+
76
+
77
+def sgmv_expand_slice(inputs: torch.Tensor,
78
+ lora_b_weights: torch.Tensor,
79
+ output_tensor: torch.Tensor,
80
+ b_seq_start_loc: torch.Tensor,
81
+ seq_len_tensor: torch.Tensor,
82
+ lora_indices_tensor: torch.Tensor,
83
+ batches: int,
84
+ max_seq_length: int,
85
+ token_nums: int,
86
+ slice_offset: int,
87
+ slice_size: int,
88
+ add_inputs: bool = False):
89
+ exploded_indices = torch.repeat_interleave(lora_indices_tensor,
90
+ seq_len_tensor)
91
+
92
+ bgmv_expand_slice(inputs, lora_b_weights, output_tensor, exploded_indices,
93
+ slice_offset, slice_size, add_inputs)
94
+
95
+
96
+def bgmv_expand_slice(inputs: torch.Tensor,
97
+ lora_b_weights: torch.Tensor,
98
+ output_tensor: torch.Tensor,
99
+ lora_indices_tensor: torch.Tensor,
100
+ slice_offset: int,
101
+ slice_size: int,
102
+ add_inputs: bool = True):
103
+ selected_loras = lora_b_weights[lora_indices_tensor].to(
104
+ dtype=output_tensor.dtype)
105
+ inputs = inputs.to(dtype=output_tensor.dtype)
106
+ if len(selected_loras.shape) == 4:
107
+ selected_loras = selected_loras.squeeze(dim=1)
108
+ outputs = torch.einsum("bi, boi -> bo", inputs, selected_loras)
109
+
110
+ if add_inputs:
111
+ output_tensor[:, slice_offset:slice_offset + slice_size] += outputs[:]
112
+ else:
113
+ output_tensor[:, slice_offset:slice_offset + slice_size] = outputs[:]
vllm/lora/ops/triton_ops/__init__.py
new
+13
@@ -0,0 +1,13 @@
1
+from vllm.lora.ops.triton_ops.bgmv_expand import bgmv_expand
2
+from vllm.lora.ops.triton_ops.bgmv_expand_slice import bgmv_expand_slice
3
+from vllm.lora.ops.triton_ops.bgmv_shrink import bgmv_shrink
4
+from vllm.lora.ops.triton_ops.sgmv_expand import sgmv_expand
5
+from vllm.lora.ops.triton_ops.sgmv_shrink import sgmv_shrink # noqa: F401
6
+
7
+__all__ = [
8
+ "bgmv_expand",
9
+ "bgmv_expand_slice",
10
+ "bgmv_shrink",
11
+ "sgmv_expand",
12
+ "sgmv_shrink",
13
+]
vllm/lora/ops/triton_ops/bgmv_expand.py
renamed
vllm/lora/ops/triton_ops/bgmv_expand_slice.py
renamed
vllm/lora/ops/triton_ops/bgmv_shrink.py
renamed
vllm/lora/ops/triton_ops/sgmv_expand.py
renamed
vllm/lora/ops/triton_ops/sgmv_shrink.py
renamed
vllm/lora/ops/triton_ops/utils.py
renamed
vllm/lora/punica_wrapper/punica_cpu.py
new
+346
@@ -0,0 +1,346 @@
1
+from typing import Callable, Optional, Tuple, Union
2
+
3
+import torch
4
+
5
+from vllm.lora.ops.torch_ops import (bgmv_expand, bgmv_expand_slice,
6
+ bgmv_shrink, sgmv_expand,
7
+ sgmv_expand_slice, sgmv_shrink)
8
+
9
+from .punica_base import PunicaWrapperBase
10
+
11
+
12
+# The platforms that are compatible with the PyTorch-native implementation can
13
+# inherit this class
14
+class PunicaWrapperCPU(PunicaWrapperBase):
15
+ """
16
+ PunicaWrapperCPU is designed to manage and provide metadata for the punica
17
+ kernel. The main function is to maintain the state information for
18
+ Multi-LoRA, and to provide the interface for the pytorch punica ops.
19
+ """
20
+
21
+ def __init__(self, max_num_batched_tokens: int, max_batches: int,
22
+ device: Union[torch.device, str], **kwargs):
23
+ PunicaWrapperBase.__init__(self, max_num_batched_tokens, max_batches,
24
+ device)
25
+
26
+ def _shrink_prefill(
27
+ self,
28
+ y: torch.Tensor,
29
+ x: torch.Tensor,
30
+ w_t_all: torch.Tensor,
31
+ scale: float,
32
+ ):
33
+ #No LoRA request, so return directly
34
+ if self.no_lora:
35
+ return
36
+ sgmv_shrink(
37
+ x,
38
+ w_t_all,
39
+ y,
40
+ *self.prefill_metadata,
41
+ scale,
42
+ )
43
+
44
+ def _shrink_decode(
45
+ self,
46
+ y: torch.Tensor,
47
+ x: torch.Tensor,
48
+ w_t_all: torch.Tensor,
49
+ scale: float,
50
+ ):
51
+ bgmv_shrink(x, w_t_all, y, self.token_lora_indices, scale)
52
+
53
+ def _expand_prefill(
54
+ self,
55
+ y: torch.Tensor,
56
+ x: torch.Tensor,
57
+ w_t_all: torch.Tensor,
58
+ add_inputs: bool,
59
+ ):
60
+ #No LoRA request, so return directly
61
+ if self.no_lora:
62
+ return
63
+ sgmv_expand(
64
+ x,
65
+ w_t_all,
66
+ y,
67
+ *self.prefill_metadata,
68
+ add_inputs,
69
+ )
70
+
71
+ def _expand_decode(
72
+ self,
73
+ y: torch.Tensor,
74
+ x: torch.Tensor,
75
+ w_t_all: torch.Tensor,
76
+ add_inputs: bool,
77
+ ):
78
+ bgmv_expand(x, w_t_all, y, self.token_lora_indices, add_inputs)
79
+
80
+ def _expand_slice_prefill(
81
+ self,
82
+ y: torch.Tensor,
83
+ x: torch.Tensor,
84
+ w_t_all: torch.Tensor,
85
+ y_offset: int,
86
+ y_slice_size: int,
87
+ add_inputs: bool,
88
+ ):
89
+ #No LoRA request, so return directly
90
+ if self.no_lora:
91
+ return
92
+ sgmv_expand_slice(
93
+ x,
94
+ w_t_all,
95
+ y,
96
+ *self.prefill_metadata,
97
+ y_offset,
98
+ y_slice_size,
99
+ add_inputs,
100
+ )
101
+
102
+ def _expand_slice_decode(
103
+ self,
104
+ y: torch.Tensor,
105
+ x: torch.Tensor,
106
+ w_t_all: torch.Tensor,
107
+ y_offset: int,
108
+ y_slice_size: int,
109
+ add_inputs: bool,
110
+ ):
111
+ bgmv_expand_slice(x, w_t_all, y, self.token_lora_indices, y_offset,
112
+ y_slice_size, add_inputs)
113
+
114
+ def _apply_expand(
115
+ self,
116
+ y: torch.Tensor,
117
+ x: torch.Tensor,
118
+ w_t_all: torch.Tensor,
119
+ y_offset: int,
120
+ y_slice_size: int,
121
+ add_inputs: bool = True,
122
+ ):
123
+ """
124
+ Perform the ` y[:,y_offset:y_offset+y_slice_size]+=x@w_t_all`
125
+ computation, which is suitable for the
126
+ GEMM of lora'b.
127
+ """
128
+
129
+ expand_slice_fun: Callable = (self._expand_slice_prefill
130
+ if self.is_prefill else
131
+ self._expand_slice_decode)
132
+ expand_slice_fun(y, x, w_t_all, y_offset, y_slice_size, add_inputs)
133
+
134
+ def _apply_shrink(self, y: torch.Tensor, x: torch.Tensor,
135
+ w_t_all: torch.Tensor, scale: float):
136
+ """
137
+ Perform the ` y+=x@w_t_all` computation, which is suitable for the
138
+ GEMM of lora'a.
139
+ When `is_prefill is` true, it indicates that it is currently the
140
+ prefill stage, and the `_shrink_prefill` function should be called.
141
+ Otherwise, it is the decode stage, and the _shrink_decode function
142
+ should be called.
143
+ """
144
+ y_org = y
145
+ y = y.view(-1, y.shape[-1])
146
+ shrink_fun: Callable = (self._shrink_prefill
147
+ if self.is_prefill else self._shrink_decode)
148
+ shrink_fun(y, x, w_t_all, scale)
149
+ y = y.view_as(y_org)
150
+
151
+ def add_shrink(self, y: Union[Tuple[torch.Tensor, ...], torch.Tensor],
152
+ x: torch.Tensor, lora_a_stacked: Tuple[torch.Tensor, ...],
153
+ scale: float, **kwargs):
154
+ """
155
+ Performs GEMM for multiple slices of lora_a.
156
+ When `is_prefill is` true, it indicates that it is currently the
157
+ prefill stage, and the `_shrink_prefill` function should be called.
158
+ Otherwise, it is the decode stage, and the _shrink_decode function
159
+ should be called.
160
+
161
+ Semantics:
162
+ for i in range(len(lora_a_stacked)):
163
+ y[i] += (x @ lora_a_stacked[i]) * scale
164
+
165
+ Args:
166
+ y (Union[Tuple[torch.Tensor, ...], torch.Tensor]): Output tensors
167
+ x (torch.Tensor): Input tensor
168
+ lora_a_stacked (Tuple[torch.Tensor, ...]): lora_a's weights
169
+ scale (float): Scaling factor for the operation
170
+ """
171
+
172
+ x = x.view(-1, x.shape[-1])
173
+ # TODO fuse these kernels
174
+ for slice_idx in range(len(lora_a_stacked)):
175
+ self._apply_shrink(y[slice_idx], x, lora_a_stacked[slice_idx],
176
+ scale)
177
+
178
+ def add_expand(self,
179
+ y: torch.Tensor,
180
+ x: Union[Tuple[torch.Tensor, ...], torch.Tensor],
181
+ lora_b_stacked: Tuple[torch.Tensor, ...],
182
+ lora_bias_stacked: Optional[Tuple[torch.Tensor, ...]],
183
+ output_slices: Tuple[int, ...],
184
+ offset_start: int = 0,
185
+ add_inputs=True,
186
+ **kwargs) -> None:
187
+ """
188
+ Performs GEMM and bias addition for multiple slices of lora_b.
189
+
190
+ Semantics:
191
+ for i in range(len(lora_b_stacked)):
192
+ slice = output_slices[i]
193
+ y[:, offset:offset+slice] += x[i] @ lora_b_stacked[i] +
194
+ lora_bias_stacked[i]
195
+ offset += slice
196
+
197
+ Args:
198
+ y (torch.Tensor): Output tensor.
199
+ x (Union[Tuple[torch.Tensor, ...], torch.Tensor]): Input tensors
200
+ lora_b_stacked (Tuple[torch.Tensor, ...]): lora_b's weight
201
+ lora_bias_stacked (Optional[Tuple[torch.Tensor, ...]]):
202
+ bias's weight
203
+ output_slices (Tuple[int, ...]): Every slice's size
204
+ add_inputs (bool): Defaults to True.
205
+ """
206
+ y_org = y
207
+ y = y.view(-1, y.shape[-1])
208
+ offset_left = offset_start
209
+ if lora_bias_stacked is not None:
210
+ self._apply_bias(self.token_lora_indices, y, output_slices,
211
+ lora_bias_stacked)
212
+ for slice_idx in range(len(lora_b_stacked)):
213
+ self._apply_expand(
214
+ y,
215
+ x[slice_idx],
216
+ lora_b_stacked[slice_idx],
217
+ offset_left,
218
+ output_slices[slice_idx],
219
+ add_inputs=add_inputs,
220
+ )
221
+ offset_left += output_slices[slice_idx]
222
+ y = y.view_as(y_org)
223
+
224
+ def add_lora_embedding(self,
225
+ y: torch.Tensor,
226
+ x: torch.Tensor,
227
+ lora_b_stacked: torch.Tensor,
228
+ add_inputs: bool = True,
229
+ **kwargs) -> None:
230
+ """
231
+ Applies lora specifically for VocabParallelEmbeddingWithLoRA.
232
+
233
+ Semantics:
234
+ y += x @ lora_b_stacked
235
+
236
+ Args:
237
+ y (torch.Tensor): Output tensor.
238
+ x (torch.Tensor): Input tensor.
239
+ lora_b_stacked (torch.Tensor): lora_b's weights.
240
+ add_inputs (bool): Default to True.
241
+ """
242
+
243
+ # Embedding layer only need expand op
244
+ expand_fun: Callable = (self._expand_prefill
245
+ if self.is_prefill else self._expand_decode)
246
+ expand_fun(y, x, lora_b_stacked, add_inputs)
247
+
248
+ def add_lora_linear(self,
249
+ y: torch.Tensor,
250
+ x: torch.Tensor,
251
+ lora_a_stacked: Tuple[torch.Tensor, ...],
252
+ lora_b_stacked: Tuple[torch.Tensor, ...],
253
+ lora_bias_stacked: Optional[Tuple[torch.Tensor, ...]],
254
+ scale: float,
255
+ output_slices: Tuple[int, ...],
256
+ *,
257
+ buffer: Optional[Tuple[torch.Tensor, ...]] = None,
258
+ **kwargs) -> None:
259
+ """
260
+ Applicable to linear-related lora.
261
+
262
+ Semantics:
263
+ for i in range(len(lora_a_stacked)):
264
+ y[i] += (
265
+ x[i].unsqueeze(0)
266
+ @ lora_a_stacked[indices[i], layer_idx, :, :]
267
+ @ lora_b_stacked[indices[i], layer_idx, :, :]
268
+ * scale
269
+ ).squeeze(0)+lora_bias_stacked[i]
270
+
271
+ Args:
272
+ y (torch.Tensor): Output tensor. Will be changed in-place.
273
+ x (torch.Tensor): Input tensor
274
+ lora_a_stacked (Tuple[torch.Tensor, ...]): lora_a's weight.
275
+ lora_b_stacked (Tuple[torch.Tensor, ...]): lora_b's weight.
276
+ lora_bias_stacked (Optional[Tuple[torch.Tensor, ...]]): lora's bias.
277
+ scale (float): Scaling factor.
278
+ output_slices (Tuple[int, ...]): Every slice's size.
279
+ buffer (Optional[Tuple[torch.Tensor, ...]]): Defaults to None.
280
+ """
281
+
282
+ assert len(lora_a_stacked) == len(lora_b_stacked) == len(output_slices)
283
+ if lora_bias_stacked is not None:
284
+ assert len(lora_bias_stacked) == len(output_slices)
285
+ y = self._apply_bias(self.token_lora_indices, y, output_slices,
286
+ lora_bias_stacked)
287
+
288
+ if buffer is None:
289
+ r = lora_b_stacked[0].size(-1)
290
+ # We set the buffer to be float32 by default, consistent with the
291
+ # triton op
292
+ buffer = tuple(
293
+ torch.zeros(
294
+ (x.size(0), r), dtype=torch.float32, device=x.device)
295
+ for _ in range(len(output_slices)))
296
+ self.add_shrink(buffer, x, lora_a_stacked, scale, **kwargs)
297
+ self.add_expand(y,
298
+ buffer,
299
+ lora_b_stacked,
300
+ None,
301
+ output_slices,
302
+ add_inputs=True,
303
+ **kwargs)
304
+
305
+ def add_lora_logits(self,
306
+ y: torch.Tensor,
307
+ x: torch.Tensor,
308
+ lora_a_stacked: torch.Tensor,
309
+ lora_b_stacked: torch.Tensor,
310
+ scale,
311
+ *,
312
+ buffer: Optional[torch.Tensor] = None,
313
+ **kwargs) -> None:
314
+ """
315
+ Applies lora specifically for LogitsProcessorWithLoRA.
316
+
317
+ Semantics:
318
+ buffer = (x @ lora_a_stacked) * scale
319
+ y += buffer @ lora_b_stacked
320
+
321
+ Args:
322
+ y (torch.Tensor): Output tensor.
323
+ x (torch.Tensor): Input tensor.
324
+ lora_a_stacked (torch.Tensor): lora_a's weights.
325
+ lora_b_stacked (torch.Tensor):lora_b's weights.
326
+ scale (float): Scaling factor.
327
+ buffer (Optional[torch.Tensor]):Default to None.
328
+ """
329
+ y_org = y
330
+ y = y.view(-1, y.shape[-1])
331
+ x = x.view(-1, x.shape[-1])
332
+ r = lora_b_stacked.size(-1)
333
+ if buffer is None:
334
+ # We set the buffer to be float32 by default, consistent with the
335
+ # triton op
336
+ buffer = torch.zeros((x.size(0), r),
337
+ dtype=torch.float32,
338
+ device=x.device)
339
+ # LogitsProcessorWithLoRA always using bgmv.
340
+ bgmv_shrink(x, lora_a_stacked, buffer, self.sampler_indices, scale)
341
+ bgmv_expand(buffer,
342
+ lora_b_stacked,
343
+ y,
344
+ self.sampler_indices,
345
+ add_inputs=True)
346
+ y = y.view_as(y_org)
vllm/lora/punica_wrapper/punica_gpu.py
+5
-5
@@ -12,11 +12,11 @@ import torch
12
from vllm.triton_utils import HAS_TRITON
13
14
if HAS_TRITON:
15
- from vllm.lora.ops.bgmv_expand import bgmv_expand
16
- from vllm.lora.ops.bgmv_expand_slice import bgmv_expand_slice
17
- from vllm.lora.ops.bgmv_shrink import bgmv_shrink
18
- from vllm.lora.ops.sgmv_expand import sgmv_expand
19
- from vllm.lora.ops.sgmv_shrink import sgmv_shrink
15
+ from vllm.lora.ops.triton_ops import bgmv_expand
16
+ from vllm.lora.ops.triton_ops import bgmv_expand_slice
17
+ from vllm.lora.ops.triton_ops import bgmv_shrink
18
+ from vllm.lora.ops.triton_ops import sgmv_expand
19
+ from vllm.lora.ops.triton_ops import sgmv_shrink
20
21
from .punica_base import PunicaWrapperBase
22
vllm/lora/punica_wrapper/punica_selector.py
+5
@@ -12,6 +12,11 @@ def get_punica_wrapper(*args, **kwargs) -> PunicaWrapperBase:
12
from vllm.lora.punica_wrapper.punica_gpu import PunicaWrapperGPU
13
logger.info_once("Using PunicaWrapperGPU.")
14
return PunicaWrapperGPU(*args, **kwargs)
15
+ elif current_platform.is_cpu():
16
+ # Lazy import to avoid ImportError
17
+ from vllm.lora.punica_wrapper.punica_cpu import PunicaWrapperCPU
18
+ logger.info_once("Using PunicaWrapperCPU.")
19
+ return PunicaWrapperCPU(*args, **kwargs)
20
elif current_platform.is_hpu():
21
# Lazy import to avoid ImportError
22
from vllm.lora.punica_wrapper.punica_hpu import PunicaWrapperHPU
vllm/worker/cpu_model_runner.py
+122
-11
@@ -2,8 +2,8 @@ import dataclasses
2
import weakref
3
from collections import defaultdict
4
from dataclasses import dataclass
5
-from typing import (TYPE_CHECKING, Any, Dict, List, Optional, Type, TypeVar,
6
- Union)
5
+from typing import (TYPE_CHECKING, Any, Dict, List, Optional, Set, Type,
6
+ TypeVar, Union)
7
8
import torch
9
from torch import nn
@@ -12,10 +12,14 @@ from vllm.attention import AttentionMetadata, get_attn_backend
12
from vllm.config import VllmConfig
13
from vllm.forward_context import set_forward_context
14
from vllm.logger import init_logger
15
+from vllm.lora.layers import LoRAMapping
16
+from vllm.lora.request import LoRARequest
17
+from vllm.lora.worker_manager import LRUCacheWorkerLoRAManager
18
from vllm.model_executor import SamplingMetadata
19
from vllm.model_executor.layers.rotary_embedding import MRotaryEmbedding
20
from vllm.model_executor.layers.sampler import SamplerOutput
21
from vllm.model_executor.model_loader import get_model
22
+from vllm.model_executor.models import supports_lora, supports_multimodal
23
from vllm.multimodal import (MULTIMODAL_REGISTRY, BatchedTensorInputs,
24
MultiModalKwargs, MultiModalPlaceholderMap)
25
from vllm.sequence import (IntermediateTensors, SequenceData,
@@ -49,6 +53,8 @@ class ModelInputForCPU(ModelRunnerInputBase):
53
virtual_engine: Optional[int] = None
54
seq_lens: Optional[List[int]] = None
55
query_lens: Optional[List[int]] = None
56
+ lora_mapping: Optional["LoRAMapping"] = None
57
+ lora_requests: Optional[Set[LoRARequest]] = None
58
59
def as_broadcastable_tensor_dict(
60
self) -> Dict[str, Union[int, torch.Tensor]]:
@@ -57,6 +63,8 @@ class ModelInputForCPU(ModelRunnerInputBase):
63
"input_positions": self.input_positions,
64
"token_type_ids": self.token_type_ids,
65
"multi_modal_kwargs": self.multi_modal_kwargs,
66
+ "lora_requests": self.lora_requests,
67
+ "lora_mapping": self.lora_mapping,
68
}
69
_add_attn_metadata_broadcastable_dict(tensor_dict, self.attn_metadata)
70
@@ -143,7 +151,11 @@ class ModelInputForCPUBuilder(ModelRunnerInputBuilderBase[ModelInputForCPU]):
151
or runner.cache_config.enable_prefix_caching)
152
self.model_input_cls = self.runner._model_input_cls
153
self.attn_backend = self.runner.attn_backend
154
+ self.sliding_window = self.runner.sliding_window
155
+ self.block_size = self.runner.block_size
156
+ self.device = self.runner.device
157
self.multi_modal_input_mapper = self.runner.multi_modal_input_mapper
158
+ self.enable_lora = self.runner.lora_config is not None
159
self.input_data = ModelInputForCPUBuilder.ModelInputData(
160
self.runner.model_config.uses_mrope)
161
self.att_metadata_builder = self.runner.attn_backend.get_builder_cls()(
@@ -183,15 +195,28 @@ class ModelInputForCPUBuilder(ModelRunnerInputBuilderBase[ModelInputForCPU]):
195
attn_metadata = self.att_metadata_builder.build(
196
input_data.seq_lens, input_data.query_lens, -1, -1)
197
186
- return self.model_input_cls(
187
- input_tokens=input_tokens,
188
- input_positions=input_positions,
189
- token_type_ids=token_type_ids,
190
- seq_lens=input_data.seq_lens,
191
- query_lens=input_data.query_lens,
192
- attn_metadata=attn_metadata,
193
- multi_modal_kwargs=multi_modal_kwargs,
194
- )
198
+ is_prompt = (self.seq_group_metadata_list[0].is_prompt
199
+ if self.seq_group_metadata_list else None)
200
+ # LoRA data.
201
+ lora_requests = set()
202
+ lora_mapping = None
203
+ if self.enable_lora:
204
+ lora_requests = set(seq.lora_request
205
+ for seq in self.seq_group_metadata_list
206
+ if seq.lora_request is not None)
207
+
208
+ lora_mapping = self._prepare_lora_input(
209
+ self.seq_group_metadata_list, is_prompt)
210
+
211
+ return self.model_input_cls(input_tokens=input_tokens,
212
+ input_positions=input_positions,
213
+ token_type_ids=token_type_ids,
214
+ seq_lens=input_data.seq_lens,
215
+ query_lens=input_data.query_lens,
216
+ attn_metadata=attn_metadata,
217
+ multi_modal_kwargs=multi_modal_kwargs,
218
+ lora_mapping=lora_mapping,
219
+ lora_requests=lora_requests)
220
221
def _build_input_data(self):
222
for seq_group_metadata in self.seq_group_metadata_list:
@@ -381,6 +406,24 @@ class ModelInputForCPUBuilder(ModelRunnerInputBuilderBase[ModelInputForCPU]):
406
self.input_data.multi_modal_placeholder_maps[modality].extend(
407
placeholder_map)
408
409
+ def _prepare_lora_input(
410
+ self, seq_group_metadata_list: List[SequenceGroupMetadata],
411
+ is_prefill: bool) -> LoRAMapping:
412
+ index_mapping = []
413
+ prompt_mapping = []
414
+ for seq in seq_group_metadata_list:
415
+ lora_id = seq.lora_int_id
416
+ query_len = seq.token_chunk_size
417
+
418
+ index_mapping += [lora_id] * query_len
419
+ prompt_mapping += [lora_id] * (
420
+ query_len if seq.sampling_params
421
+ and seq.sampling_params.prompt_logprobs is not None else 1)
422
+
423
+ return LoRAMapping(index_mapping=tuple(index_mapping),
424
+ prompt_mapping=tuple(prompt_mapping),
425
+ is_prefill=is_prefill)
426
+
427
428
class CPUModelRunnerBase(ModelRunnerBase[TModelInputForCPU]):
429
"""
@@ -431,10 +474,41 @@ class CPUModelRunnerBase(ModelRunnerBase[TModelInputForCPU]):
474
475
# Lazy initialization.
476
self.model: nn.Module # Set after init_Model
477
+ # Set after load_model.
478
+ self.lora_manager: Optional[LRUCacheWorkerLoRAManager] = None
479
480
def load_model(self) -> None:
481
self.model = get_model(vllm_config=self.vllm_config)
482
483
+ if self.lora_config:
484
+ assert supports_lora(
485
+ self.model
486
+ ), f"{self.model.__class__.__name__} does not support LoRA yet."
487
+
488
+ if supports_multimodal(self.model):
489
+ logger.warning("Regarding multimodal models, vLLM currently "
490
+ "only supports adding LoRA to language model.")
491
+
492
+ # It's necessary to distinguish between the max_position_embeddings
493
+ # of VLMs and LLMs.
494
+ if hasattr(self.model.config, "max_position_embeddings"):
495
+ max_pos_embeddings = self.model.config.max_position_embeddings
496
+ else:
497
+ max_pos_embeddings = (
498
+ self.model.config.text_config.max_position_embeddings)
499
+
500
+ self.lora_manager = LRUCacheWorkerLoRAManager(
501
+ self.scheduler_config.max_num_seqs,
502
+ self.scheduler_config.max_num_batched_tokens,
503
+ self.vocab_size,
504
+ self.lora_config,
505
+ self.device,
506
+ self.model.embedding_modules,
507
+ self.model.embedding_padding_modules,
508
+ max_position_embeddings=max_pos_embeddings,
509
+ )
510
+ self.model = self.lora_manager.create_lora_manager(self.model)
511
+
512
def _prepare_model_input_tensors(
513
self,
514
seq_group_metadata_list: List[SequenceGroupMetadata],
@@ -459,6 +533,37 @@ class CPUModelRunnerBase(ModelRunnerBase[TModelInputForCPU]):
533
def vocab_size(self) -> int:
534
return self.model_config.get_vocab_size()
535
536
+ def remove_all_loras(self):
537
+ if not self.lora_manager:
538
+ raise RuntimeError("LoRA is not enabled.")
539
+ self.lora_manager.remove_all_adapters()
540
+
541
+ def set_active_loras(self, lora_requests: Set[LoRARequest],
542
+ lora_mapping: LoRAMapping) -> None:
543
+ if not self.lora_manager:
544
+ raise RuntimeError("LoRA is not enabled.")
545
+ self.lora_manager.set_active_adapters(lora_requests, lora_mapping)
546
+
547
+ def add_lora(self, lora_request: LoRARequest) -> bool:
548
+ if not self.lora_manager:
549
+ raise RuntimeError("LoRA is not enabled.")
550
+ return self.lora_manager.add_adapter(lora_request)
551
+
552
+ def remove_lora(self, lora_id: int) -> bool:
553
+ if not self.lora_manager:
554
+ raise RuntimeError("LoRA is not enabled.")
555
+ return self.lora_manager.remove_adapter(lora_id)
556
+
557
+ def pin_lora(self, lora_id: int) -> bool:
558
+ if not self.lora_manager:
559
+ raise RuntimeError("LoRA is not enabled.")
560
+ return self.lora_manager.pin_adapter(lora_id)
561
+
562
+ def list_loras(self) -> Set[int]:
563
+ if not self.lora_manager:
564
+ raise RuntimeError("LoRA is not enabled.")
565
+ return self.lora_manager.list_adapters()
566
+
567
568
class CPUModelRunner(CPUModelRunnerBase[ModelInputForCPUWithSamplingMetadata]):
569
_model_input_cls: Type[ModelInputForCPUWithSamplingMetadata] = (
@@ -515,6 +620,12 @@ class CPUModelRunner(CPUModelRunnerBase[ModelInputForCPUWithSamplingMetadata]):
620
raise ValueError(
621
"CPU worker does not support multi-step execution.")
622
623
+ if self.lora_config:
624
+ assert model_input.lora_requests is not None
625
+ assert model_input.lora_mapping is not None
626
+ self.set_active_loras(model_input.lora_requests,
627
+ model_input.lora_mapping)
628
+
629
model_executable = self.model
630
631
multimodal_kwargs = {}
vllm/worker/cpu_worker.py
+16
-4
@@ -1,5 +1,5 @@
1
"""A CPU worker class."""
2
-from typing import Dict, List, Optional, Tuple, Type
2
+from typing import Dict, List, Optional, Set, Tuple, Type
3
4
import torch
5
import torch.distributed
@@ -11,14 +11,14 @@ from vllm.config import (CacheConfig, DeviceConfig, ModelConfig,
11
from vllm.distributed import (ensure_model_parallel_initialized,
12
init_distributed_environment)
13
from vllm.logger import init_logger
14
+from vllm.lora.request import LoRARequest
15
from vllm.model_executor import set_random_seed
16
from vllm.sequence import ExecuteModelRequest
17
from vllm.utils import STR_DTYPE_TO_TORCH_DTYPE, bind_kv_cache
18
from vllm.worker.cpu_enc_dec_model_runner import CPUEncoderDecoderModelRunner
19
from vllm.worker.cpu_model_runner import CPUModelRunner, CPUModelRunnerBase
20
from vllm.worker.cpu_pooling_model_runner import CPUPoolingModelRunner
20
-from vllm.worker.worker_base import (LocalOrDistributedWorkerBase,
21
- LoraNotSupportedWorkerBase, WorkerBase,
21
+from vllm.worker.worker_base import (LocalOrDistributedWorkerBase, WorkerBase,
22
WorkerInput)
23
24
logger = init_logger(__name__)
@@ -111,7 +111,7 @@ class CPUCacheEngine:
111
return dtype_size * total
112
113
114
-class CPUWorker(LoraNotSupportedWorkerBase, LocalOrDistributedWorkerBase):
114
+class CPUWorker(LocalOrDistributedWorkerBase):
115
"""A worker class that executes (a partition of) the model on a CPU socket.
116
117
Each worker is associated with a single CPU socket. The worker is
@@ -266,6 +266,18 @@ class CPUWorker(LoraNotSupportedWorkerBase, LocalOrDistributedWorkerBase):
266
# Initialize the cache.
267
self._init_cache_engine()
268
269
+ def add_lora(self, lora_request: LoRARequest) -> bool:
270
+ return self.model_runner.add_lora(lora_request)
271
+
272
+ def remove_lora(self, lora_id: int) -> bool:
273
+ return self.model_runner.remove_lora(lora_id)
274
+
275
+ def pin_lora(self, lora_id: int) -> bool:
276
+ return self.model_runner.pin_lora(lora_id)
277
+
278
+ def list_loras(self) -> Set[int]:
279
+ return self.model_runner.list_loras()
280
+
281
def _validate_num_cpu_blocks(self, num_cpu_blocks: int) -> None:
282
"""Raise errors if the num_cpu_blocks is invalid.
283
"""