@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 8bddb73512
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 02:00 UTC

[Hardware][CPU] Multi-LoRA implementation for the CPU backend (#11100)

Signed-off-by: Akshat Tripathi <akshat@krai.ai> Signed-off-by: Oleg Mosalov <oleg@krai.ai> Signed-off-by: Jee Jee Li <pandaleefree@gmail.com> Co-authored-by: Oleg Mosalov <oleg@krai.ai> Co-authored-by: Jee Jee Li <pandaleefree@gmail.com> Co-authored-by: Isotr0py <2037008807@qq.com>

Akshat Tripathi committed Jan 12, 2025 at 13:01 UTC 8bddb735123204872788a8ffe117321de7550e6c
25 files changed +855 -193
.buildkite/run-cpu-test.sh
+6
@@ -75,6 +75,12 @@ function cpu_tests() {
75 --num-prompts 20 \
76 --endpoint /v1/completions \
77 --tokenizer facebook/opt-125m"
78 +
79 + # Run multi-lora tests
80 + docker exec cpu-test-"$BUILDKITE_BUILD_NUMBER"-"$NUMA_NODE" bash -c "
81 + set -e
82 + pytest -s -v \
83 + tests/lora/test_qwen2vl.py"
84 }
85
86 # All of CPU tests are expected to be finished less than 25 mins.
docs/source/features/compatibility_matrix.md
+1 -1
@@ -359,7 +359,7 @@ Check the '✗' with links to see tracking issue for unsupported feature/hardwar
359 - ✅
360 - ✅
361 - ✅
362 - - [✗](gh-pr:4830)
362 + - ✅
363 - ✅
364 * - <abbr title="Prompt Adapter">prmpt adptr</abbr>
365 - ✅
tests/lora/conftest.py
+19 -13
@@ -21,6 +21,7 @@ from vllm.model_executor.layers.logits_processor import LogitsProcessor
21 from vllm.model_executor.layers.sampler import Sampler
22 from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
23 from vllm.model_executor.model_loader import get_model
24 +from vllm.platforms import current_platform
25
26
27 class ContextIDInfo(TypedDict):
@@ -65,13 +66,16 @@ def cleanup_fixture(should_do_global_cleanup_after_test: bool):
66 @pytest.fixture
67 def dist_init():
68 temp_file = tempfile.mkstemp()[1]
68 - init_distributed_environment(
69 - world_size=1,
70 - rank=0,
71 - distributed_init_method=f"file://{temp_file}",
72 - local_rank=0,
73 - backend="nccl",
74 - )
69 +
70 + backend = "nccl"
71 + if current_platform.is_cpu():
72 + backend = "gloo"
73 +
74 + init_distributed_environment(world_size=1,
75 + rank=0,
76 + distributed_init_method=f"file://{temp_file}",
77 + local_rank=0,
78 + backend=backend)
79 initialize_model_parallel(1, 1)
80 yield
81 cleanup_dist_env_and_memory(shutdown_ray=True)
@@ -81,13 +85,15 @@ def dist_init():
85 def dist_init_torch_only():
86 if torch.distributed.is_initialized():
87 return
88 + backend = "nccl"
89 + if current_platform.is_cpu():
90 + backend = "gloo"
91 +
92 temp_file = tempfile.mkstemp()[1]
85 - torch.distributed.init_process_group(
86 - backend="nccl",
87 - world_size=1,
88 - rank=0,
89 - init_method=f"file://{temp_file}",
90 - )
93 + torch.distributed.init_process_group(world_size=1,
94 + rank=0,
95 + init_method=f"file://{temp_file}",
96 + backend=backend)
97
98
99 @pytest.fixture
tests/lora/test_layers.py
+30 -9
@@ -48,10 +48,14 @@ TOLERANCES = {
48 torch.float32: (5e-3, 5e-3),
49 torch.bfloat16: (3e-2, 2e-2),
50 }
51 -# TODO: Modify this based on platform
52 -DEVICES = [
51 +
52 +pytestmark = pytest.mark.skipif(
53 + not (current_platform.is_cuda_alike() or current_platform.is_cpu()),
54 + reason="Backend not supported")
55 +
56 +DEVICES = ([
57 f"cuda:{i}" for i in range(1 if torch.cuda.device_count() == 1 else 2)
54 -]
58 +] if current_platform.is_cuda_alike() else ["cpu"])
59
60 #For GPU, we will launch different triton kernels between the prefill and decode
61 # stages, so we need to verify this. prefill stage(True) or decode stage(False)
@@ -198,6 +202,10 @@ def check_punica_wrapper(punica_wrapper) -> bool:
202 from vllm.lora.punica_wrapper.punica_gpu import PunicaWrapperGPU
203
204 return type(punica_wrapper) is PunicaWrapperGPU
205 + elif current_platform.is_cpu():
206 + from vllm.lora.punica_wrapper.punica_cpu import PunicaWrapperCPU
207 +
208 + return type(punica_wrapper) is PunicaWrapperCPU
209 else:
210 return False
211
@@ -211,7 +219,8 @@ def test_embeddings(dist_init, num_loras, device, vocab_size, stage) -> None:
219 # For multi-GPU testing of Triton kernel, we must explicitly set the CUDA
220 # device, see: https://github.com/triton-lang/triton/issues/2925
221 # Same below.
214 - torch.cuda.set_device(device)
222 + if current_platform.is_cuda_alike():
223 + torch.cuda.set_device(device)
224
225 torch.set_default_device(device)
226 max_loras = 8
@@ -313,7 +322,9 @@ def test_embeddings(dist_init, num_loras, device, vocab_size, stage) -> None:
322 def test_embeddings_with_new_embeddings(dist_init, num_loras, device,
323 vocab_size, stage) -> None:
324
316 - torch.cuda.set_device(device)
325 + if current_platform.is_cuda_alike():
326 + torch.cuda.set_device(device)
327 +
328 torch.set_default_device(device)
329 max_loras = 8
330 punica_wrapper = get_punica_wrapper(8192, 256, device)
@@ -450,7 +461,9 @@ def test_embeddings_with_new_embeddings(dist_init, num_loras, device,
461 def test_lm_head_logits_processor(dist_init, num_loras, device, vocab_size,
462 stage) -> None:
463
453 - torch.cuda.set_device(device)
464 + if current_platform.is_cuda_alike():
465 + torch.cuda.set_device(device)
466 +
467 torch.set_default_device(device)
468 max_loras = 8
469 punica_wrapper = get_punica_wrapper(8192, 256, device)
@@ -582,7 +595,9 @@ def test_lm_head_logits_processor(dist_init, num_loras, device, vocab_size,
595 def test_linear_replicated(dist_init, num_loras, device, stage,
596 bias_enabled) -> None:
597
585 - torch.cuda.set_device(device)
598 + if current_platform.is_cuda_alike():
599 + torch.cuda.set_device(device)
600 +
601 torch.set_default_device(device)
602 punica_wrapper = get_punica_wrapper(8192, 256, device)
603 assert check_punica_wrapper(punica_wrapper)
@@ -695,7 +710,9 @@ def test_linear_replicated(dist_init, num_loras, device, stage,
710 def test_linear_parallel(dist_init, num_loras, orientation, fully_shard,
711 device, stage, bias_enabled) -> None:
712
698 - torch.cuda.set_device(device)
713 + if current_platform.is_cuda_alike():
714 + torch.cuda.set_device(device)
715 +
716 torch.set_default_device(device)
717 punica_wrapper = get_punica_wrapper(8192, 256, device)
718 assert check_punica_wrapper(punica_wrapper)
@@ -818,7 +835,9 @@ def test_linear_parallel(dist_init, num_loras, orientation, fully_shard,
835 def test_column_parallel_packed(dist_init, num_loras, repeats, fully_shard,
836 device, stage, bias_enabled) -> None:
837
821 - torch.cuda.set_device(device)
838 + if current_platform.is_cuda_alike():
839 + torch.cuda.set_device(device)
840 +
841 torch.set_default_device(device)
842 punica_wrapper = get_punica_wrapper(8192, 256, device)
843 assert check_punica_wrapper(punica_wrapper)
@@ -971,6 +990,8 @@ def test_column_parallel_packed(dist_init, num_loras, repeats, fully_shard,
990 @pytest.mark.parametrize("rotary_dim", [None, 32])
991 @pytest.mark.parametrize("head_size", [32, 108])
992 @pytest.mark.parametrize("seq_len", [11, 1024])
993 +@pytest.mark.skipif(not current_platform.is_cuda_alike(),
994 + reason="Only CUDA backends are supported")
995 def test_rotary_embedding_long_context(dist_init, num_loras, device,
996 scaling_factors, max_position,
997 is_neox_style, rotary_dim, head_size,
tests/lora/test_lora_manager.py
+11 -10
@@ -20,6 +20,7 @@ from vllm.lora.request import LoRARequest
20 from vllm.lora.worker_manager import (LRUCacheWorkerLoRAManager,
21 WorkerLoRAManager)
22 from vllm.model_executor.layers.linear import RowParallelLinear
23 +from vllm.platforms import current_platform
24
25 EMBEDDING_MODULES = {
26 "embed_tokens": "input_embeddings",
@@ -28,9 +29,9 @@ EMBEDDING_MODULES = {
29
30 EMBEDDING_PADDING_MODULES = ["lm_head"]
31
31 -CUDA_DEVICES = [
32 +DEVICES = ([
33 f"cuda:{i}" for i in range(1 if torch.cuda.device_count() == 1 else 2)
33 -]
34 +] if current_platform.is_cuda_alike() else ["cpu"])
35
36
37 def test_peft_helper(sql_lora_files):
@@ -83,7 +84,7 @@ def test_peft_helper(sql_lora_files):
84 PEFTHelper.from_dict(config)
85
86
86 -@pytest.mark.parametrize("device", CUDA_DEVICES)
87 +@pytest.mark.parametrize("device", DEVICES)
88 def test_from_lora_tensors(sql_lora_files, device):
89 tensors = load_file(
90 os.path.join(sql_lora_files, "adapter_model.safetensors"))
@@ -171,7 +172,7 @@ def test_replace_submodules(dist_init, dummy_model):
172 manager = LoRAModelManager(
173 model, 1, 1, 1,
174 LoRAConfig(max_lora_rank=8, max_cpu_loras=8, max_loras=8),
174 - torch.device("cuda"))
175 + torch.device(DEVICES[0]))
176 model = manager.model
177
178 assert isinstance(model.get_submodule("dense1"),
@@ -183,7 +184,7 @@ def test_replace_submodules(dist_init, dummy_model):
184 RowParallelLinearWithLoRA)
185
186
186 -@pytest.mark.parametrize("device", CUDA_DEVICES)
187 +@pytest.mark.parametrize("device", DEVICES)
188 def test_lora_model_manager(dist_init, dummy_model, device):
189 model = dummy_model
190 model.supported_lora_modules = ["dense1", "dense2", "lm_head"]
@@ -244,7 +245,7 @@ def test_lora_model_manager(dist_init, dummy_model, device):
245 assert manager.punica_wrapper.device == device
246
247
247 -@pytest.mark.parametrize("device", CUDA_DEVICES)
248 +@pytest.mark.parametrize("device", DEVICES)
249 def test_lora_lru_cache_model_manager(dist_init, dummy_model, device):
250 model = dummy_model
251 model.supported_lora_modules = ["dense1", "dense2", "lm_head"]
@@ -336,7 +337,7 @@ def test_lora_lru_cache_model_manager(dist_init, dummy_model, device):
337 assert manager.device == device
338
339
339 -@pytest.mark.parametrize("device", CUDA_DEVICES)
340 +@pytest.mark.parametrize("device", DEVICES)
341 def test_lru_lora_model_manager(dist_init, dummy_model, device):
342 # This tests just the LRU cache functionality, everything else is
343 # tested in test_lora_model_manager
@@ -466,7 +467,7 @@ def test_lru_lora_model_manager(dist_init, dummy_model, device):
467 assert manager.device == device
468
469
469 -@pytest.mark.parametrize("device", CUDA_DEVICES)
470 +@pytest.mark.parametrize("device", DEVICES)
471 def test_lru_cache_worker_adapter_manager(llama_2_7b_model_extra_embeddings,
472 sql_lora_files, device):
473 lora_config = LoRAConfig(max_lora_rank=8, max_cpu_loras=4, max_loras=4)
@@ -545,7 +546,7 @@ def test_lru_cache_worker_adapter_manager(llama_2_7b_model_extra_embeddings,
546 device)
547
548
548 -@pytest.mark.parametrize("device", CUDA_DEVICES)
549 +@pytest.mark.parametrize("device", DEVICES)
550 def test_worker_adapter_manager(llama_2_7b_model_extra_embeddings,
551 sql_lora_files, device):
552 # Should remove every LoRA not specified in the request.
@@ -621,7 +622,7 @@ def test_worker_adapter_manager(llama_2_7b_model_extra_embeddings,
622 device)
623
624
624 -@pytest.mark.parametrize("device", CUDA_DEVICES)
625 +@pytest.mark.parametrize("device", DEVICES)
626 def test_packed_loras(dist_init, dummy_model_gate_up, device):
627 model = dummy_model_gate_up
628 model.supported_lora_modules = ["gate_up_proj"]
tests/lora/test_mixtral.py
+3 -1
@@ -5,6 +5,7 @@ import torch
5
6 import vllm
7 from vllm.lora.request import LoRARequest
8 +from vllm.platforms import current_platform
9
10 MODEL_PATH = "mistralai/Mixtral-8x7B-Instruct-v0.1"
11
@@ -31,7 +32,8 @@ def do_sample(llm: vllm.LLM, lora_path: str, lora_id: int,
32 @pytest.mark.parametrize("tp_size", [4])
33 def test_mixtral_lora(mixtral_lora_files, tp_size):
34 """Original test, the LoRA model has the common target modules, not all"""
34 - if torch.cuda.device_count() < tp_size:
35 + if torch.cuda.device_count(
36 + ) < tp_size and tp_size > 1 and current_platform.is_cuda_alike():
37 pytest.skip(f"Not enough GPUs for tensor parallelism {tp_size}")
38
39 prompts = [
tests/lora/test_punica_ops_sizes.py renamed
+75 -49
@@ -9,17 +9,16 @@ from threading import Lock
9 import pytest
10 import torch
11
12 -from vllm.lora.ops.bgmv_expand import bgmv_expand
13 -from vllm.lora.ops.bgmv_expand_slice import bgmv_expand_slice
14 -from vllm.lora.ops.bgmv_shrink import bgmv_shrink
15 -from vllm.lora.ops.sgmv_expand import sgmv_expand
16 -from vllm.lora.ops.sgmv_shrink import sgmv_shrink
17 -from vllm.lora.ops.utils import _LORA_A_PTR_DICT, _LORA_B_PTR_DICT
12 +import vllm.lora.ops.triton_ops # noqa: F401
13 +from vllm.lora.ops.torch_ops import (bgmv_expand, bgmv_expand_slice,
14 + bgmv_shrink, sgmv_expand,
15 + sgmv_expand_slice, sgmv_shrink)
16 +from vllm.lora.ops.triton_ops.utils import _LORA_A_PTR_DICT, _LORA_B_PTR_DICT
17 from vllm.platforms import current_platform
18
19 from .utils import (assert_close, generate_data,
20 generate_data_for_expand_nslices,
22 - generate_data_for_nslices, ref_torch_groupgemm)
21 + generate_data_for_nslices)
22
23 HIDDEN_SIZES = [
24 128,
@@ -113,7 +112,7 @@ DTYPES = [torch.float16, torch.bfloat16]
112 MAX_RANKS = [32]
113 SCALES = [0.5]
114 SEED = [0]
116 -CUDA_DEVICES = [f"cuda:{0}"]
115 +DEVICES = [f"cuda:{0}"]
116
117 _dict_lock = Lock()
118
@@ -127,7 +126,7 @@ _dict_lock = Lock()
126 @pytest.mark.parametrize("dtype", DTYPES)
127 @pytest.mark.parametrize("op_type", ["shrink", "expand"])
128 @pytest.mark.parametrize("seed", SEED)
130 -@pytest.mark.parametrize("device", CUDA_DEVICES)
129 +@pytest.mark.parametrize("device", DEVICES)
130 def test_punica_sgmv(
131 batches: int,
132 num_loras: int,
@@ -174,7 +173,7 @@ def test_punica_sgmv(
173 # Preventing cache error pointer.
174 with _dict_lock:
175 _LORA_A_PTR_DICT.clear()
177 - sgmv_shrink(
176 + torch.ops.vllm.sgmv_shrink(
177 inputs_tensor,
178 lora_weights_lst,
179 our_out_tensor,
@@ -187,20 +186,23 @@ def test_punica_sgmv(
186 scaling,
187 )
188 for index in range(nslices):
190 - ref_torch_groupgemm(
191 - ref_out_tensor[index],
189 + sgmv_shrink(
190 inputs_tensor,
191 lora_weights_lst[index],
194 - lora_indices_tensor,
192 + ref_out_tensor[index],
193 + b_seq_start_loc,
194 seq_len_tensor,
195 + lora_indices_tensor,
196 batches,
197 + max_seq_length,
198 + token_nums,
199 scaling,
198 - op_type,
200 )
201 +
202 else:
203 with _dict_lock:
204 _LORA_B_PTR_DICT.clear()
203 - sgmv_expand(
205 + torch.ops.vllm.sgmv_expand(
206 inputs_tensor,
207 lora_weights_lst,
208 our_out_tensor,
@@ -213,21 +215,39 @@ def test_punica_sgmv(
215 offset_start=0,
216 add_inputs=True,
217 )
216 -
217 - slice_offset = 0
218 - for index in range(nslices):
219 - lora_weights = lora_weights_lst[index]
220 - ref_torch_groupgemm(
221 - ref_out_tensor[:, slice_offset:slice_offset + hidden_size],
222 - inputs_tensor[index],
223 - lora_weights,
224 - lora_indices_tensor,
218 + if nslices == 1:
219 + # Verify the torch's sgmv_expand op
220 + sgmv_expand(
221 + inputs_tensor[0],
222 + lora_weights_lst[0],
223 + ref_out_tensor,
224 + b_seq_start_loc,
225 seq_len_tensor,
226 + lora_indices_tensor,
227 batches,
227 - 1.0,
228 - op_type,
228 + max_seq_length,
229 + token_nums,
230 + add_inputs=True,
231 )
230 - slice_offset += hidden_size
232 + else:
233 + slice_offset = 0
234 + for index in range(nslices):
235 + lora_weights = lora_weights_lst[index]
236 + sgmv_expand_slice(
237 + inputs_tensor[index],
238 + lora_weights,
239 + ref_out_tensor,
240 + b_seq_start_loc,
241 + seq_len_tensor,
242 + lora_indices_tensor,
243 + batches,
244 + max_seq_length,
245 + token_nums,
246 + slice_offset,
247 + hidden_size,
248 + add_inputs=True,
249 + )
250 + slice_offset += hidden_size
251
252 assert_close(our_out_tensor, ref_out_tensor)
253
@@ -240,7 +260,7 @@ def test_punica_sgmv(
260 @pytest.mark.parametrize("dtype", DTYPES)
261 @pytest.mark.parametrize("op_type", ["shrink", "expand"])
262 @pytest.mark.parametrize("seed", SEED)
243 -@pytest.mark.parametrize("device", CUDA_DEVICES)
263 +@pytest.mark.parametrize("device", DEVICES)
264 def test_punica_bgmv(
265 batches: int,
266 num_loras: int,
@@ -276,31 +296,38 @@ def test_punica_bgmv(
296 device,
297 )
298 if op_type == "shrink":
279 - bgmv_shrink(
299 + torch.ops.vllm.bgmv_shrink(
300 inputs_tensor,
301 lora_weights,
302 our_out_tensor,
303 indices,
304 scaling,
305 )
306 +
307 + bgmv_shrink(
308 + inputs_tensor,
309 + lora_weights,
310 + ref_out_tensor,
311 + indices,
312 + scaling,
313 + )
314 +
315 else:
287 - bgmv_expand(
316 + torch.ops.vllm.bgmv_expand(
317 inputs_tensor,
318 lora_weights,
319 our_out_tensor,
320 indices,
321 add_inputs=True,
322 )
294 - ref_torch_groupgemm(
295 - ref_out_tensor,
296 - inputs_tensor,
297 - lora_weights,
298 - lora_indices_tensor,
299 - seq_len_tensor,
300 - batches,
301 - scaling if op_type == "shrink" else 1.0,
302 - op_type,
303 - )
323 + bgmv_expand(
324 + inputs_tensor,
325 + lora_weights,
326 + ref_out_tensor,
327 + indices,
328 + add_inputs=True,
329 + )
330 +
331 if op_type == "shrink":
332 ref_out_tensor = ref_out_tensor.to(torch.float32)
333 assert_close(our_out_tensor, ref_out_tensor)
@@ -313,7 +340,7 @@ def test_punica_bgmv(
340 @pytest.mark.parametrize("nslices", [2, 3])
341 @pytest.mark.parametrize("dtype", DTYPES)
342 @pytest.mark.parametrize("seed", SEED)
316 -@pytest.mark.parametrize("device", CUDA_DEVICES)
343 +@pytest.mark.parametrize("device", DEVICES)
344 def test_punica_bgmv_expand_nslices(
345 batches: int,
346 num_loras: int,
@@ -350,7 +377,7 @@ def test_punica_bgmv_expand_nslices(
377 slice_offset = 0
378 for index in range(nslices):
379 lora_weights = lora_weights_lst[index]
353 - bgmv_expand_slice(
380 + torch.ops.vllm.bgmv_expand_slice(
381 inputs_tensor,
382 lora_weights,
383 our_outputs,
@@ -359,15 +386,14 @@ def test_punica_bgmv_expand_nslices(
386 slice_size=hidden_size,
387 add_inputs=True,
388 )
362 - ref_torch_groupgemm(
363 - ref_outputs[:, slice_offset:slice_offset + hidden_size],
389 + bgmv_expand_slice(
390 inputs_tensor,
391 lora_weights,
366 - lora_indices_tensor,
367 - seq_len_tensor,
368 - batches,
369 - 1.0,
370 - op_type="expand",
392 + ref_outputs,
393 + indices,
394 + slice_offset,
395 + slice_size=hidden_size,
396 + add_inputs=True,
397 )
398
399 slice_offset += hidden_size
tests/lora/test_punica_ops_variation.py renamed
+75 -59
@@ -9,19 +9,18 @@ import pytest
9 import torch
10
11 # Enable custom op register
12 -import vllm.lora.ops.bgmv_expand
13 -import vllm.lora.ops.bgmv_expand_slice
14 -import vllm.lora.ops.bgmv_shrink
15 -import vllm.lora.ops.sgmv_expand
16 -import vllm.lora.ops.sgmv_shrink # noqa: F401
17 -from vllm.lora.ops.utils import _LORA_A_PTR_DICT, _LORA_B_PTR_DICT
12 +import vllm.lora.ops.triton_ops # noqa: F401
13 +from vllm.lora.ops.torch_ops import (bgmv_expand, bgmv_expand_slice,
14 + bgmv_shrink, sgmv_expand,
15 + sgmv_expand_slice, sgmv_shrink)
16 +from vllm.lora.ops.triton_ops.utils import _LORA_A_PTR_DICT, _LORA_B_PTR_DICT
17 from vllm.platforms import current_platform
18
19 from .utils import (assert_close, generate_data,
20 generate_data_for_expand_nslices,
22 - generate_data_for_nslices, ref_torch_groupgemm)
21 + generate_data_for_nslices)
22
24 -HIDDEN_SIZES = [4097]
23 +HIDDEN_SIZES = [2049]
24
25 BATCHES = [1, 4, 16, 32]
26 NUM_LORA = [1, 8, 32, 128]
@@ -29,15 +28,7 @@ DTYPES = [torch.float16, torch.bfloat16]
28 MAX_RANKS = [1, 4, 8, 16, 32, 64, 128, 256]
29 SCALES = [0.5]
30 SEED = [0]
32 -CUDA_DEVICES = [f"cuda:{0}"]
33 -
34 -# Unlike test_punica_sizes.py, we directly utilize custom op for
35 -# testing, which verifies the correct registration of these ops.
36 -bgmv_expand = torch.ops.vllm.bgmv_expand
37 -bgmv_expand_slice = torch.ops.vllm.bgmv_expand_slice
38 -bgmv_shrink = torch.ops.vllm.bgmv_shrink
39 -sgmv_expand = torch.ops.vllm.sgmv_expand
40 -sgmv_shrink = torch.ops.vllm.sgmv_shrink
31 +DEVICES = [f"cuda:{0}"]
32
33 _dict_lock = Lock()
34
@@ -51,7 +42,7 @@ _dict_lock = Lock()
42 @pytest.mark.parametrize("dtype", DTYPES)
43 @pytest.mark.parametrize("op_type", ["shrink", "expand"])
44 @pytest.mark.parametrize("seed", SEED)
54 -@pytest.mark.parametrize("device", CUDA_DEVICES)
45 +@pytest.mark.parametrize("device", DEVICES)
46 def test_punica_sgmv(
47 batches: int,
48 num_loras: int,
@@ -98,7 +89,7 @@ def test_punica_sgmv(
89 # Preventing cache error pointer.
90 with _dict_lock:
91 _LORA_A_PTR_DICT.clear()
101 - sgmv_shrink(
92 + torch.ops.vllm.sgmv_shrink(
93 inputs_tensor,
94 lora_weights_lst,
95 our_out_tensor,
@@ -111,20 +102,23 @@ def test_punica_sgmv(
102 scaling,
103 )
104 for index in range(nslices):
114 - ref_torch_groupgemm(
115 - ref_out_tensor[index],
105 + sgmv_shrink(
106 inputs_tensor,
107 lora_weights_lst[index],
118 - lora_indices_tensor,
108 + ref_out_tensor[index],
109 + b_seq_start_loc,
110 seq_len_tensor,
111 + lora_indices_tensor,
112 batches,
113 + max_seq_length,
114 + token_nums,
115 scaling,
122 - op_type,
116 )
117 +
118 else:
119 with _dict_lock:
120 _LORA_B_PTR_DICT.clear()
127 - sgmv_expand(
121 + torch.ops.vllm.sgmv_expand(
122 inputs_tensor,
123 lora_weights_lst,
124 our_out_tensor,
@@ -137,21 +131,39 @@ def test_punica_sgmv(
131 offset_start=0,
132 add_inputs=True,
133 )
140 -
134 slice_offset = 0
142 - for index in range(nslices):
143 - lora_weights = lora_weights_lst[index]
144 - ref_torch_groupgemm(
145 - ref_out_tensor[:, slice_offset:slice_offset + hidden_size],
146 - inputs_tensor[index],
147 - lora_weights,
148 - lora_indices_tensor,
135 + if nslices == 1:
136 + # Verify the torch's sgmv_expand op
137 + sgmv_expand(
138 + inputs_tensor[0],
139 + lora_weights_lst[0],
140 + ref_out_tensor,
141 + b_seq_start_loc,
142 seq_len_tensor,
143 + lora_indices_tensor,
144 batches,
151 - 1.0,
152 - op_type,
145 + max_seq_length,
146 + token_nums,
147 + add_inputs=True,
148 )
154 - slice_offset += hidden_size
149 + else:
150 + for index in range(nslices):
151 + lora_weights = lora_weights_lst[index]
152 + sgmv_expand_slice(
153 + inputs_tensor[index],
154 + lora_weights,
155 + ref_out_tensor,
156 + b_seq_start_loc,
157 + seq_len_tensor,
158 + lora_indices_tensor,
159 + batches,
160 + max_seq_length,
161 + token_nums,
162 + slice_offset,
163 + hidden_size,
164 + add_inputs=True,
165 + )
166 + slice_offset += hidden_size
167
168 assert_close(our_out_tensor, ref_out_tensor)
169
@@ -164,7 +176,7 @@ def test_punica_sgmv(
176 @pytest.mark.parametrize("dtype", DTYPES)
177 @pytest.mark.parametrize("op_type", ["shrink", "expand"])
178 @pytest.mark.parametrize("seed", SEED)
167 -@pytest.mark.parametrize("device", CUDA_DEVICES)
179 +@pytest.mark.parametrize("device", DEVICES)
180 def test_punica_bgmv(
181 batches: int,
182 num_loras: int,
@@ -176,7 +188,6 @@ def test_punica_bgmv(
188 seed: int,
189 device: str,
190 ):
179 -
191 torch.set_default_device(device)
192 current_platform.seed_everything(seed)
193
@@ -201,32 +212,38 @@ def test_punica_bgmv(
212 device,
213 )
214 if op_type == "shrink":
204 - bgmv_shrink(
215 + torch.ops.vllm.bgmv_shrink(
216 inputs_tensor,
217 lora_weights,
218 our_out_tensor,
219 indices,
220 scaling,
221 )
211 - else:
222
213 - bgmv_expand(
223 + bgmv_shrink(
224 + inputs_tensor,
225 + lora_weights,
226 + ref_out_tensor,
227 + indices,
228 + scaling,
229 + )
230 +
231 + else:
232 + torch.ops.vllm.bgmv_expand(
233 inputs_tensor,
234 lora_weights,
235 our_out_tensor,
236 indices,
237 add_inputs=True,
238 )
220 - ref_torch_groupgemm(
221 - ref_out_tensor,
222 - inputs_tensor,
223 - lora_weights,
224 - lora_indices_tensor,
225 - seq_len_tensor,
226 - batches,
227 - scaling if op_type == "shrink" else 1.0,
228 - op_type,
229 - )
239 + bgmv_expand(
240 + inputs_tensor,
241 + lora_weights,
242 + ref_out_tensor,
243 + indices,
244 + add_inputs=True,
245 + )
246 +
247 if op_type == "shrink":
248 ref_out_tensor = ref_out_tensor.to(torch.float32)
249 assert_close(our_out_tensor, ref_out_tensor)
@@ -239,7 +256,7 @@ def test_punica_bgmv(
256 @pytest.mark.parametrize("nslices", [2, 3])
257 @pytest.mark.parametrize("dtype", DTYPES)
258 @pytest.mark.parametrize("seed", SEED)
242 -@pytest.mark.parametrize("device", CUDA_DEVICES)
259 +@pytest.mark.parametrize("device", DEVICES)
260 def test_punica_bgmv_expand_nslices(
261 batches: int,
262 num_loras: int,
@@ -276,7 +293,7 @@ def test_punica_bgmv_expand_nslices(
293 slice_offset = 0
294 for index in range(nslices):
295 lora_weights = lora_weights_lst[index]
279 - bgmv_expand_slice(
296 + torch.ops.vllm.bgmv_expand_slice(
297 inputs_tensor,
298 lora_weights,
299 our_outputs,
@@ -285,15 +302,14 @@ def test_punica_bgmv_expand_nslices(
302 slice_size=hidden_size,
303 add_inputs=True,
304 )
288 - ref_torch_groupgemm(
289 - ref_outputs[:, slice_offset:slice_offset + hidden_size],
305 + bgmv_expand_slice(
306 inputs_tensor,
307 lora_weights,
292 - lora_indices_tensor,
293 - seq_len_tensor,
294 - batches,
295 - 1.0,
296 - op_type="expand",
308 + ref_outputs,
309 + indices,
310 + slice_offset,
311 + slice_size=hidden_size,
312 + add_inputs=True,
313 )
314
315 slice_offset += hidden_size
tests/lora/test_quant_model.py
+2 -1
@@ -72,7 +72,8 @@ def do_sample(llm: vllm.LLM,
72 @pytest.mark.parametrize("tp_size", [1])
73 def test_quant_model_lora(tinyllama_lora_files, num_gpus_available, model,
74 tp_size):
75 - if num_gpus_available < tp_size:
75 + if num_gpus_available < tp_size and \
76 + tp_size > 1 and current_platform.is_cuda_alike():
77 pytest.skip(f"Not enough GPUs for tensor parallelism {tp_size}")
78
79 llm = vllm.LLM(
tests/lora/utils.py
-27
@@ -104,33 +104,6 @@ def assert_close(a, b):
104 torch.testing.assert_close(a, b, rtol=rtol, atol=atol)
105
106
107 -def ref_torch_groupgemm(
108 - out_tensor,
109 - inputs,
110 - lora_weights,
111 - lora_indices_tensor,
112 - seq_len_tensor,
113 - batches,
114 - scaling,
115 - op_type,
116 -) -> torch.Tensor:
117 - out_list = []
118 - current_offset = 0
119 - for lora_index, b_length in zip(range(batches), seq_len_tensor):
120 - input_weight = inputs[current_offset:b_length + current_offset, :]
121 - current_offset += b_length
122 - lora_weight = lora_weights[lora_indices_tensor[lora_index]]
123 - result = torch.nn.functional.linear(input_weight, lora_weight)
124 - result *= scaling
125 - out_list.append(result)
126 - cat_result = torch.cat(out_list, dim=0)
127 - if op_type == "expand":
128 - out_tensor += cat_result
129 - else:
130 - out_tensor.copy_(cat_result)
131 - return
132 -
133 -
107 def generate_data(
108 batches,
109 hidden_size,
vllm/executor/cpu_executor.py
-3
@@ -22,9 +22,6 @@ class CPUExecutor(ExecutorBase):
22
23 def _init_executor(self) -> None:
24 assert self.device_config.device_type == "cpu"
25 - # Reminder: Please update docs/source/features/compatibility_matrix.md
26 - # If the feature combo become valid
27 - assert self.lora_config is None, "cpu backend doesn't support LoRA"
25
26 #
27 # Environment variables for CPU executor
vllm/lora/ops/torch_ops/__init__.py new
+13
@@ -0,0 +1,13 @@
1 +from vllm.lora.ops.torch_ops.lora_ops import bgmv_expand # noqa: F401
2 +from vllm.lora.ops.torch_ops.lora_ops import (bgmv_expand_slice, bgmv_shrink,
3 + sgmv_expand, sgmv_expand_slice,
4 + sgmv_shrink)
5 +
6 +__all__ = [
7 + "bgmv_expand",
8 + "bgmv_expand_slice",
9 + "bgmv_shrink",
10 + "sgmv_expand",
11 + "sgmv_expand_slice",
12 + "sgmv_shrink",
13 +]
vllm/lora/ops/torch_ops/lora_ops.py new
+113
@@ -0,0 +1,113 @@
1 +import torch
2 +
3 +
4 +def sgmv_expand(inputs: torch.Tensor,
5 + lora_b_weights: torch.Tensor,
6 + output_tensor: torch.Tensor,
7 + b_seq_start_loc: torch.Tensor,
8 + seq_len_tensor: torch.Tensor,
9 + lora_indices_tensor: torch.Tensor,
10 + batches: int,
11 + max_seq_length: int,
12 + token_nums: int,
13 + add_inputs: bool = False):
14 + exploded_indices = torch.repeat_interleave(lora_indices_tensor,
15 + seq_len_tensor)
16 +
17 + bgmv_expand(inputs, lora_b_weights, output_tensor, exploded_indices,
18 + add_inputs)
19 +
20 +
21 +def bgmv_expand(inputs: torch.Tensor,
22 + lora_b_weights: torch.Tensor,
23 + output_tensor: torch.Tensor,
24 + lora_indices_tensor: torch.Tensor,
25 + add_inputs: bool = True):
26 + selected_loras = lora_b_weights[lora_indices_tensor].to(
27 + dtype=output_tensor.dtype)
28 + if len(selected_loras.shape) == 4:
29 + selected_loras = selected_loras.squeeze(dim=1)
30 + inputs = inputs.to(dtype=output_tensor.dtype)
31 + outputs = torch.einsum("bi, boi -> bo", inputs, selected_loras)
32 +
33 + limit = output_tensor.shape[0]
34 + if outputs.shape[0] == 1 and output_tensor.shape[0] != 1:
35 + limit = 1
36 +
37 + if add_inputs:
38 + output_tensor[:, :outputs.shape[1]] += outputs[:limit, :]
39 + else:
40 + output_tensor[:, :outputs.shape[1]] = outputs[:limit, :]
41 +
42 +
43 +def sgmv_shrink(
44 + inputs: torch.Tensor,
45 + lora_a_weights: torch.Tensor,
46 + output_tensor: torch.Tensor,
47 + b_seq_start_loc: torch.Tensor,
48 + seq_len_tensor: torch.Tensor,
49 + lora_indices_tensor: torch.Tensor,
50 + batches: int,
51 + max_seq_length: int,
52 + token_nums: int,
53 + scaling: float,
54 +):
55 + exploded_indices = torch.repeat_interleave(lora_indices_tensor,
56 + seq_len_tensor)
57 +
58 + bgmv_shrink(inputs, lora_a_weights, output_tensor, exploded_indices,
59 + scaling)
60 +
61 +
62 +def bgmv_shrink(inputs: torch.Tensor,
63 + lora_b_weights: torch.Tensor,
64 + output_tensor: torch.Tensor,
65 + lora_indices_tensor: torch.Tensor,
66 + scaling: float = 1.0):
67 + selected_loras = lora_b_weights[lora_indices_tensor].to(
68 + dtype=output_tensor.dtype)
69 + if len(selected_loras.shape) == 4:
70 + selected_loras = selected_loras.squeeze(dim=1)
71 + inputs = inputs.to(dtype=output_tensor.dtype)
72 + outputs = torch.einsum("bi, boi -> bo", inputs, selected_loras)
73 +
74 + output_tensor[:, :outputs.shape[1]] = scaling * outputs[:]
75 +
76 +
77 +def sgmv_expand_slice(inputs: torch.Tensor,
78 + lora_b_weights: torch.Tensor,
79 + output_tensor: torch.Tensor,
80 + b_seq_start_loc: torch.Tensor,
81 + seq_len_tensor: torch.Tensor,
82 + lora_indices_tensor: torch.Tensor,
83 + batches: int,
84 + max_seq_length: int,
85 + token_nums: int,
86 + slice_offset: int,
87 + slice_size: int,
88 + add_inputs: bool = False):
89 + exploded_indices = torch.repeat_interleave(lora_indices_tensor,
90 + seq_len_tensor)
91 +
92 + bgmv_expand_slice(inputs, lora_b_weights, output_tensor, exploded_indices,
93 + slice_offset, slice_size, add_inputs)
94 +
95 +
96 +def bgmv_expand_slice(inputs: torch.Tensor,
97 + lora_b_weights: torch.Tensor,
98 + output_tensor: torch.Tensor,
99 + lora_indices_tensor: torch.Tensor,
100 + slice_offset: int,
101 + slice_size: int,
102 + add_inputs: bool = True):
103 + selected_loras = lora_b_weights[lora_indices_tensor].to(
104 + dtype=output_tensor.dtype)
105 + inputs = inputs.to(dtype=output_tensor.dtype)
106 + if len(selected_loras.shape) == 4:
107 + selected_loras = selected_loras.squeeze(dim=1)
108 + outputs = torch.einsum("bi, boi -> bo", inputs, selected_loras)
109 +
110 + if add_inputs:
111 + output_tensor[:, slice_offset:slice_offset + slice_size] += outputs[:]
112 + else:
113 + output_tensor[:, slice_offset:slice_offset + slice_size] = outputs[:]
vllm/lora/ops/triton_ops/__init__.py new
+13
@@ -0,0 +1,13 @@
1 +from vllm.lora.ops.triton_ops.bgmv_expand import bgmv_expand
2 +from vllm.lora.ops.triton_ops.bgmv_expand_slice import bgmv_expand_slice
3 +from vllm.lora.ops.triton_ops.bgmv_shrink import bgmv_shrink
4 +from vllm.lora.ops.triton_ops.sgmv_expand import sgmv_expand
5 +from vllm.lora.ops.triton_ops.sgmv_shrink import sgmv_shrink # noqa: F401
6 +
7 +__all__ = [
8 + "bgmv_expand",
9 + "bgmv_expand_slice",
10 + "bgmv_shrink",
11 + "sgmv_expand",
12 + "sgmv_shrink",
13 +]
vllm/lora/ops/triton_ops/bgmv_expand.py renamed
vllm/lora/ops/triton_ops/bgmv_expand_slice.py renamed
vllm/lora/ops/triton_ops/bgmv_shrink.py renamed
vllm/lora/ops/triton_ops/sgmv_expand.py renamed
vllm/lora/ops/triton_ops/sgmv_shrink.py renamed
vllm/lora/ops/triton_ops/utils.py renamed
vllm/lora/punica_wrapper/punica_cpu.py new
+346
@@ -0,0 +1,346 @@
1 +from typing import Callable, Optional, Tuple, Union
2 +
3 +import torch
4 +
5 +from vllm.lora.ops.torch_ops import (bgmv_expand, bgmv_expand_slice,
6 + bgmv_shrink, sgmv_expand,
7 + sgmv_expand_slice, sgmv_shrink)
8 +
9 +from .punica_base import PunicaWrapperBase
10 +
11 +
12 +# The platforms that are compatible with the PyTorch-native implementation can
13 +# inherit this class
14 +class PunicaWrapperCPU(PunicaWrapperBase):
15 + """
16 + PunicaWrapperCPU is designed to manage and provide metadata for the punica
17 + kernel. The main function is to maintain the state information for
18 + Multi-LoRA, and to provide the interface for the pytorch punica ops.
19 + """
20 +
21 + def __init__(self, max_num_batched_tokens: int, max_batches: int,
22 + device: Union[torch.device, str], **kwargs):
23 + PunicaWrapperBase.__init__(self, max_num_batched_tokens, max_batches,
24 + device)
25 +
26 + def _shrink_prefill(
27 + self,
28 + y: torch.Tensor,
29 + x: torch.Tensor,
30 + w_t_all: torch.Tensor,
31 + scale: float,
32 + ):
33 + #No LoRA request, so return directly
34 + if self.no_lora:
35 + return
36 + sgmv_shrink(
37 + x,
38 + w_t_all,
39 + y,
40 + *self.prefill_metadata,
41 + scale,
42 + )
43 +
44 + def _shrink_decode(
45 + self,
46 + y: torch.Tensor,
47 + x: torch.Tensor,
48 + w_t_all: torch.Tensor,
49 + scale: float,
50 + ):
51 + bgmv_shrink(x, w_t_all, y, self.token_lora_indices, scale)
52 +
53 + def _expand_prefill(
54 + self,
55 + y: torch.Tensor,
56 + x: torch.Tensor,
57 + w_t_all: torch.Tensor,
58 + add_inputs: bool,
59 + ):
60 + #No LoRA request, so return directly
61 + if self.no_lora:
62 + return
63 + sgmv_expand(
64 + x,
65 + w_t_all,
66 + y,
67 + *self.prefill_metadata,
68 + add_inputs,
69 + )
70 +
71 + def _expand_decode(
72 + self,
73 + y: torch.Tensor,
74 + x: torch.Tensor,
75 + w_t_all: torch.Tensor,
76 + add_inputs: bool,
77 + ):
78 + bgmv_expand(x, w_t_all, y, self.token_lora_indices, add_inputs)
79 +
80 + def _expand_slice_prefill(
81 + self,
82 + y: torch.Tensor,
83 + x: torch.Tensor,
84 + w_t_all: torch.Tensor,
85 + y_offset: int,
86 + y_slice_size: int,
87 + add_inputs: bool,
88 + ):
89 + #No LoRA request, so return directly
90 + if self.no_lora:
91 + return
92 + sgmv_expand_slice(
93 + x,
94 + w_t_all,
95 + y,
96 + *self.prefill_metadata,
97 + y_offset,
98 + y_slice_size,
99 + add_inputs,
100 + )
101 +
102 + def _expand_slice_decode(
103 + self,
104 + y: torch.Tensor,
105 + x: torch.Tensor,
106 + w_t_all: torch.Tensor,
107 + y_offset: int,
108 + y_slice_size: int,
109 + add_inputs: bool,
110 + ):
111 + bgmv_expand_slice(x, w_t_all, y, self.token_lora_indices, y_offset,
112 + y_slice_size, add_inputs)
113 +
114 + def _apply_expand(
115 + self,
116 + y: torch.Tensor,
117 + x: torch.Tensor,
118 + w_t_all: torch.Tensor,
119 + y_offset: int,
120 + y_slice_size: int,
121 + add_inputs: bool = True,
122 + ):
123 + """
124 + Perform the ` y[:,y_offset:y_offset+y_slice_size]+=x@w_t_all`
125 + computation, which is suitable for the
126 + GEMM of lora'b.
127 + """
128 +
129 + expand_slice_fun: Callable = (self._expand_slice_prefill
130 + if self.is_prefill else
131 + self._expand_slice_decode)
132 + expand_slice_fun(y, x, w_t_all, y_offset, y_slice_size, add_inputs)
133 +
134 + def _apply_shrink(self, y: torch.Tensor, x: torch.Tensor,
135 + w_t_all: torch.Tensor, scale: float):
136 + """
137 + Perform the ` y+=x@w_t_all` computation, which is suitable for the
138 + GEMM of lora'a.
139 + When `is_prefill is` true, it indicates that it is currently the
140 + prefill stage, and the `_shrink_prefill` function should be called.
141 + Otherwise, it is the decode stage, and the _shrink_decode function
142 + should be called.
143 + """
144 + y_org = y
145 + y = y.view(-1, y.shape[-1])
146 + shrink_fun: Callable = (self._shrink_prefill
147 + if self.is_prefill else self._shrink_decode)
148 + shrink_fun(y, x, w_t_all, scale)
149 + y = y.view_as(y_org)
150 +
151 + def add_shrink(self, y: Union[Tuple[torch.Tensor, ...], torch.Tensor],
152 + x: torch.Tensor, lora_a_stacked: Tuple[torch.Tensor, ...],
153 + scale: float, **kwargs):
154 + """
155 + Performs GEMM for multiple slices of lora_a.
156 + When `is_prefill is` true, it indicates that it is currently the
157 + prefill stage, and the `_shrink_prefill` function should be called.
158 + Otherwise, it is the decode stage, and the _shrink_decode function
159 + should be called.
160 +
161 + Semantics:
162 + for i in range(len(lora_a_stacked)):
163 + y[i] += (x @ lora_a_stacked[i]) * scale
164 +
165 + Args:
166 + y (Union[Tuple[torch.Tensor, ...], torch.Tensor]): Output tensors
167 + x (torch.Tensor): Input tensor
168 + lora_a_stacked (Tuple[torch.Tensor, ...]): lora_a's weights
169 + scale (float): Scaling factor for the operation
170 + """
171 +
172 + x = x.view(-1, x.shape[-1])
173 + # TODO fuse these kernels
174 + for slice_idx in range(len(lora_a_stacked)):
175 + self._apply_shrink(y[slice_idx], x, lora_a_stacked[slice_idx],
176 + scale)
177 +
178 + def add_expand(self,
179 + y: torch.Tensor,
180 + x: Union[Tuple[torch.Tensor, ...], torch.Tensor],
181 + lora_b_stacked: Tuple[torch.Tensor, ...],
182 + lora_bias_stacked: Optional[Tuple[torch.Tensor, ...]],
183 + output_slices: Tuple[int, ...],
184 + offset_start: int = 0,
185 + add_inputs=True,
186 + **kwargs) -> None:
187 + """
188 + Performs GEMM and bias addition for multiple slices of lora_b.
189 +
190 + Semantics:
191 + for i in range(len(lora_b_stacked)):
192 + slice = output_slices[i]
193 + y[:, offset:offset+slice] += x[i] @ lora_b_stacked[i] +
194 + lora_bias_stacked[i]
195 + offset += slice
196 +
197 + Args:
198 + y (torch.Tensor): Output tensor.
199 + x (Union[Tuple[torch.Tensor, ...], torch.Tensor]): Input tensors
200 + lora_b_stacked (Tuple[torch.Tensor, ...]): lora_b's weight
201 + lora_bias_stacked (Optional[Tuple[torch.Tensor, ...]]):
202 + bias's weight
203 + output_slices (Tuple[int, ...]): Every slice's size
204 + add_inputs (bool): Defaults to True.
205 + """
206 + y_org = y
207 + y = y.view(-1, y.shape[-1])
208 + offset_left = offset_start
209 + if lora_bias_stacked is not None:
210 + self._apply_bias(self.token_lora_indices, y, output_slices,
211 + lora_bias_stacked)
212 + for slice_idx in range(len(lora_b_stacked)):
213 + self._apply_expand(
214 + y,
215 + x[slice_idx],
216 + lora_b_stacked[slice_idx],
217 + offset_left,
218 + output_slices[slice_idx],
219 + add_inputs=add_inputs,
220 + )
221 + offset_left += output_slices[slice_idx]
222 + y = y.view_as(y_org)
223 +
224 + def add_lora_embedding(self,
225 + y: torch.Tensor,
226 + x: torch.Tensor,
227 + lora_b_stacked: torch.Tensor,
228 + add_inputs: bool = True,
229 + **kwargs) -> None:
230 + """
231 + Applies lora specifically for VocabParallelEmbeddingWithLoRA.
232 +
233 + Semantics:
234 + y += x @ lora_b_stacked
235 +
236 + Args:
237 + y (torch.Tensor): Output tensor.
238 + x (torch.Tensor): Input tensor.
239 + lora_b_stacked (torch.Tensor): lora_b's weights.
240 + add_inputs (bool): Default to True.
241 + """
242 +
243 + # Embedding layer only need expand op
244 + expand_fun: Callable = (self._expand_prefill
245 + if self.is_prefill else self._expand_decode)
246 + expand_fun(y, x, lora_b_stacked, add_inputs)
247 +
248 + def add_lora_linear(self,
249 + y: torch.Tensor,
250 + x: torch.Tensor,
251 + lora_a_stacked: Tuple[torch.Tensor, ...],
252 + lora_b_stacked: Tuple[torch.Tensor, ...],
253 + lora_bias_stacked: Optional[Tuple[torch.Tensor, ...]],
254 + scale: float,
255 + output_slices: Tuple[int, ...],
256 + *,
257 + buffer: Optional[Tuple[torch.Tensor, ...]] = None,
258 + **kwargs) -> None:
259 + """
260 + Applicable to linear-related lora.
261 +
262 + Semantics:
263 + for i in range(len(lora_a_stacked)):
264 + y[i] += (
265 + x[i].unsqueeze(0)
266 + @ lora_a_stacked[indices[i], layer_idx, :, :]
267 + @ lora_b_stacked[indices[i], layer_idx, :, :]
268 + * scale
269 + ).squeeze(0)+lora_bias_stacked[i]
270 +
271 + Args:
272 + y (torch.Tensor): Output tensor. Will be changed in-place.
273 + x (torch.Tensor): Input tensor
274 + lora_a_stacked (Tuple[torch.Tensor, ...]): lora_a's weight.
275 + lora_b_stacked (Tuple[torch.Tensor, ...]): lora_b's weight.
276 + lora_bias_stacked (Optional[Tuple[torch.Tensor, ...]]): lora's bias.
277 + scale (float): Scaling factor.
278 + output_slices (Tuple[int, ...]): Every slice's size.
279 + buffer (Optional[Tuple[torch.Tensor, ...]]): Defaults to None.
280 + """
281 +
282 + assert len(lora_a_stacked) == len(lora_b_stacked) == len(output_slices)
283 + if lora_bias_stacked is not None:
284 + assert len(lora_bias_stacked) == len(output_slices)
285 + y = self._apply_bias(self.token_lora_indices, y, output_slices,
286 + lora_bias_stacked)
287 +
288 + if buffer is None:
289 + r = lora_b_stacked[0].size(-1)
290 + # We set the buffer to be float32 by default, consistent with the
291 + # triton op
292 + buffer = tuple(
293 + torch.zeros(
294 + (x.size(0), r), dtype=torch.float32, device=x.device)
295 + for _ in range(len(output_slices)))
296 + self.add_shrink(buffer, x, lora_a_stacked, scale, **kwargs)
297 + self.add_expand(y,
298 + buffer,
299 + lora_b_stacked,
300 + None,
301 + output_slices,
302 + add_inputs=True,
303 + **kwargs)
304 +
305 + def add_lora_logits(self,
306 + y: torch.Tensor,
307 + x: torch.Tensor,
308 + lora_a_stacked: torch.Tensor,
309 + lora_b_stacked: torch.Tensor,
310 + scale,
311 + *,
312 + buffer: Optional[torch.Tensor] = None,
313 + **kwargs) -> None:
314 + """
315 + Applies lora specifically for LogitsProcessorWithLoRA.
316 +
317 + Semantics:
318 + buffer = (x @ lora_a_stacked) * scale
319 + y += buffer @ lora_b_stacked
320 +
321 + Args:
322 + y (torch.Tensor): Output tensor.
323 + x (torch.Tensor): Input tensor.
324 + lora_a_stacked (torch.Tensor): lora_a's weights.
325 + lora_b_stacked (torch.Tensor):lora_b's weights.
326 + scale (float): Scaling factor.
327 + buffer (Optional[torch.Tensor]):Default to None.
328 + """
329 + y_org = y
330 + y = y.view(-1, y.shape[-1])
331 + x = x.view(-1, x.shape[-1])
332 + r = lora_b_stacked.size(-1)
333 + if buffer is None:
334 + # We set the buffer to be float32 by default, consistent with the
335 + # triton op
336 + buffer = torch.zeros((x.size(0), r),
337 + dtype=torch.float32,
338 + device=x.device)
339 + # LogitsProcessorWithLoRA always using bgmv.
340 + bgmv_shrink(x, lora_a_stacked, buffer, self.sampler_indices, scale)
341 + bgmv_expand(buffer,
342 + lora_b_stacked,
343 + y,
344 + self.sampler_indices,
345 + add_inputs=True)
346 + y = y.view_as(y_org)
vllm/lora/punica_wrapper/punica_gpu.py
+5 -5
@@ -12,11 +12,11 @@ import torch
12 from vllm.triton_utils import HAS_TRITON
13
14 if HAS_TRITON:
15 - from vllm.lora.ops.bgmv_expand import bgmv_expand
16 - from vllm.lora.ops.bgmv_expand_slice import bgmv_expand_slice
17 - from vllm.lora.ops.bgmv_shrink import bgmv_shrink
18 - from vllm.lora.ops.sgmv_expand import sgmv_expand
19 - from vllm.lora.ops.sgmv_shrink import sgmv_shrink
15 + from vllm.lora.ops.triton_ops import bgmv_expand
16 + from vllm.lora.ops.triton_ops import bgmv_expand_slice
17 + from vllm.lora.ops.triton_ops import bgmv_shrink
18 + from vllm.lora.ops.triton_ops import sgmv_expand
19 + from vllm.lora.ops.triton_ops import sgmv_shrink
20
21 from .punica_base import PunicaWrapperBase
22
vllm/lora/punica_wrapper/punica_selector.py
+5
@@ -12,6 +12,11 @@ def get_punica_wrapper(*args, **kwargs) -> PunicaWrapperBase:
12 from vllm.lora.punica_wrapper.punica_gpu import PunicaWrapperGPU
13 logger.info_once("Using PunicaWrapperGPU.")
14 return PunicaWrapperGPU(*args, **kwargs)
15 + elif current_platform.is_cpu():
16 + # Lazy import to avoid ImportError
17 + from vllm.lora.punica_wrapper.punica_cpu import PunicaWrapperCPU
18 + logger.info_once("Using PunicaWrapperCPU.")
19 + return PunicaWrapperCPU(*args, **kwargs)
20 elif current_platform.is_hpu():
21 # Lazy import to avoid ImportError
22 from vllm.lora.punica_wrapper.punica_hpu import PunicaWrapperHPU
vllm/worker/cpu_model_runner.py
+122 -11
@@ -2,8 +2,8 @@ import dataclasses
2 import weakref
3 from collections import defaultdict
4 from dataclasses import dataclass
5 -from typing import (TYPE_CHECKING, Any, Dict, List, Optional, Type, TypeVar,
6 - Union)
5 +from typing import (TYPE_CHECKING, Any, Dict, List, Optional, Set, Type,
6 + TypeVar, Union)
7
8 import torch
9 from torch import nn
@@ -12,10 +12,14 @@ from vllm.attention import AttentionMetadata, get_attn_backend
12 from vllm.config import VllmConfig
13 from vllm.forward_context import set_forward_context
14 from vllm.logger import init_logger
15 +from vllm.lora.layers import LoRAMapping
16 +from vllm.lora.request import LoRARequest
17 +from vllm.lora.worker_manager import LRUCacheWorkerLoRAManager
18 from vllm.model_executor import SamplingMetadata
19 from vllm.model_executor.layers.rotary_embedding import MRotaryEmbedding
20 from vllm.model_executor.layers.sampler import SamplerOutput
21 from vllm.model_executor.model_loader import get_model
22 +from vllm.model_executor.models import supports_lora, supports_multimodal
23 from vllm.multimodal import (MULTIMODAL_REGISTRY, BatchedTensorInputs,
24 MultiModalKwargs, MultiModalPlaceholderMap)
25 from vllm.sequence import (IntermediateTensors, SequenceData,
@@ -49,6 +53,8 @@ class ModelInputForCPU(ModelRunnerInputBase):
53 virtual_engine: Optional[int] = None
54 seq_lens: Optional[List[int]] = None
55 query_lens: Optional[List[int]] = None
56 + lora_mapping: Optional["LoRAMapping"] = None
57 + lora_requests: Optional[Set[LoRARequest]] = None
58
59 def as_broadcastable_tensor_dict(
60 self) -> Dict[str, Union[int, torch.Tensor]]:
@@ -57,6 +63,8 @@ class ModelInputForCPU(ModelRunnerInputBase):
63 "input_positions": self.input_positions,
64 "token_type_ids": self.token_type_ids,
65 "multi_modal_kwargs": self.multi_modal_kwargs,
66 + "lora_requests": self.lora_requests,
67 + "lora_mapping": self.lora_mapping,
68 }
69 _add_attn_metadata_broadcastable_dict(tensor_dict, self.attn_metadata)
70
@@ -143,7 +151,11 @@ class ModelInputForCPUBuilder(ModelRunnerInputBuilderBase[ModelInputForCPU]):
151 or runner.cache_config.enable_prefix_caching)
152 self.model_input_cls = self.runner._model_input_cls
153 self.attn_backend = self.runner.attn_backend
154 + self.sliding_window = self.runner.sliding_window
155 + self.block_size = self.runner.block_size
156 + self.device = self.runner.device
157 self.multi_modal_input_mapper = self.runner.multi_modal_input_mapper
158 + self.enable_lora = self.runner.lora_config is not None
159 self.input_data = ModelInputForCPUBuilder.ModelInputData(
160 self.runner.model_config.uses_mrope)
161 self.att_metadata_builder = self.runner.attn_backend.get_builder_cls()(
@@ -183,15 +195,28 @@ class ModelInputForCPUBuilder(ModelRunnerInputBuilderBase[ModelInputForCPU]):
195 attn_metadata = self.att_metadata_builder.build(
196 input_data.seq_lens, input_data.query_lens, -1, -1)
197
186 - return self.model_input_cls(
187 - input_tokens=input_tokens,
188 - input_positions=input_positions,
189 - token_type_ids=token_type_ids,
190 - seq_lens=input_data.seq_lens,
191 - query_lens=input_data.query_lens,
192 - attn_metadata=attn_metadata,
193 - multi_modal_kwargs=multi_modal_kwargs,
194 - )
198 + is_prompt = (self.seq_group_metadata_list[0].is_prompt
199 + if self.seq_group_metadata_list else None)
200 + # LoRA data.
201 + lora_requests = set()
202 + lora_mapping = None
203 + if self.enable_lora:
204 + lora_requests = set(seq.lora_request
205 + for seq in self.seq_group_metadata_list
206 + if seq.lora_request is not None)
207 +
208 + lora_mapping = self._prepare_lora_input(
209 + self.seq_group_metadata_list, is_prompt)
210 +
211 + return self.model_input_cls(input_tokens=input_tokens,
212 + input_positions=input_positions,
213 + token_type_ids=token_type_ids,
214 + seq_lens=input_data.seq_lens,
215 + query_lens=input_data.query_lens,
216 + attn_metadata=attn_metadata,
217 + multi_modal_kwargs=multi_modal_kwargs,
218 + lora_mapping=lora_mapping,
219 + lora_requests=lora_requests)
220
221 def _build_input_data(self):
222 for seq_group_metadata in self.seq_group_metadata_list:
@@ -381,6 +406,24 @@ class ModelInputForCPUBuilder(ModelRunnerInputBuilderBase[ModelInputForCPU]):
406 self.input_data.multi_modal_placeholder_maps[modality].extend(
407 placeholder_map)
408
409 + def _prepare_lora_input(
410 + self, seq_group_metadata_list: List[SequenceGroupMetadata],
411 + is_prefill: bool) -> LoRAMapping:
412 + index_mapping = []
413 + prompt_mapping = []
414 + for seq in seq_group_metadata_list:
415 + lora_id = seq.lora_int_id
416 + query_len = seq.token_chunk_size
417 +
418 + index_mapping += [lora_id] * query_len
419 + prompt_mapping += [lora_id] * (
420 + query_len if seq.sampling_params
421 + and seq.sampling_params.prompt_logprobs is not None else 1)
422 +
423 + return LoRAMapping(index_mapping=tuple(index_mapping),
424 + prompt_mapping=tuple(prompt_mapping),
425 + is_prefill=is_prefill)
426 +
427
428 class CPUModelRunnerBase(ModelRunnerBase[TModelInputForCPU]):
429 """
@@ -431,10 +474,41 @@ class CPUModelRunnerBase(ModelRunnerBase[TModelInputForCPU]):
474
475 # Lazy initialization.
476 self.model: nn.Module # Set after init_Model
477 + # Set after load_model.
478 + self.lora_manager: Optional[LRUCacheWorkerLoRAManager] = None
479
480 def load_model(self) -> None:
481 self.model = get_model(vllm_config=self.vllm_config)
482
483 + if self.lora_config:
484 + assert supports_lora(
485 + self.model
486 + ), f"{self.model.__class__.__name__} does not support LoRA yet."
487 +
488 + if supports_multimodal(self.model):
489 + logger.warning("Regarding multimodal models, vLLM currently "
490 + "only supports adding LoRA to language model.")
491 +
492 + # It's necessary to distinguish between the max_position_embeddings
493 + # of VLMs and LLMs.
494 + if hasattr(self.model.config, "max_position_embeddings"):
495 + max_pos_embeddings = self.model.config.max_position_embeddings
496 + else:
497 + max_pos_embeddings = (
498 + self.model.config.text_config.max_position_embeddings)
499 +
500 + self.lora_manager = LRUCacheWorkerLoRAManager(
501 + self.scheduler_config.max_num_seqs,
502 + self.scheduler_config.max_num_batched_tokens,
503 + self.vocab_size,
504 + self.lora_config,
505 + self.device,
506 + self.model.embedding_modules,
507 + self.model.embedding_padding_modules,
508 + max_position_embeddings=max_pos_embeddings,
509 + )
510 + self.model = self.lora_manager.create_lora_manager(self.model)
511 +
512 def _prepare_model_input_tensors(
513 self,
514 seq_group_metadata_list: List[SequenceGroupMetadata],
@@ -459,6 +533,37 @@ class CPUModelRunnerBase(ModelRunnerBase[TModelInputForCPU]):
533 def vocab_size(self) -> int:
534 return self.model_config.get_vocab_size()
535
536 + def remove_all_loras(self):
537 + if not self.lora_manager:
538 + raise RuntimeError("LoRA is not enabled.")
539 + self.lora_manager.remove_all_adapters()
540 +
541 + def set_active_loras(self, lora_requests: Set[LoRARequest],
542 + lora_mapping: LoRAMapping) -> None:
543 + if not self.lora_manager:
544 + raise RuntimeError("LoRA is not enabled.")
545 + self.lora_manager.set_active_adapters(lora_requests, lora_mapping)
546 +
547 + def add_lora(self, lora_request: LoRARequest) -> bool:
548 + if not self.lora_manager:
549 + raise RuntimeError("LoRA is not enabled.")
550 + return self.lora_manager.add_adapter(lora_request)
551 +
552 + def remove_lora(self, lora_id: int) -> bool:
553 + if not self.lora_manager:
554 + raise RuntimeError("LoRA is not enabled.")
555 + return self.lora_manager.remove_adapter(lora_id)
556 +
557 + def pin_lora(self, lora_id: int) -> bool:
558 + if not self.lora_manager:
559 + raise RuntimeError("LoRA is not enabled.")
560 + return self.lora_manager.pin_adapter(lora_id)
561 +
562 + def list_loras(self) -> Set[int]:
563 + if not self.lora_manager:
564 + raise RuntimeError("LoRA is not enabled.")
565 + return self.lora_manager.list_adapters()
566 +
567
568 class CPUModelRunner(CPUModelRunnerBase[ModelInputForCPUWithSamplingMetadata]):
569 _model_input_cls: Type[ModelInputForCPUWithSamplingMetadata] = (
@@ -515,6 +620,12 @@ class CPUModelRunner(CPUModelRunnerBase[ModelInputForCPUWithSamplingMetadata]):
620 raise ValueError(
621 "CPU worker does not support multi-step execution.")
622
623 + if self.lora_config:
624 + assert model_input.lora_requests is not None
625 + assert model_input.lora_mapping is not None
626 + self.set_active_loras(model_input.lora_requests,
627 + model_input.lora_mapping)
628 +
629 model_executable = self.model
630
631 multimodal_kwargs = {}
vllm/worker/cpu_worker.py
+16 -4
@@ -1,5 +1,5 @@
1 """A CPU worker class."""
2 -from typing import Dict, List, Optional, Tuple, Type
2 +from typing import Dict, List, Optional, Set, Tuple, Type
3
4 import torch
5 import torch.distributed
@@ -11,14 +11,14 @@ from vllm.config import (CacheConfig, DeviceConfig, ModelConfig,
11 from vllm.distributed import (ensure_model_parallel_initialized,
12 init_distributed_environment)
13 from vllm.logger import init_logger
14 +from vllm.lora.request import LoRARequest
15 from vllm.model_executor import set_random_seed
16 from vllm.sequence import ExecuteModelRequest
17 from vllm.utils import STR_DTYPE_TO_TORCH_DTYPE, bind_kv_cache
18 from vllm.worker.cpu_enc_dec_model_runner import CPUEncoderDecoderModelRunner
19 from vllm.worker.cpu_model_runner import CPUModelRunner, CPUModelRunnerBase
20 from vllm.worker.cpu_pooling_model_runner import CPUPoolingModelRunner
20 -from vllm.worker.worker_base import (LocalOrDistributedWorkerBase,
21 - LoraNotSupportedWorkerBase, WorkerBase,
21 +from vllm.worker.worker_base import (LocalOrDistributedWorkerBase, WorkerBase,
22 WorkerInput)
23
24 logger = init_logger(__name__)
@@ -111,7 +111,7 @@ class CPUCacheEngine:
111 return dtype_size * total
112
113
114 -class CPUWorker(LoraNotSupportedWorkerBase, LocalOrDistributedWorkerBase):
114 +class CPUWorker(LocalOrDistributedWorkerBase):
115 """A worker class that executes (a partition of) the model on a CPU socket.
116
117 Each worker is associated with a single CPU socket. The worker is
@@ -266,6 +266,18 @@ class CPUWorker(LoraNotSupportedWorkerBase, LocalOrDistributedWorkerBase):
266 # Initialize the cache.
267 self._init_cache_engine()
268
269 + def add_lora(self, lora_request: LoRARequest) -> bool:
270 + return self.model_runner.add_lora(lora_request)
271 +
272 + def remove_lora(self, lora_id: int) -> bool:
273 + return self.model_runner.remove_lora(lora_id)
274 +
275 + def pin_lora(self, lora_id: int) -> bool:
276 + return self.model_runner.pin_lora(lora_id)
277 +
278 + def list_loras(self) -> Set[int]:
279 + return self.model_runner.list_loras()
280 +
281 def _validate_num_cpu_blocks(self, num_cpu_blocks: int) -> None:
282 """Raise errors if the num_cpu_blocks is invalid.
283 """