A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 08:15 UTC
[Misc][LoRA] Replace hardcoded cuda device with configurable argument (#10223)
Signed-off-by: Jee Jee Li <pandaleefree@gmail.com>
Jee Jee Li committed
Nov 12, 2024 at 11:10 UTC
7f5edb5900c4010c1daa5bfeb3829974d3f6dff1
6 files changed
+174
-80
tests/lora/test_layers.py
+36
-20
@@ -51,6 +51,7 @@ TOLERANCES = {
51
CUDA_DEVICES = [
52
f"cuda:{i}" for i in range(1 if torch.cuda.device_count() == 1 else 2)
53
]
54
+
55
# We will launch different triton kernels between the prefill and decode
56
# stages, so we need to verify this. prefill stage(True) or decode stage(False)
57
STAGES = [True, False]
@@ -120,11 +121,12 @@ def populate_loras(
121
subloras: List[LoRALayerWeights] = []
122
sublora_len = layer_weights.shape[0] // repeats
123
for i in range(repeats):
123
- sublora = DummyLoRAManager().init_random_lora(
124
- module_name=f"fake_{i}",
125
- weight=layer_weights,
126
- generate_embeddings_tensor=generate_embeddings_tensor,
127
- )
124
+ sublora = DummyLoRAManager(
125
+ layer_weights.device).init_random_lora(
126
+ module_name=f"fake_{i}",
127
+ weight=layer_weights,
128
+ generate_embeddings_tensor=generate_embeddings_tensor,
129
+ )
130
sublora.lora_b = sublora.lora_b[:, (sublora_len *
131
i):(sublora_len * (i + 1))]
132
sublora.optimize()
@@ -152,6 +154,7 @@ def create_random_inputs(
154
input_size: Tuple[int, ...],
155
input_range: Tuple[float, float],
156
input_type: torch.dtype = torch.int,
157
+ device: torch.device = "cuda"
158
) -> Tuple[List[torch.Tensor], List[int], List[int]]:
159
"""Creates random inputs.
160
@@ -173,10 +176,14 @@ def create_random_inputs(
176
for _ in range(num_inputs):
177
if input_type == torch.int:
178
inputs.append(
176
- torch.randint(low=int(low), high=int(high), size=input_size))
179
+ torch.randint(low=int(low),
180
+ high=int(high),
181
+ size=input_size,
182
+ device=device))
183
else:
184
inputs.append(
179
- torch.rand(size=input_size, dtype=input_type) * high + low)
185
+ torch.rand(size=input_size, dtype=input_type, device=device) *
186
+ high + low)
187
188
lora_id = random.choice(active_lora_ids)
189
index_mapping += [lora_id] * input_size[0]
@@ -191,6 +198,10 @@ def create_random_inputs(
198
@pytest.mark.parametrize("vocab_size", [512, 32000, 64000, 128000])
199
@pytest.mark.parametrize("stage", STAGES)
200
def test_embeddings(dist_init, num_loras, device, vocab_size, stage) -> None:
201
+ # For multi-GPU testing of Triton kernel, we must explicitly set the CUDA
202
+ # device, see: https://github.com/triton-lang/triton/issues/2925
203
+ # Same below.
204
+ torch.cuda.set_device(device)
205
206
torch.set_default_device(device)
207
max_loras = 8
@@ -225,7 +236,7 @@ def test_embeddings(dist_init, num_loras, device, vocab_size, stage) -> None:
236
num_inputs=num_loras * 3,
237
input_size=(200, ),
238
input_range=(1, vocab_size),
228
- )
239
+ device=device)
240
lora_mapping = LoRAMapping(index_mapping,
241
prompt_mapping,
242
is_prefill=stage)
@@ -263,7 +274,7 @@ def test_embeddings(dist_init, num_loras, device, vocab_size, stage) -> None:
274
num_inputs=num_loras * 3,
275
input_size=(200, ),
276
input_range=(1, vocab_size),
266
- )
277
+ device=device)
278
lora_mapping = LoRAMapping(index_mapping,
279
prompt_mapping,
280
is_prefill=stage)
@@ -291,6 +302,7 @@ def test_embeddings(dist_init, num_loras, device, vocab_size, stage) -> None:
302
def test_embeddings_with_new_embeddings(dist_init, num_loras, device,
303
vocab_size, stage) -> None:
304
305
+ torch.cuda.set_device(device)
306
torch.set_default_device(device)
307
max_loras = 8
308
punica_wrapper = PunicaWrapper(8192, 256, device)
@@ -345,7 +357,7 @@ def test_embeddings_with_new_embeddings(dist_init, num_loras, device,
357
num_inputs=num_loras * 3,
358
input_size=(200, ),
359
input_range=(1, vocab_size),
348
- )
360
+ device=device)
361
lora_mapping = LoRAMapping(index_mapping,
362
prompt_mapping,
363
is_prefill=stage)
@@ -400,7 +412,7 @@ def test_embeddings_with_new_embeddings(dist_init, num_loras, device,
412
num_inputs=num_loras * 3,
413
input_size=(200, ),
414
input_range=(1, vocab_size),
403
- )
415
+ device=device)
416
original_inputs = deepcopy(inputs)
417
lora_mapping = LoRAMapping(index_mapping,
418
prompt_mapping,
@@ -426,6 +438,7 @@ def test_embeddings_with_new_embeddings(dist_init, num_loras, device,
438
def test_lm_head_logits_processor(dist_init, num_loras, device, vocab_size,
439
stage) -> None:
440
441
+ torch.cuda.set_device(device)
442
torch.set_default_device(device)
443
max_loras = 8
444
punica_wrapper = PunicaWrapper(8192, 256, device)
@@ -471,7 +484,7 @@ def test_lm_head_logits_processor(dist_init, num_loras, device, vocab_size,
484
input_size=(1, 1024),
485
input_range=(0, 1),
486
input_type=torch.float16,
474
- )
487
+ device=device)
488
lora_mapping = LoRAMapping(index_mapping,
489
prompt_mapping,
490
is_prefill=stage)
@@ -520,7 +533,7 @@ def test_lm_head_logits_processor(dist_init, num_loras, device, vocab_size,
533
input_size=(1, 1024),
534
input_range=(0, 1),
535
input_type=torch.float16,
523
- )
536
+ device=device)
537
lora_mapping = LoRAMapping(index_mapping,
538
prompt_mapping,
539
is_prefill=stage)
@@ -554,6 +567,7 @@ def test_lm_head_logits_processor(dist_init, num_loras, device, vocab_size,
567
@pytest.mark.parametrize("stage", STAGES)
568
def test_linear_replicated(dist_init, num_loras, device, stage) -> None:
569
570
+ torch.cuda.set_device(device)
571
torch.set_default_device(device)
572
punica_wrapper = PunicaWrapper(8192, 256, device)
573
max_loras = 8
@@ -592,7 +606,7 @@ def test_linear_replicated(dist_init, num_loras, device, stage) -> None:
606
input_size=(1, 4096),
607
input_range=(0, 1),
608
input_type=torch.float16,
595
- )
609
+ device=device)
610
lora_mapping = LoRAMapping(index_mapping,
611
prompt_mapping,
612
is_prefill=stage)
@@ -631,7 +645,7 @@ def test_linear_replicated(dist_init, num_loras, device, stage) -> None:
645
input_size=(1, 4096),
646
input_range=(0, 1),
647
input_type=torch.float16,
634
- )
648
+ device=device)
649
lora_mapping = LoRAMapping(index_mapping,
650
prompt_mapping,
651
is_prefill=stage)
@@ -658,6 +672,7 @@ def test_linear_replicated(dist_init, num_loras, device, stage) -> None:
672
def test_linear_parallel(dist_init, num_loras, orientation, fully_shard,
673
device, stage) -> None:
674
675
+ torch.cuda.set_device(device)
676
torch.set_default_device(device)
677
punica_wrapper = PunicaWrapper(8192, 256, device)
678
max_loras = 8
@@ -706,7 +721,7 @@ def test_linear_parallel(dist_init, num_loras, orientation, fully_shard,
721
input_size=(1, 4096),
722
input_range=(0, 1),
723
input_type=torch.float16,
709
- )
724
+ device=device)
725
lora_mapping = LoRAMapping(index_mapping,
726
prompt_mapping,
727
is_prefill=stage)
@@ -745,7 +760,7 @@ def test_linear_parallel(dist_init, num_loras, orientation, fully_shard,
760
input_size=(1, 4096),
761
input_range=(0, 1),
762
input_type=torch.float16,
748
- )
763
+ device=device)
764
lora_mapping = LoRAMapping(index_mapping,
765
prompt_mapping,
766
is_prefill=stage)
@@ -772,6 +787,7 @@ def test_linear_parallel(dist_init, num_loras, orientation, fully_shard,
787
def test_column_parallel_packed(dist_init, num_loras, repeats, fully_shard,
788
device, stage) -> None:
789
790
+ torch.cuda.set_device(device)
791
torch.set_default_device(device)
792
punica_wrapper = PunicaWrapper(8192, 256, device)
793
max_loras = 8
@@ -842,7 +858,7 @@ def test_column_parallel_packed(dist_init, num_loras, repeats, fully_shard,
858
input_size=(1, 4096),
859
input_range=(0, 1),
860
input_type=torch.float16,
845
- )
861
+ device=device)
862
lora_mapping = LoRAMapping(index_mapping,
863
prompt_mapping,
864
is_prefill=stage)
@@ -883,7 +899,7 @@ def test_column_parallel_packed(dist_init, num_loras, repeats, fully_shard,
899
input_size=(1, 4096),
900
input_range=(0, 1),
901
input_type=torch.float16,
886
- )
902
+ device=device)
903
lora_mapping = LoRAMapping(index_mapping,
904
prompt_mapping,
905
is_prefill=stage)
@@ -962,7 +978,7 @@ def test_rotary_embedding_long_context(dist_init, num_loras, device,
978
input_size=(1, max_position),
979
input_range=(0, lora_config.lora_extra_vocab_size),
980
input_type=torch.float16,
965
- )
981
+ device=device)
982
983
lora_mapping = LoRAMapping(index_mapping, prompt_mapping)
984
long_lora_context = LongContextLoRAContext(list(scaling_factors),
tests/lora/test_lora_manager.py
+113
-40
@@ -25,8 +25,13 @@ EMBEDDING_MODULES = {
25
26
EMBEDDING_PADDING_MODULES = ["lm_head"]
27
28
+CUDA_DEVICES = [
29
+ f"cuda:{i}" for i in range(1 if torch.cuda.device_count() == 1 else 2)
30
+]
31
29
-def test_from_lora_tensors(sql_lora_files):
32
+
33
+@pytest.mark.parametrize("device", CUDA_DEVICES)
34
+def test_from_lora_tensors(sql_lora_files, device):
35
tensors = load_file(
36
os.path.join(sql_lora_files, "adapter_model.safetensors"))
37
new_embeddings = load_file(
@@ -36,7 +41,7 @@ def test_from_lora_tensors(sql_lora_files):
41
8,
42
16,
43
tensors,
39
- "cuda",
44
+ device,
45
embeddings=new_embeddings,
46
embedding_modules=EMBEDDING_MODULES,
47
embedding_padding_modules=EMBEDDING_PADDING_MODULES)
@@ -46,6 +51,8 @@ def test_from_lora_tensors(sql_lora_files):
51
assert lora.lora_alpha == 16
52
assert lora.lora_a is not None
53
assert lora.lora_b is not None
54
+ assert lora.lora_a.device == torch.device(device)
55
+ assert lora.lora_b.device == torch.device(device)
56
assert (lora.lora_a.shape[1] == lora.lora_b.shape[0]
57
), f"{lora.lora_a.shape=}, {lora.lora_b.shape=}"
58
assert lora.lora_a.shape[1] == 8
@@ -60,8 +67,8 @@ def test_from_lora_tensors(sql_lora_files):
67
assert lora.embeddings_tensor is None
68
69
63
-def create_lora(lora_id: int, model: nn.Module,
64
- sub_modules: List[str]) -> LoRAModel:
70
+def create_lora(lora_id: int, model: nn.Module, sub_modules: List[str],
71
+ device: torch.device) -> LoRAModel:
72
loras: Dict[str, LoRALayerWeights] = {}
73
for name in sub_modules:
74
w = model.get_submodule(name).weight
@@ -69,8 +76,8 @@ def create_lora(lora_id: int, model: nn.Module,
76
name,
77
8,
78
16,
72
- torch.rand([w.shape[1], 8], device="cuda"),
73
- torch.rand([8, w.shape[0]], device="cuda"),
79
+ torch.rand([w.shape[1], 8], device=device),
80
+ torch.rand([8, w.shape[0]], device=device),
81
)
82
return LoRAModel(lora_id, 8, loras)
83
@@ -80,6 +87,7 @@ def create_packed_lora(
87
model: nn.Module,
88
module_name,
89
replaced_module_names,
90
+ device: torch.device,
91
empty_replaced_module_name=None,
92
) -> LoRAModel:
93
w = model.get_submodule(module_name).weight
@@ -91,9 +99,9 @@ def create_packed_lora(
99
replaced_module_name,
100
8,
101
16,
94
- torch.rand([w.shape[1], 8], device="cuda"),
102
+ torch.rand([w.shape[1], 8], device=device),
103
torch.rand([8, w.shape[0] // len(replaced_module_names)],
96
- device="cuda"),
104
+ device=device),
105
)
106
return LoRAModel(lora_id, 8, loras)
107
@@ -104,7 +112,8 @@ def test_replace_submodules(dist_init, dummy_model):
112
model.packed_modules_mapping = {}
113
manager = LoRAModelManager(
114
model, 1, 1, 1,
107
- LoRAConfig(max_lora_rank=8, max_cpu_loras=8, max_loras=8))
115
+ LoRAConfig(max_lora_rank=8, max_cpu_loras=8, max_loras=8),
116
+ torch.device("cuda"))
117
model = manager.model
118
119
assert isinstance(model.get_submodule("dense1"),
@@ -116,16 +125,28 @@ def test_replace_submodules(dist_init, dummy_model):
125
RowParallelLinearWithLoRA)
126
127
119
-def test_lora_model_manager(dist_init, dummy_model):
128
+@pytest.mark.parametrize("device", CUDA_DEVICES)
129
+def test_lora_model_manager(dist_init, dummy_model, device):
130
model = dummy_model
131
model.supported_lora_modules = ["dense1", "dense2", "lm_head"]
132
model.packed_modules_mapping = {}
123
- model_lora1 = create_lora(1, model, ["layer1.dense1", "dense2", "lm_head"])
124
- model_lora2 = create_lora(2, model, ["dense1", "dense2", "lm_head"])
125
- model_lora3 = create_lora(3, model, ["dense1", "dense2", "lm_head"])
126
- manager = LoRAModelManager(
127
- model, 2, 2, 2,
128
- LoRAConfig(max_lora_rank=8, max_cpu_loras=3, max_loras=2))
133
+ model_lora1 = create_lora(1,
134
+ model, ["layer1.dense1", "dense2", "lm_head"],
135
+ device=device)
136
+ model_lora2 = create_lora(2,
137
+ model, ["dense1", "dense2", "lm_head"],
138
+ device=device)
139
+ model_lora3 = create_lora(3,
140
+ model, ["dense1", "dense2", "lm_head"],
141
+ device=device)
142
+ manager = LoRAModelManager(model,
143
+ 2,
144
+ 2,
145
+ 2,
146
+ LoRAConfig(max_lora_rank=8,
147
+ max_cpu_loras=3,
148
+ max_loras=2),
149
+ device=device)
150
assert all(x is None for x in manager.lora_index_to_id)
151
assert manager.add_adapter(model_lora1)
152
assert manager.activate_adapter(1)
@@ -161,17 +182,32 @@ def test_lora_model_manager(dist_init, dummy_model):
182
assert manager.lora_index_to_id[0] == 3
183
assert manager.lora_index_to_id[1] == 2
184
185
+ assert manager.device == device
186
+ assert manager.punica_wrapper.device == device
187
165
-def test_lora_lru_cache_model_manager(dist_init, dummy_model):
188
+
189
+@pytest.mark.parametrize("device", CUDA_DEVICES)
190
+def test_lora_lru_cache_model_manager(dist_init, dummy_model, device):
191
model = dummy_model
192
model.supported_lora_modules = ["dense1", "dense2", "lm_head"]
193
model.packed_modules_mapping = {}
169
- model_lora1 = create_lora(1, model, ["layer1.dense1", "dense2", "lm_head"])
170
- model_lora2 = create_lora(2, model, ["dense1", "dense2", "lm_head"])
171
- model_lora3 = create_lora(3, model, ["dense1", "dense2", "lm_head"])
172
- manager = LRUCacheLoRAModelManager(
173
- model, 2, 2, 2,
174
- LoRAConfig(max_lora_rank=8, max_cpu_loras=3, max_loras=2))
194
+ model_lora1 = create_lora(1,
195
+ model, ["layer1.dense1", "dense2", "lm_head"],
196
+ device=device)
197
+ model_lora2 = create_lora(2,
198
+ model, ["dense1", "dense2", "lm_head"],
199
+ device=device)
200
+ model_lora3 = create_lora(3,
201
+ model, ["dense1", "dense2", "lm_head"],
202
+ device=device)
203
+ manager = LRUCacheLoRAModelManager(model,
204
+ 2,
205
+ 2,
206
+ 2,
207
+ LoRAConfig(max_lora_rank=8,
208
+ max_cpu_loras=3,
209
+ max_loras=2),
210
+ device=device)
211
assert all(x is None for x in manager.lora_index_to_id)
212
assert manager.add_adapter(model_lora1)
213
assert manager.activate_adapter(1)
@@ -238,20 +274,37 @@ def test_lora_lru_cache_model_manager(dist_init, dummy_model):
274
with pytest.raises(ValueError):
275
assert manager.pin_adapter(3)
276
277
+ assert manager.punica_wrapper.device == device
278
+ assert manager.device == device
279
+
280
242
-def test_lru_lora_model_manager(dist_init, dummy_model):
281
+@pytest.mark.parametrize("device", CUDA_DEVICES)
282
+def test_lru_lora_model_manager(dist_init, dummy_model, device):
283
# This tests just the LRU cache functionality, everything else is
284
# tested in test_lora_model_manager
285
model = dummy_model
286
model.supported_lora_modules = ["dense1", "dense2", "lm_head"]
287
model.packed_modules_mapping = {}
248
- model_lora1 = create_lora(1, model, ["layer1.dense1", "dense2", "lm_head"])
249
- model_lora2 = create_lora(2, model, ["dense1", "dense2", "lm_head"])
250
- model_lora3 = create_lora(3, model, ["dense1", "dense2", "lm_head"])
251
- model_lora4 = create_lora(4, model, ["dense1", "dense2", "lm_head"])
252
- manager = LRUCacheLoRAModelManager(
253
- model, 2, 2, 2,
254
- LoRAConfig(max_lora_rank=8, max_cpu_loras=2, max_loras=2))
288
+ model_lora1 = create_lora(1,
289
+ model, ["layer1.dense1", "dense2", "lm_head"],
290
+ device=device)
291
+ model_lora2 = create_lora(2,
292
+ model, ["dense1", "dense2", "lm_head"],
293
+ device=device)
294
+ model_lora3 = create_lora(3,
295
+ model, ["dense1", "dense2", "lm_head"],
296
+ device=device)
297
+ model_lora4 = create_lora(4,
298
+ model, ["dense1", "dense2", "lm_head"],
299
+ device=device)
300
+ manager = LRUCacheLoRAModelManager(model,
301
+ 2,
302
+ 2,
303
+ 2,
304
+ LoRAConfig(max_lora_rank=8,
305
+ max_cpu_loras=2,
306
+ max_loras=2),
307
+ device=device)
308
309
assert all(x is None for x in manager.lora_index_to_id)
310
@@ -351,14 +404,17 @@ def test_lru_lora_model_manager(dist_init, dummy_model):
404
assert manager.remove_oldest_adapter()
405
406
assert set(manager.list_adapters()) == {1}
407
+ assert manager.punica_wrapper.device == device
408
+ assert manager.device == device
409
410
411
+@pytest.mark.parametrize("device", CUDA_DEVICES)
412
def test_lru_cache_worker_adapter_manager(llama_2_7b_model_extra_embeddings,
357
- sql_lora_files):
413
+ sql_lora_files, device):
414
lora_config = LoRAConfig(max_lora_rank=8, max_cpu_loras=4, max_loras=4)
415
worker_adapter_manager = LRUCacheWorkerLoRAManager(
416
4, 2, llama_2_7b_model_extra_embeddings.unpadded_vocab_size -
361
- lora_config.lora_extra_vocab_size, lora_config, torch.device("cuda"),
417
+ lora_config.lora_extra_vocab_size, lora_config, device,
418
EMBEDDING_MODULES, EMBEDDING_PADDING_MODULES)
419
worker_adapter_manager.create_lora_manager(
420
llama_2_7b_model_extra_embeddings)
@@ -426,14 +482,19 @@ def test_lru_cache_worker_adapter_manager(llama_2_7b_model_extra_embeddings,
482
LoRARequest("14", 14, sql_lora_files)
483
], mapping)
484
485
+ assert worker_adapter_manager.device == device
486
+ assert (worker_adapter_manager._adapter_manager.punica_wrapper.device ==
487
+ device)
488
489
+
490
+@pytest.mark.parametrize("device", CUDA_DEVICES)
491
def test_worker_adapter_manager(llama_2_7b_model_extra_embeddings,
431
- sql_lora_files):
492
+ sql_lora_files, device):
493
# Should remove every LoRA not specified in the request.
494
lora_config = LoRAConfig(max_lora_rank=8, max_cpu_loras=4, max_loras=4)
495
worker_adapter_manager = WorkerLoRAManager(
496
4, 2, llama_2_7b_model_extra_embeddings.unpadded_vocab_size -
436
- lora_config.lora_extra_vocab_size, lora_config, torch.device("cuda"),
497
+ lora_config.lora_extra_vocab_size, lora_config, device,
498
EMBEDDING_MODULES, EMBEDDING_PADDING_MODULES)
499
worker_adapter_manager.create_lora_manager(
500
llama_2_7b_model_extra_embeddings)
@@ -497,8 +558,13 @@ def test_worker_adapter_manager(llama_2_7b_model_extra_embeddings,
558
LoRARequest("14", 14, sql_lora_files)
559
], mapping)
560
561
+ assert worker_adapter_manager.device == device
562
+ assert (worker_adapter_manager._adapter_manager.punica_wrapper.device ==
563
+ device)
564
+
565
501
-def test_packed_loras(dist_init, dummy_model_gate_up):
566
+@pytest.mark.parametrize("device", CUDA_DEVICES)
567
+def test_packed_loras(dist_init, dummy_model_gate_up, device):
568
model = dummy_model_gate_up
569
model.supported_lora_modules = ["gate_up_proj"]
570
model.packed_modules_mapping = {
@@ -511,18 +577,25 @@ def test_packed_loras(dist_init, dummy_model_gate_up):
577
1,
578
model,
579
module_name="gate_up_proj",
514
- replaced_module_names=["gate_proj", "up_proj"])
580
+ replaced_module_names=["gate_proj", "up_proj"],
581
+ device=device)
582
model_lora1 = create_packed_lora(
583
2,
584
model,
585
module_name="gate_up_proj",
586
replaced_module_names=["gate_proj", "up_proj"],
587
+ device=device,
588
empty_replaced_module_name="gate_proj",
589
)
590
523
- manager = LoRAModelManager(
524
- model, 2, 2, 2,
525
- LoRAConfig(max_lora_rank=8, max_cpu_loras=2, max_loras=2))
591
+ manager = LoRAModelManager(model,
592
+ 2,
593
+ 2,
594
+ 2,
595
+ LoRAConfig(max_lora_rank=8,
596
+ max_cpu_loras=2,
597
+ max_loras=2),
598
+ device=device)
599
model = manager.model
600
601
assert isinstance(model.get_submodule("gate_up_proj"),
tests/lora/utils.py
+5
-4
@@ -7,9 +7,10 @@ from vllm.lora.lora import LoRALayerWeights, PackedLoRALayerWeights
7
8
class DummyLoRAManager:
9
10
- def __init__(self):
10
+ def __init__(self, device: torch.device = "cuda:0"):
11
super().__init__()
12
self._loras: Dict[str, LoRALayerWeights] = {}
13
+ self._device = device
14
15
def set_module_lora(self, module_name: str, lora: LoRALayerWeights):
16
self._loras[module_name] = lora
@@ -28,16 +29,16 @@ class DummyLoRAManager:
29
lora_alpha=1,
30
lora_a=torch.rand([weight.shape[1], rank],
31
dtype=weight.dtype,
31
- device="cuda"),
32
+ device=self._device),
33
lora_b=torch.rand([rank, weight.shape[0]],
34
dtype=weight.dtype,
34
- device="cuda"),
35
+ device=self._device),
36
)
37
if generate_embeddings_tensor:
38
lora.embeddings_tensor = torch.rand(5,
39
generate_embeddings_tensor,
40
dtype=weight.dtype,
40
- device="cuda")
41
+ device=self._device)
42
self.set_module_lora(module_name, lora)
43
44
return lora
vllm/lora/models.py
+9
-10
@@ -301,6 +301,7 @@ class LoRAModelManager(AdapterModelManager):
301
max_num_batched_tokens: int,
302
vocab_size: int,
303
lora_config: LoRAConfig,
304
+ device: torch.device,
305
):
306
"""Create a LoRAModelManager and adapter for a given model.
307
@@ -314,6 +315,7 @@ class LoRAModelManager(AdapterModelManager):
315
lora_config: the LoRA configuration.
316
"""
317
self.lora_config = lora_config
318
+ self.device = device
319
self.max_num_seqs = max_num_seqs
320
assert self.capacity >= self.lora_slots
321
self.max_num_batched_tokens = math.ceil(max_num_batched_tokens / 8) * 8
@@ -322,7 +324,7 @@ class LoRAModelManager(AdapterModelManager):
324
self.long_lora_context: Optional[LongContextLoRAContext] = None
325
self.punica_wrapper = PunicaWrapper(max_num_batched_tokens,
326
max_batches=self.max_num_seqs,
325
- device="cuda")
327
+ device=self.device)
328
# Scaling factor -> offset to the sin_cos_cache to it.
329
# Used for long context lora.
330
self.scaling_factor_to_offset: Dict[float, int] = {}
@@ -653,16 +655,11 @@ class LoRALRUCache(AdapterLRUCache[LoRAModel]):
655
class LRUCacheLoRAModelManager(LoRAModelManager):
656
"""A model manager that manages multiple LoRAs with LRU cache."""
657
656
- def __init__(
657
- self,
658
- model: nn.Module,
659
- max_num_seqs: int,
660
- max_num_batched_tokens: int,
661
- vocab_size: int,
662
- lora_config: LoRAConfig,
663
- ):
658
+ def __init__(self, model: nn.Module, max_num_seqs: int,
659
+ max_num_batched_tokens: int, vocab_size: int,
660
+ lora_config: LoRAConfig, device: torch.device):
661
super().__init__(model, max_num_seqs, max_num_batched_tokens,
665
- vocab_size, lora_config)
662
+ vocab_size, lora_config, device)
663
self._registered_adapters: LoRALRUCache = LoRALRUCache(
664
self.capacity, self.deactivate_adapter)
665
self._active_adapters: LoRALRUCache = LoRALRUCache(
@@ -732,6 +729,7 @@ def create_lora_manager(
729
max_num_batched_tokens: int,
730
vocab_size: int,
731
lora_config: LoRAConfig,
732
+ device: torch.device,
733
lora_manager_cls: Type[LoRAModelManager] = LoRAModelManager,
734
**kwargs) -> LoRAModelManager:
735
"""Create a LoRA adapter for a given model."""
@@ -743,5 +741,6 @@ def create_lora_manager(
741
max_num_batched_tokens=max_num_batched_tokens,
742
vocab_size=vocab_size,
743
lora_config=lora_config,
744
+ device=device,
745
**kwargs)
746
return lora_manager
vllm/lora/punica.py
+9
-6
@@ -62,6 +62,7 @@ def convert_mapping(
62
max_loras: int,
63
vocab_size: int,
64
extra_vocab_size: int,
65
+ device: torch.device,
66
long_lora_context: Optional["LongContextLoRAContext"] = None,
67
) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor,
68
Optional[torch.Tensor], List[int]]:
@@ -104,7 +105,7 @@ def convert_mapping(
105
long_lora_offsets: Optional[torch.Tensor] = None
106
if long_lora_context:
107
long_lora_offsets = torch.zeros(len(index_mapping_indices),
107
- device="cuda",
108
+ device=device,
109
dtype=torch.long)
110
prompt_mapping: List[int] = [
111
lora_index_to_id.index(x) if x > 0 else -1
@@ -131,10 +132,10 @@ def convert_mapping(
132
if long_lora_context:
133
assert long_lora_offsets is not None
134
indices_list.append(long_lora_offsets)
134
- indices = torch.tensor(indices_list, dtype=torch.long, device="cuda")
135
+ indices = torch.tensor(indices_list, dtype=torch.long, device=device)
136
prompt_mapping_tensor = torch.tensor(prompt_mapping,
136
- device="cuda",
137
- dtype=torch.long)
137
+ dtype=torch.long,
138
+ device=device)
139
embeddings_indices = torch.stack([
140
indices[2] * extra_vocab_size,
141
indices[2] * (vocab_size + extra_vocab_size),
@@ -145,7 +146,7 @@ def convert_mapping(
146
sampler_indices_padded = sampler_indices.clone()
147
sampler_indices_padded[sampler_indices_padded == -1] = max_loras - 1
148
sampler_indices_padded = torch.arange(
148
- 0, len(sampler_indices_padded), device="cuda", dtype=torch.long) + (
149
+ 0, len(sampler_indices_padded), device=device, dtype=torch.long) + (
150
sampler_indices_padded * len(sampler_indices_padded))
151
long_lora_indices = None
152
long_lora_indices_len: Optional[int] = None
@@ -183,7 +184,7 @@ class PunicaWrapper:
184
"""
185
186
def __init__(self, max_num_batched_tokens: int, max_batches: int,
186
- device: str):
187
+ device: Union[torch.device, str]):
188
self._token_lora_indices = torch.empty(max_num_batched_tokens,
189
dtype=torch.long,
190
device=device)
@@ -215,6 +216,7 @@ class PunicaWrapper:
216
self._lora_indices_per_batch = torch.empty(max_batches,
217
dtype=torch.long,
218
device=device)
219
+ self.device: torch.device = device
220
self.max_length: int = 0
221
self.token_nums: int = 0
222
self.batch_size: int = -1
@@ -263,6 +265,7 @@ class PunicaWrapper:
265
max_loras,
266
vocab_size,
267
extra_vocab_size,
268
+ self.device,
269
long_lora_context,
270
)
271
self._token_lora_indices[:base_indices.shape[0]].copy_(base_indices)
vllm/lora/worker_manager.py
+2
@@ -73,6 +73,7 @@ class WorkerLoRAManager(AbstractWorkerManager):
73
max_num_batched_tokens=self.max_num_batched_tokens,
74
vocab_size=self.vocab_size,
75
lora_config=self.lora_config,
76
+ device=self.device,
77
lora_manager_cls=self._manager_cls,
78
)
79
self._adapter_manager = lora_manager
@@ -176,6 +177,7 @@ class LRUCacheWorkerLoRAManager(WorkerLoRAManager):
177
max_num_seqs=self.max_num_seqs,
178
vocab_size=self.vocab_size,
179
lora_config=self.lora_config,
180
+ device=self.device,
181
max_num_batched_tokens=self.max_num_batched_tokens,
182
)
183
self._adapter_manager = lora_manager