@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 7f5edb5900
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 08:15 UTC

[Misc][LoRA] Replace hardcoded cuda device with configurable argument (#10223)

Signed-off-by: Jee Jee Li <pandaleefree@gmail.com>

Jee Jee Li committed Nov 12, 2024 at 11:10 UTC 7f5edb5900c4010c1daa5bfeb3829974d3f6dff1
6 files changed +174 -80
tests/lora/test_layers.py
+36 -20
@@ -51,6 +51,7 @@ TOLERANCES = {
51 CUDA_DEVICES = [
52 f"cuda:{i}" for i in range(1 if torch.cuda.device_count() == 1 else 2)
53 ]
54 +
55 # We will launch different triton kernels between the prefill and decode
56 # stages, so we need to verify this. prefill stage(True) or decode stage(False)
57 STAGES = [True, False]
@@ -120,11 +121,12 @@ def populate_loras(
121 subloras: List[LoRALayerWeights] = []
122 sublora_len = layer_weights.shape[0] // repeats
123 for i in range(repeats):
123 - sublora = DummyLoRAManager().init_random_lora(
124 - module_name=f"fake_{i}",
125 - weight=layer_weights,
126 - generate_embeddings_tensor=generate_embeddings_tensor,
127 - )
124 + sublora = DummyLoRAManager(
125 + layer_weights.device).init_random_lora(
126 + module_name=f"fake_{i}",
127 + weight=layer_weights,
128 + generate_embeddings_tensor=generate_embeddings_tensor,
129 + )
130 sublora.lora_b = sublora.lora_b[:, (sublora_len *
131 i):(sublora_len * (i + 1))]
132 sublora.optimize()
@@ -152,6 +154,7 @@ def create_random_inputs(
154 input_size: Tuple[int, ...],
155 input_range: Tuple[float, float],
156 input_type: torch.dtype = torch.int,
157 + device: torch.device = "cuda"
158 ) -> Tuple[List[torch.Tensor], List[int], List[int]]:
159 """Creates random inputs.
160
@@ -173,10 +176,14 @@ def create_random_inputs(
176 for _ in range(num_inputs):
177 if input_type == torch.int:
178 inputs.append(
176 - torch.randint(low=int(low), high=int(high), size=input_size))
179 + torch.randint(low=int(low),
180 + high=int(high),
181 + size=input_size,
182 + device=device))
183 else:
184 inputs.append(
179 - torch.rand(size=input_size, dtype=input_type) * high + low)
185 + torch.rand(size=input_size, dtype=input_type, device=device) *
186 + high + low)
187
188 lora_id = random.choice(active_lora_ids)
189 index_mapping += [lora_id] * input_size[0]
@@ -191,6 +198,10 @@ def create_random_inputs(
198 @pytest.mark.parametrize("vocab_size", [512, 32000, 64000, 128000])
199 @pytest.mark.parametrize("stage", STAGES)
200 def test_embeddings(dist_init, num_loras, device, vocab_size, stage) -> None:
201 + # For multi-GPU testing of Triton kernel, we must explicitly set the CUDA
202 + # device, see: https://github.com/triton-lang/triton/issues/2925
203 + # Same below.
204 + torch.cuda.set_device(device)
205
206 torch.set_default_device(device)
207 max_loras = 8
@@ -225,7 +236,7 @@ def test_embeddings(dist_init, num_loras, device, vocab_size, stage) -> None:
236 num_inputs=num_loras * 3,
237 input_size=(200, ),
238 input_range=(1, vocab_size),
228 - )
239 + device=device)
240 lora_mapping = LoRAMapping(index_mapping,
241 prompt_mapping,
242 is_prefill=stage)
@@ -263,7 +274,7 @@ def test_embeddings(dist_init, num_loras, device, vocab_size, stage) -> None:
274 num_inputs=num_loras * 3,
275 input_size=(200, ),
276 input_range=(1, vocab_size),
266 - )
277 + device=device)
278 lora_mapping = LoRAMapping(index_mapping,
279 prompt_mapping,
280 is_prefill=stage)
@@ -291,6 +302,7 @@ def test_embeddings(dist_init, num_loras, device, vocab_size, stage) -> None:
302 def test_embeddings_with_new_embeddings(dist_init, num_loras, device,
303 vocab_size, stage) -> None:
304
305 + torch.cuda.set_device(device)
306 torch.set_default_device(device)
307 max_loras = 8
308 punica_wrapper = PunicaWrapper(8192, 256, device)
@@ -345,7 +357,7 @@ def test_embeddings_with_new_embeddings(dist_init, num_loras, device,
357 num_inputs=num_loras * 3,
358 input_size=(200, ),
359 input_range=(1, vocab_size),
348 - )
360 + device=device)
361 lora_mapping = LoRAMapping(index_mapping,
362 prompt_mapping,
363 is_prefill=stage)
@@ -400,7 +412,7 @@ def test_embeddings_with_new_embeddings(dist_init, num_loras, device,
412 num_inputs=num_loras * 3,
413 input_size=(200, ),
414 input_range=(1, vocab_size),
403 - )
415 + device=device)
416 original_inputs = deepcopy(inputs)
417 lora_mapping = LoRAMapping(index_mapping,
418 prompt_mapping,
@@ -426,6 +438,7 @@ def test_embeddings_with_new_embeddings(dist_init, num_loras, device,
438 def test_lm_head_logits_processor(dist_init, num_loras, device, vocab_size,
439 stage) -> None:
440
441 + torch.cuda.set_device(device)
442 torch.set_default_device(device)
443 max_loras = 8
444 punica_wrapper = PunicaWrapper(8192, 256, device)
@@ -471,7 +484,7 @@ def test_lm_head_logits_processor(dist_init, num_loras, device, vocab_size,
484 input_size=(1, 1024),
485 input_range=(0, 1),
486 input_type=torch.float16,
474 - )
487 + device=device)
488 lora_mapping = LoRAMapping(index_mapping,
489 prompt_mapping,
490 is_prefill=stage)
@@ -520,7 +533,7 @@ def test_lm_head_logits_processor(dist_init, num_loras, device, vocab_size,
533 input_size=(1, 1024),
534 input_range=(0, 1),
535 input_type=torch.float16,
523 - )
536 + device=device)
537 lora_mapping = LoRAMapping(index_mapping,
538 prompt_mapping,
539 is_prefill=stage)
@@ -554,6 +567,7 @@ def test_lm_head_logits_processor(dist_init, num_loras, device, vocab_size,
567 @pytest.mark.parametrize("stage", STAGES)
568 def test_linear_replicated(dist_init, num_loras, device, stage) -> None:
569
570 + torch.cuda.set_device(device)
571 torch.set_default_device(device)
572 punica_wrapper = PunicaWrapper(8192, 256, device)
573 max_loras = 8
@@ -592,7 +606,7 @@ def test_linear_replicated(dist_init, num_loras, device, stage) -> None:
606 input_size=(1, 4096),
607 input_range=(0, 1),
608 input_type=torch.float16,
595 - )
609 + device=device)
610 lora_mapping = LoRAMapping(index_mapping,
611 prompt_mapping,
612 is_prefill=stage)
@@ -631,7 +645,7 @@ def test_linear_replicated(dist_init, num_loras, device, stage) -> None:
645 input_size=(1, 4096),
646 input_range=(0, 1),
647 input_type=torch.float16,
634 - )
648 + device=device)
649 lora_mapping = LoRAMapping(index_mapping,
650 prompt_mapping,
651 is_prefill=stage)
@@ -658,6 +672,7 @@ def test_linear_replicated(dist_init, num_loras, device, stage) -> None:
672 def test_linear_parallel(dist_init, num_loras, orientation, fully_shard,
673 device, stage) -> None:
674
675 + torch.cuda.set_device(device)
676 torch.set_default_device(device)
677 punica_wrapper = PunicaWrapper(8192, 256, device)
678 max_loras = 8
@@ -706,7 +721,7 @@ def test_linear_parallel(dist_init, num_loras, orientation, fully_shard,
721 input_size=(1, 4096),
722 input_range=(0, 1),
723 input_type=torch.float16,
709 - )
724 + device=device)
725 lora_mapping = LoRAMapping(index_mapping,
726 prompt_mapping,
727 is_prefill=stage)
@@ -745,7 +760,7 @@ def test_linear_parallel(dist_init, num_loras, orientation, fully_shard,
760 input_size=(1, 4096),
761 input_range=(0, 1),
762 input_type=torch.float16,
748 - )
763 + device=device)
764 lora_mapping = LoRAMapping(index_mapping,
765 prompt_mapping,
766 is_prefill=stage)
@@ -772,6 +787,7 @@ def test_linear_parallel(dist_init, num_loras, orientation, fully_shard,
787 def test_column_parallel_packed(dist_init, num_loras, repeats, fully_shard,
788 device, stage) -> None:
789
790 + torch.cuda.set_device(device)
791 torch.set_default_device(device)
792 punica_wrapper = PunicaWrapper(8192, 256, device)
793 max_loras = 8
@@ -842,7 +858,7 @@ def test_column_parallel_packed(dist_init, num_loras, repeats, fully_shard,
858 input_size=(1, 4096),
859 input_range=(0, 1),
860 input_type=torch.float16,
845 - )
861 + device=device)
862 lora_mapping = LoRAMapping(index_mapping,
863 prompt_mapping,
864 is_prefill=stage)
@@ -883,7 +899,7 @@ def test_column_parallel_packed(dist_init, num_loras, repeats, fully_shard,
899 input_size=(1, 4096),
900 input_range=(0, 1),
901 input_type=torch.float16,
886 - )
902 + device=device)
903 lora_mapping = LoRAMapping(index_mapping,
904 prompt_mapping,
905 is_prefill=stage)
@@ -962,7 +978,7 @@ def test_rotary_embedding_long_context(dist_init, num_loras, device,
978 input_size=(1, max_position),
979 input_range=(0, lora_config.lora_extra_vocab_size),
980 input_type=torch.float16,
965 - )
981 + device=device)
982
983 lora_mapping = LoRAMapping(index_mapping, prompt_mapping)
984 long_lora_context = LongContextLoRAContext(list(scaling_factors),
tests/lora/test_lora_manager.py
+113 -40
@@ -25,8 +25,13 @@ EMBEDDING_MODULES = {
25
26 EMBEDDING_PADDING_MODULES = ["lm_head"]
27
28 +CUDA_DEVICES = [
29 + f"cuda:{i}" for i in range(1 if torch.cuda.device_count() == 1 else 2)
30 +]
31
29 -def test_from_lora_tensors(sql_lora_files):
32 +
33 +@pytest.mark.parametrize("device", CUDA_DEVICES)
34 +def test_from_lora_tensors(sql_lora_files, device):
35 tensors = load_file(
36 os.path.join(sql_lora_files, "adapter_model.safetensors"))
37 new_embeddings = load_file(
@@ -36,7 +41,7 @@ def test_from_lora_tensors(sql_lora_files):
41 8,
42 16,
43 tensors,
39 - "cuda",
44 + device,
45 embeddings=new_embeddings,
46 embedding_modules=EMBEDDING_MODULES,
47 embedding_padding_modules=EMBEDDING_PADDING_MODULES)
@@ -46,6 +51,8 @@ def test_from_lora_tensors(sql_lora_files):
51 assert lora.lora_alpha == 16
52 assert lora.lora_a is not None
53 assert lora.lora_b is not None
54 + assert lora.lora_a.device == torch.device(device)
55 + assert lora.lora_b.device == torch.device(device)
56 assert (lora.lora_a.shape[1] == lora.lora_b.shape[0]
57 ), f"{lora.lora_a.shape=}, {lora.lora_b.shape=}"
58 assert lora.lora_a.shape[1] == 8
@@ -60,8 +67,8 @@ def test_from_lora_tensors(sql_lora_files):
67 assert lora.embeddings_tensor is None
68
69
63 -def create_lora(lora_id: int, model: nn.Module,
64 - sub_modules: List[str]) -> LoRAModel:
70 +def create_lora(lora_id: int, model: nn.Module, sub_modules: List[str],
71 + device: torch.device) -> LoRAModel:
72 loras: Dict[str, LoRALayerWeights] = {}
73 for name in sub_modules:
74 w = model.get_submodule(name).weight
@@ -69,8 +76,8 @@ def create_lora(lora_id: int, model: nn.Module,
76 name,
77 8,
78 16,
72 - torch.rand([w.shape[1], 8], device="cuda"),
73 - torch.rand([8, w.shape[0]], device="cuda"),
79 + torch.rand([w.shape[1], 8], device=device),
80 + torch.rand([8, w.shape[0]], device=device),
81 )
82 return LoRAModel(lora_id, 8, loras)
83
@@ -80,6 +87,7 @@ def create_packed_lora(
87 model: nn.Module,
88 module_name,
89 replaced_module_names,
90 + device: torch.device,
91 empty_replaced_module_name=None,
92 ) -> LoRAModel:
93 w = model.get_submodule(module_name).weight
@@ -91,9 +99,9 @@ def create_packed_lora(
99 replaced_module_name,
100 8,
101 16,
94 - torch.rand([w.shape[1], 8], device="cuda"),
102 + torch.rand([w.shape[1], 8], device=device),
103 torch.rand([8, w.shape[0] // len(replaced_module_names)],
96 - device="cuda"),
104 + device=device),
105 )
106 return LoRAModel(lora_id, 8, loras)
107
@@ -104,7 +112,8 @@ def test_replace_submodules(dist_init, dummy_model):
112 model.packed_modules_mapping = {}
113 manager = LoRAModelManager(
114 model, 1, 1, 1,
107 - LoRAConfig(max_lora_rank=8, max_cpu_loras=8, max_loras=8))
115 + LoRAConfig(max_lora_rank=8, max_cpu_loras=8, max_loras=8),
116 + torch.device("cuda"))
117 model = manager.model
118
119 assert isinstance(model.get_submodule("dense1"),
@@ -116,16 +125,28 @@ def test_replace_submodules(dist_init, dummy_model):
125 RowParallelLinearWithLoRA)
126
127
119 -def test_lora_model_manager(dist_init, dummy_model):
128 +@pytest.mark.parametrize("device", CUDA_DEVICES)
129 +def test_lora_model_manager(dist_init, dummy_model, device):
130 model = dummy_model
131 model.supported_lora_modules = ["dense1", "dense2", "lm_head"]
132 model.packed_modules_mapping = {}
123 - model_lora1 = create_lora(1, model, ["layer1.dense1", "dense2", "lm_head"])
124 - model_lora2 = create_lora(2, model, ["dense1", "dense2", "lm_head"])
125 - model_lora3 = create_lora(3, model, ["dense1", "dense2", "lm_head"])
126 - manager = LoRAModelManager(
127 - model, 2, 2, 2,
128 - LoRAConfig(max_lora_rank=8, max_cpu_loras=3, max_loras=2))
133 + model_lora1 = create_lora(1,
134 + model, ["layer1.dense1", "dense2", "lm_head"],
135 + device=device)
136 + model_lora2 = create_lora(2,
137 + model, ["dense1", "dense2", "lm_head"],
138 + device=device)
139 + model_lora3 = create_lora(3,
140 + model, ["dense1", "dense2", "lm_head"],
141 + device=device)
142 + manager = LoRAModelManager(model,
143 + 2,
144 + 2,
145 + 2,
146 + LoRAConfig(max_lora_rank=8,
147 + max_cpu_loras=3,
148 + max_loras=2),
149 + device=device)
150 assert all(x is None for x in manager.lora_index_to_id)
151 assert manager.add_adapter(model_lora1)
152 assert manager.activate_adapter(1)
@@ -161,17 +182,32 @@ def test_lora_model_manager(dist_init, dummy_model):
182 assert manager.lora_index_to_id[0] == 3
183 assert manager.lora_index_to_id[1] == 2
184
185 + assert manager.device == device
186 + assert manager.punica_wrapper.device == device
187
165 -def test_lora_lru_cache_model_manager(dist_init, dummy_model):
188 +
189 +@pytest.mark.parametrize("device", CUDA_DEVICES)
190 +def test_lora_lru_cache_model_manager(dist_init, dummy_model, device):
191 model = dummy_model
192 model.supported_lora_modules = ["dense1", "dense2", "lm_head"]
193 model.packed_modules_mapping = {}
169 - model_lora1 = create_lora(1, model, ["layer1.dense1", "dense2", "lm_head"])
170 - model_lora2 = create_lora(2, model, ["dense1", "dense2", "lm_head"])
171 - model_lora3 = create_lora(3, model, ["dense1", "dense2", "lm_head"])
172 - manager = LRUCacheLoRAModelManager(
173 - model, 2, 2, 2,
174 - LoRAConfig(max_lora_rank=8, max_cpu_loras=3, max_loras=2))
194 + model_lora1 = create_lora(1,
195 + model, ["layer1.dense1", "dense2", "lm_head"],
196 + device=device)
197 + model_lora2 = create_lora(2,
198 + model, ["dense1", "dense2", "lm_head"],
199 + device=device)
200 + model_lora3 = create_lora(3,
201 + model, ["dense1", "dense2", "lm_head"],
202 + device=device)
203 + manager = LRUCacheLoRAModelManager(model,
204 + 2,
205 + 2,
206 + 2,
207 + LoRAConfig(max_lora_rank=8,
208 + max_cpu_loras=3,
209 + max_loras=2),
210 + device=device)
211 assert all(x is None for x in manager.lora_index_to_id)
212 assert manager.add_adapter(model_lora1)
213 assert manager.activate_adapter(1)
@@ -238,20 +274,37 @@ def test_lora_lru_cache_model_manager(dist_init, dummy_model):
274 with pytest.raises(ValueError):
275 assert manager.pin_adapter(3)
276
277 + assert manager.punica_wrapper.device == device
278 + assert manager.device == device
279 +
280
242 -def test_lru_lora_model_manager(dist_init, dummy_model):
281 +@pytest.mark.parametrize("device", CUDA_DEVICES)
282 +def test_lru_lora_model_manager(dist_init, dummy_model, device):
283 # This tests just the LRU cache functionality, everything else is
284 # tested in test_lora_model_manager
285 model = dummy_model
286 model.supported_lora_modules = ["dense1", "dense2", "lm_head"]
287 model.packed_modules_mapping = {}
248 - model_lora1 = create_lora(1, model, ["layer1.dense1", "dense2", "lm_head"])
249 - model_lora2 = create_lora(2, model, ["dense1", "dense2", "lm_head"])
250 - model_lora3 = create_lora(3, model, ["dense1", "dense2", "lm_head"])
251 - model_lora4 = create_lora(4, model, ["dense1", "dense2", "lm_head"])
252 - manager = LRUCacheLoRAModelManager(
253 - model, 2, 2, 2,
254 - LoRAConfig(max_lora_rank=8, max_cpu_loras=2, max_loras=2))
288 + model_lora1 = create_lora(1,
289 + model, ["layer1.dense1", "dense2", "lm_head"],
290 + device=device)
291 + model_lora2 = create_lora(2,
292 + model, ["dense1", "dense2", "lm_head"],
293 + device=device)
294 + model_lora3 = create_lora(3,
295 + model, ["dense1", "dense2", "lm_head"],
296 + device=device)
297 + model_lora4 = create_lora(4,
298 + model, ["dense1", "dense2", "lm_head"],
299 + device=device)
300 + manager = LRUCacheLoRAModelManager(model,
301 + 2,
302 + 2,
303 + 2,
304 + LoRAConfig(max_lora_rank=8,
305 + max_cpu_loras=2,
306 + max_loras=2),
307 + device=device)
308
309 assert all(x is None for x in manager.lora_index_to_id)
310
@@ -351,14 +404,17 @@ def test_lru_lora_model_manager(dist_init, dummy_model):
404 assert manager.remove_oldest_adapter()
405
406 assert set(manager.list_adapters()) == {1}
407 + assert manager.punica_wrapper.device == device
408 + assert manager.device == device
409
410
411 +@pytest.mark.parametrize("device", CUDA_DEVICES)
412 def test_lru_cache_worker_adapter_manager(llama_2_7b_model_extra_embeddings,
357 - sql_lora_files):
413 + sql_lora_files, device):
414 lora_config = LoRAConfig(max_lora_rank=8, max_cpu_loras=4, max_loras=4)
415 worker_adapter_manager = LRUCacheWorkerLoRAManager(
416 4, 2, llama_2_7b_model_extra_embeddings.unpadded_vocab_size -
361 - lora_config.lora_extra_vocab_size, lora_config, torch.device("cuda"),
417 + lora_config.lora_extra_vocab_size, lora_config, device,
418 EMBEDDING_MODULES, EMBEDDING_PADDING_MODULES)
419 worker_adapter_manager.create_lora_manager(
420 llama_2_7b_model_extra_embeddings)
@@ -426,14 +482,19 @@ def test_lru_cache_worker_adapter_manager(llama_2_7b_model_extra_embeddings,
482 LoRARequest("14", 14, sql_lora_files)
483 ], mapping)
484
485 + assert worker_adapter_manager.device == device
486 + assert (worker_adapter_manager._adapter_manager.punica_wrapper.device ==
487 + device)
488
489 +
490 +@pytest.mark.parametrize("device", CUDA_DEVICES)
491 def test_worker_adapter_manager(llama_2_7b_model_extra_embeddings,
431 - sql_lora_files):
492 + sql_lora_files, device):
493 # Should remove every LoRA not specified in the request.
494 lora_config = LoRAConfig(max_lora_rank=8, max_cpu_loras=4, max_loras=4)
495 worker_adapter_manager = WorkerLoRAManager(
496 4, 2, llama_2_7b_model_extra_embeddings.unpadded_vocab_size -
436 - lora_config.lora_extra_vocab_size, lora_config, torch.device("cuda"),
497 + lora_config.lora_extra_vocab_size, lora_config, device,
498 EMBEDDING_MODULES, EMBEDDING_PADDING_MODULES)
499 worker_adapter_manager.create_lora_manager(
500 llama_2_7b_model_extra_embeddings)
@@ -497,8 +558,13 @@ def test_worker_adapter_manager(llama_2_7b_model_extra_embeddings,
558 LoRARequest("14", 14, sql_lora_files)
559 ], mapping)
560
561 + assert worker_adapter_manager.device == device
562 + assert (worker_adapter_manager._adapter_manager.punica_wrapper.device ==
563 + device)
564 +
565
501 -def test_packed_loras(dist_init, dummy_model_gate_up):
566 +@pytest.mark.parametrize("device", CUDA_DEVICES)
567 +def test_packed_loras(dist_init, dummy_model_gate_up, device):
568 model = dummy_model_gate_up
569 model.supported_lora_modules = ["gate_up_proj"]
570 model.packed_modules_mapping = {
@@ -511,18 +577,25 @@ def test_packed_loras(dist_init, dummy_model_gate_up):
577 1,
578 model,
579 module_name="gate_up_proj",
514 - replaced_module_names=["gate_proj", "up_proj"])
580 + replaced_module_names=["gate_proj", "up_proj"],
581 + device=device)
582 model_lora1 = create_packed_lora(
583 2,
584 model,
585 module_name="gate_up_proj",
586 replaced_module_names=["gate_proj", "up_proj"],
587 + device=device,
588 empty_replaced_module_name="gate_proj",
589 )
590
523 - manager = LoRAModelManager(
524 - model, 2, 2, 2,
525 - LoRAConfig(max_lora_rank=8, max_cpu_loras=2, max_loras=2))
591 + manager = LoRAModelManager(model,
592 + 2,
593 + 2,
594 + 2,
595 + LoRAConfig(max_lora_rank=8,
596 + max_cpu_loras=2,
597 + max_loras=2),
598 + device=device)
599 model = manager.model
600
601 assert isinstance(model.get_submodule("gate_up_proj"),
tests/lora/utils.py
+5 -4
@@ -7,9 +7,10 @@ from vllm.lora.lora import LoRALayerWeights, PackedLoRALayerWeights
7
8 class DummyLoRAManager:
9
10 - def __init__(self):
10 + def __init__(self, device: torch.device = "cuda:0"):
11 super().__init__()
12 self._loras: Dict[str, LoRALayerWeights] = {}
13 + self._device = device
14
15 def set_module_lora(self, module_name: str, lora: LoRALayerWeights):
16 self._loras[module_name] = lora
@@ -28,16 +29,16 @@ class DummyLoRAManager:
29 lora_alpha=1,
30 lora_a=torch.rand([weight.shape[1], rank],
31 dtype=weight.dtype,
31 - device="cuda"),
32 + device=self._device),
33 lora_b=torch.rand([rank, weight.shape[0]],
34 dtype=weight.dtype,
34 - device="cuda"),
35 + device=self._device),
36 )
37 if generate_embeddings_tensor:
38 lora.embeddings_tensor = torch.rand(5,
39 generate_embeddings_tensor,
40 dtype=weight.dtype,
40 - device="cuda")
41 + device=self._device)
42 self.set_module_lora(module_name, lora)
43
44 return lora
vllm/lora/models.py
+9 -10
@@ -301,6 +301,7 @@ class LoRAModelManager(AdapterModelManager):
301 max_num_batched_tokens: int,
302 vocab_size: int,
303 lora_config: LoRAConfig,
304 + device: torch.device,
305 ):
306 """Create a LoRAModelManager and adapter for a given model.
307
@@ -314,6 +315,7 @@ class LoRAModelManager(AdapterModelManager):
315 lora_config: the LoRA configuration.
316 """
317 self.lora_config = lora_config
318 + self.device = device
319 self.max_num_seqs = max_num_seqs
320 assert self.capacity >= self.lora_slots
321 self.max_num_batched_tokens = math.ceil(max_num_batched_tokens / 8) * 8
@@ -322,7 +324,7 @@ class LoRAModelManager(AdapterModelManager):
324 self.long_lora_context: Optional[LongContextLoRAContext] = None
325 self.punica_wrapper = PunicaWrapper(max_num_batched_tokens,
326 max_batches=self.max_num_seqs,
325 - device="cuda")
327 + device=self.device)
328 # Scaling factor -> offset to the sin_cos_cache to it.
329 # Used for long context lora.
330 self.scaling_factor_to_offset: Dict[float, int] = {}
@@ -653,16 +655,11 @@ class LoRALRUCache(AdapterLRUCache[LoRAModel]):
655 class LRUCacheLoRAModelManager(LoRAModelManager):
656 """A model manager that manages multiple LoRAs with LRU cache."""
657
656 - def __init__(
657 - self,
658 - model: nn.Module,
659 - max_num_seqs: int,
660 - max_num_batched_tokens: int,
661 - vocab_size: int,
662 - lora_config: LoRAConfig,
663 - ):
658 + def __init__(self, model: nn.Module, max_num_seqs: int,
659 + max_num_batched_tokens: int, vocab_size: int,
660 + lora_config: LoRAConfig, device: torch.device):
661 super().__init__(model, max_num_seqs, max_num_batched_tokens,
665 - vocab_size, lora_config)
662 + vocab_size, lora_config, device)
663 self._registered_adapters: LoRALRUCache = LoRALRUCache(
664 self.capacity, self.deactivate_adapter)
665 self._active_adapters: LoRALRUCache = LoRALRUCache(
@@ -732,6 +729,7 @@ def create_lora_manager(
729 max_num_batched_tokens: int,
730 vocab_size: int,
731 lora_config: LoRAConfig,
732 + device: torch.device,
733 lora_manager_cls: Type[LoRAModelManager] = LoRAModelManager,
734 **kwargs) -> LoRAModelManager:
735 """Create a LoRA adapter for a given model."""
@@ -743,5 +741,6 @@ def create_lora_manager(
741 max_num_batched_tokens=max_num_batched_tokens,
742 vocab_size=vocab_size,
743 lora_config=lora_config,
744 + device=device,
745 **kwargs)
746 return lora_manager
vllm/lora/punica.py
+9 -6
@@ -62,6 +62,7 @@ def convert_mapping(
62 max_loras: int,
63 vocab_size: int,
64 extra_vocab_size: int,
65 + device: torch.device,
66 long_lora_context: Optional["LongContextLoRAContext"] = None,
67 ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor,
68 Optional[torch.Tensor], List[int]]:
@@ -104,7 +105,7 @@ def convert_mapping(
105 long_lora_offsets: Optional[torch.Tensor] = None
106 if long_lora_context:
107 long_lora_offsets = torch.zeros(len(index_mapping_indices),
107 - device="cuda",
108 + device=device,
109 dtype=torch.long)
110 prompt_mapping: List[int] = [
111 lora_index_to_id.index(x) if x > 0 else -1
@@ -131,10 +132,10 @@ def convert_mapping(
132 if long_lora_context:
133 assert long_lora_offsets is not None
134 indices_list.append(long_lora_offsets)
134 - indices = torch.tensor(indices_list, dtype=torch.long, device="cuda")
135 + indices = torch.tensor(indices_list, dtype=torch.long, device=device)
136 prompt_mapping_tensor = torch.tensor(prompt_mapping,
136 - device="cuda",
137 - dtype=torch.long)
137 + dtype=torch.long,
138 + device=device)
139 embeddings_indices = torch.stack([
140 indices[2] * extra_vocab_size,
141 indices[2] * (vocab_size + extra_vocab_size),
@@ -145,7 +146,7 @@ def convert_mapping(
146 sampler_indices_padded = sampler_indices.clone()
147 sampler_indices_padded[sampler_indices_padded == -1] = max_loras - 1
148 sampler_indices_padded = torch.arange(
148 - 0, len(sampler_indices_padded), device="cuda", dtype=torch.long) + (
149 + 0, len(sampler_indices_padded), device=device, dtype=torch.long) + (
150 sampler_indices_padded * len(sampler_indices_padded))
151 long_lora_indices = None
152 long_lora_indices_len: Optional[int] = None
@@ -183,7 +184,7 @@ class PunicaWrapper:
184 """
185
186 def __init__(self, max_num_batched_tokens: int, max_batches: int,
186 - device: str):
187 + device: Union[torch.device, str]):
188 self._token_lora_indices = torch.empty(max_num_batched_tokens,
189 dtype=torch.long,
190 device=device)
@@ -215,6 +216,7 @@ class PunicaWrapper:
216 self._lora_indices_per_batch = torch.empty(max_batches,
217 dtype=torch.long,
218 device=device)
219 + self.device: torch.device = device
220 self.max_length: int = 0
221 self.token_nums: int = 0
222 self.batch_size: int = -1
@@ -263,6 +265,7 @@ class PunicaWrapper:
265 max_loras,
266 vocab_size,
267 extra_vocab_size,
268 + self.device,
269 long_lora_context,
270 )
271 self._token_lora_indices[:base_indices.shape[0]].copy_(base_indices)
vllm/lora/worker_manager.py
+2
@@ -73,6 +73,7 @@ class WorkerLoRAManager(AbstractWorkerManager):
73 max_num_batched_tokens=self.max_num_batched_tokens,
74 vocab_size=self.vocab_size,
75 lora_config=self.lora_config,
76 + device=self.device,
77 lora_manager_cls=self._manager_cls,
78 )
79 self._adapter_manager = lora_manager
@@ -176,6 +177,7 @@ class LRUCacheWorkerLoRAManager(WorkerLoRAManager):
177 max_num_seqs=self.max_num_seqs,
178 vocab_size=self.vocab_size,
179 lora_config=self.lora_config,
180 + device=self.device,
181 max_num_batched_tokens=self.max_num_batched_tokens,
182 )
183 self._adapter_manager = lora_manager