@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 99d7cabd7b
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 19:30 UTC

[LoRA] ReplicatedLinear support LoRA (#7081)

Jee Jee Li committed Aug 3, 2024 at 13:40 UTC 99d7cabd7b8b789e837a0682982fd7ec94a843b1
3 files changed +199
tests/lora/test_layers.py
+103
@@ -22,6 +22,7 @@ from vllm.lora.layers import (BaseLayerWithLoRA, ColumnParallelLinearWithLoRA,
22 MergedColumnParallelLinearWithLoRA,
23 MergedQKVParallelLinearWithLora,
24 QKVParallelLinearWithLora,
25 + ReplicatedLinearWithLoRA,
26 RowParallelLinearWithLoRA,
27 VocabParallelEmbeddingWithLoRA)
28 # yapf: enable
@@ -31,6 +32,7 @@ from vllm.lora.punica import PunicaWrapper
32 from vllm.model_executor.layers.linear import (ColumnParallelLinear,
33 MergedColumnParallelLinear,
34 QKVParallelLinear,
35 + ReplicatedLinear,
36 RowParallelLinear)
37 from vllm.model_executor.layers.logits_processor import LogitsProcessor
38 from vllm.model_executor.layers.rotary_embedding import get_rope
@@ -545,6 +547,107 @@ def test_lm_head_logits_processor(dist_init, num_loras, device, vocab_size,
547 atol=atol)
548
549
550 +@torch.inference_mode()
551 +@pytest.mark.parametrize("num_loras", [1, 2, 4, 8])
552 +@pytest.mark.parametrize("device", CUDA_DEVICES)
553 +@pytest.mark.parametrize("stage", STAGES)
554 +def test_linear_replicated(dist_init, num_loras, device, stage) -> None:
555 +
556 + torch.set_default_device(device)
557 + punica_wrapper = PunicaWrapper(8192, 256, device)
558 + max_loras = 8
559 + lora_config = LoRAConfig(max_loras=max_loras,
560 + max_lora_rank=8,
561 + lora_dtype=torch.float16)
562 +
563 + def create_random_linear_replicated_layer():
564 +
565 + linear = ReplicatedLinear(4096,
566 + 4096,
567 + bias=False,
568 + params_dtype=torch.float16)
569 + linear.weight.data = torch.rand_like(linear.weight.data)
570 + lora_linear = ReplicatedLinearWithLoRA(linear)
571 +
572 + lora_linear.create_lora_weights(max_loras, lora_config)
573 +
574 + return linear, lora_linear
575 +
576 + for i in range(10):
577 + set_random_seed(i)
578 +
579 + id_to_index = get_random_id_to_index(num_loras, max_loras)
580 + linear, lora_linear = create_random_linear_replicated_layer()
581 + lora_linear.set_mapping(punica_wrapper)
582 + lora_dict, _ = populate_loras(
583 + id_to_index,
584 + layer=lora_linear,
585 + layer_weights=linear.weight,
586 + )
587 +
588 + inputs, index_mapping, prompt_mapping = create_random_inputs(
589 + active_lora_ids=list(lora_dict.keys()),
590 + num_inputs=32 * num_loras,
591 + input_size=(1, 4096),
592 + input_range=(0, 1),
593 + input_type=torch.float16,
594 + )
595 + lora_mapping = LoRAMapping(index_mapping,
596 + prompt_mapping,
597 + is_prefill=stage)
598 + punica_wrapper.update_metadata(
599 + lora_mapping,
600 + id_to_index,
601 + max_loras,
602 + 512,
603 + lora_config.lora_extra_vocab_size,
604 + )
605 +
606 + lora_result = lora_linear(torch.cat(inputs))[0]
607 +
608 + expected_results: List[torch.Tensor] = []
609 + for input_, lora_id in zip(inputs, prompt_mapping):
610 + lora = lora_dict[lora_id]
611 + result = linear(input_)[0]
612 + result += input_ @ lora.lora_a @ lora.lora_b * lora.scaling
613 + expected_results.append(result)
614 + expected_result = torch.cat(expected_results)
615 +
616 + rtol, atol = TOLERANCES[lora_result.dtype]
617 + assert torch.allclose(lora_result,
618 + expected_result,
619 + rtol=rtol,
620 + atol=atol)
621 +
622 + # Check that resetting the lora weights succeeds
623 +
624 + for slot_idx in range(max_loras):
625 + lora_linear.reset_lora(slot_idx)
626 +
627 + inputs, index_mapping, prompt_mapping = create_random_inputs(
628 + active_lora_ids=[0],
629 + num_inputs=32 * num_loras,
630 + input_size=(1, 4096),
631 + input_range=(0, 1),
632 + input_type=torch.float16,
633 + )
634 + lora_mapping = LoRAMapping(index_mapping,
635 + prompt_mapping,
636 + is_prefill=stage)
637 +
638 + punica_wrapper.update_metadata(lora_mapping, id_to_index, max_loras,
639 + 512, lora_config.lora_extra_vocab_size)
640 +
641 + lora_result = lora_linear(torch.cat(inputs))[0]
642 + expected_result = linear(torch.cat(inputs))[0]
643 +
644 + rtol, atol = TOLERANCES[lora_result.dtype]
645 + assert torch.allclose(lora_result,
646 + expected_result,
647 + rtol=rtol,
648 + atol=atol)
649 +
650 +
651 @torch.inference_mode()
652 @pytest.mark.parametrize("num_loras", [1, 2, 4, 8])
653 @pytest.mark.parametrize("orientation", ["row", "column"])
vllm/lora/layers.py
+94
@@ -21,6 +21,7 @@ from vllm.lora.punica import PunicaWrapper
21 from vllm.model_executor.layers.linear import (ColumnParallelLinear,
22 MergedColumnParallelLinear,
23 QKVParallelLinear,
24 + ReplicatedLinear,
25 RowParallelLinear)
26 from vllm.model_executor.layers.logits_processor import LogitsProcessor
27 from vllm.model_executor.layers.rotary_embedding import (
@@ -262,6 +263,99 @@ class VocabParallelEmbeddingWithLoRA(BaseLayerWithLoRA):
263 return type(source_layer) is VocabParallelEmbedding
264
265
266 +class ReplicatedLinearWithLoRA(BaseLayerWithLoRA):
267 +
268 + def __init__(self, base_layer: ReplicatedLinear) -> None:
269 + super().__init__()
270 + self.base_layer = base_layer
271 + self.input_size = self.base_layer.input_size
272 + self.output_size = self.base_layer.output_size
273 + self.device = _get_lora_device(self.base_layer)
274 +
275 + def create_lora_weights(
276 + self,
277 + max_loras: int,
278 + lora_config: LoRAConfig,
279 + model_config: Optional[PretrainedConfig] = None,
280 + ) -> None:
281 + self.lora_config = lora_config
282 + lora_a_output_size = lora_config.max_lora_rank
283 + self.lora_a_stacked = torch.zeros(
284 + max_loras,
285 + 1,
286 + lora_a_output_size,
287 + self.input_size,
288 + dtype=lora_config.lora_dtype,
289 + device=self.device,
290 + )
291 + self.lora_b_stacked = torch.zeros(
292 + max_loras,
293 + 1,
294 + self.output_size,
295 + lora_config.max_lora_rank,
296 + dtype=lora_config.lora_dtype,
297 + device=self.device,
298 + )
299 +
300 + def reset_lora(self, index: int):
301 + self.lora_a_stacked[index] = 0
302 + self.lora_b_stacked[index] = 0
303 +
304 + def set_lora(
305 + self,
306 + index: int,
307 + lora_a: torch.Tensor,
308 + lora_b: torch.Tensor,
309 + embeddings_tensor: Optional[torch.Tensor],
310 + ):
311 + self.reset_lora(index)
312 +
313 + self.lora_a_stacked[index,
314 + 0, :lora_a.shape[1], :lora_a.shape[0]].copy_(
315 + lora_a.T, non_blocking=True)
316 + self.lora_b_stacked[index,
317 + 0, :lora_b.shape[1], :lora_b.shape[0]].copy_(
318 + lora_b.T, non_blocking=True)
319 +
320 + def apply(self, x: torch.Tensor,
321 + bias: Optional[torch.Tensor]) -> torch.Tensor:
322 + output = self.base_layer.quant_method.apply(self.base_layer, x, bias)
323 + self.punica_wrapper.add_lora(output, x, self.lora_a_stacked,
324 + self.lora_b_stacked, 1.0)
325 + return output
326 +
327 + def forward(self, input_):
328 + """Forward of ReplicatedLinearWithLoRA
329 +
330 + Args:
331 + input_: Tensor whose last dimension is `input_size`.
332 +
333 + Returns:
334 + - output
335 + - bias
336 + """
337 + bias = (self.base_layer.bias
338 + if not self.base_layer.skip_bias_add else None)
339 +
340 + # Matrix multiply.
341 + output = self.apply(input_, bias)
342 +
343 + output_bias = (self.base_layer.bias
344 + if self.base_layer.skip_bias_add else None)
345 + return output, output_bias
346 +
347 + @classmethod
348 + @_not_fully_sharded_can_replace
349 + def can_replace_layer(
350 + cls,
351 + source_layer: nn.Module,
352 + lora_config: LoRAConfig,
353 + packed_modules_list: List,
354 + model_config: Optional[PretrainedConfig],
355 + ) -> bool:
356 + return type(source_layer) is ReplicatedLinear
357 +
358 +
359 class ColumnParallelLinearWithLoRA(BaseLayerWithLoRA):
360 """
361 LoRA on top of ColumnParallelLinear layer.
vllm/lora/utils.py
+2
@@ -23,6 +23,7 @@ from vllm.lora.layers import (BaseLayerWithLoRA, ColumnParallelLinearWithLoRA,
23 MergedColumnParallelLinearWithLoRA,
24 MergedQKVParallelLinearWithLora,
25 QKVParallelLinearWithLora,
26 + ReplicatedLinearWithLoRA,
27 RowParallelLinearWithLoRA,
28 VocabParallelEmbeddingWithLoRA)
29 # yapf: enable
@@ -38,6 +39,7 @@ _all_lora_classes: Set[Type[BaseLayerWithLoRA]] = {
39 QKVParallelLinearWithLora,
40 MergedQKVParallelLinearWithLora,
41 RowParallelLinearWithLoRA,
42 + ReplicatedLinearWithLoRA,
43 LogitsProcessorWithLoRA,
44 ColumnParallelLinearWithShardedLoRA,
45 QKVParallelLinearWithShardedLora,