@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / b45f0d7946
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 05:45 UTC

[Misc][LoRA] Move the implementation of lora bias to punica.py (#10829)

Signed-off-by: Jee Jee Li <pandaleefree@gmail.com>

Jee Jee Li committed Dec 3, 2024 at 01:53 UTC b45f0d79469f583736052b80bfc8b3bab29f50d8
4 files changed +156 -175
tests/lora/test_llama_tp.py
+27 -33
@@ -55,15 +55,7 @@ def do_sample(llm: vllm.LLM, lora_path: str, lora_id: int) -> List[str]:
55 return generated_texts
56
57
58 -@fork_new_process_for_each_test
59 -def test_llama_lora(sql_lora_files):
60 -
61 - llm = vllm.LLM(MODEL_PATH,
62 - enable_lora=True,
63 - max_num_seqs=16,
64 - max_loras=4,
65 - tensor_parallel_size=1)
66 -
58 +def generate_and_test(llm, sql_lora_files):
59 print("lora adapter created")
60 assert do_sample(llm, sql_lora_files, lora_id=0) == EXPECTED_NO_LORA_OUTPUT
61
@@ -79,6 +71,17 @@ def test_llama_lora(sql_lora_files):
71 print("removing lora")
72
73
74 +@fork_new_process_for_each_test
75 +def test_llama_lora(sql_lora_files):
76 +
77 + llm = vllm.LLM(MODEL_PATH,
78 + enable_lora=True,
79 + max_num_seqs=16,
80 + max_loras=4,
81 + tensor_parallel_size=1)
82 + generate_and_test(llm, sql_lora_files)
83 +
84 +
85 @fork_new_process_for_each_test
86 def test_llama_lora_warmup(sql_lora_files):
87 """Test that the LLM initialization works with a warmup LORA path and
@@ -118,20 +121,7 @@ def test_llama_lora_tp4(sql_lora_files):
121 max_loras=4,
122 tensor_parallel_size=4,
123 )
121 -
122 - print("lora adapter created")
123 - assert do_sample(llm, sql_lora_files, lora_id=0) == EXPECTED_NO_LORA_OUTPUT
124 -
125 - print("lora 1")
126 - assert do_sample(llm, sql_lora_files, lora_id=1) == EXPECTED_LORA_OUTPUT
127 -
128 - print("no lora")
129 - assert do_sample(llm, sql_lora_files, lora_id=0) == EXPECTED_NO_LORA_OUTPUT
130 -
131 - print("lora 2")
132 - assert do_sample(llm, sql_lora_files, lora_id=2) == EXPECTED_LORA_OUTPUT
133 -
134 - print("removing lora")
124 + generate_and_test(llm, sql_lora_files)
125
126
127 @multi_gpu_test(num_gpus=4)
@@ -146,16 +136,20 @@ def test_llama_lora_tp4_fully_sharded_loras(sql_lora_files):
136 tensor_parallel_size=4,
137 fully_sharded_loras=True,
138 )
149 - print("lora adapter created")
150 - assert do_sample(llm, sql_lora_files, lora_id=0) == EXPECTED_NO_LORA_OUTPUT
151 -
152 - print("lora 1")
153 - assert do_sample(llm, sql_lora_files, lora_id=1) == EXPECTED_LORA_OUTPUT
139 + generate_and_test(llm, sql_lora_files)
140
155 - print("no lora")
156 - assert do_sample(llm, sql_lora_files, lora_id=0) == EXPECTED_NO_LORA_OUTPUT
141
158 - print("lora 2")
159 - assert do_sample(llm, sql_lora_files, lora_id=2) == EXPECTED_LORA_OUTPUT
142 +@multi_gpu_test(num_gpus=4)
143 +@fork_new_process_for_each_test
144 +def test_llama_lora_tp4_fully_sharded_enable_bias(sql_lora_files):
145
161 - print("removing lora")
146 + llm = vllm.LLM(
147 + MODEL_PATH,
148 + enable_lora=True,
149 + max_num_seqs=16,
150 + max_loras=4,
151 + tensor_parallel_size=4,
152 + fully_sharded_loras=True,
153 + enable_lora_bias=True,
154 + )
155 + generate_and_test(llm, sql_lora_files)
vllm/lora/fully_sharded_layers.py
+12 -29
@@ -73,6 +73,7 @@ class ColumnParallelLinearWithShardedLoRA(ColumnParallelLinearWithLoRA):
73 self.punica_wrapper.add_expand(output,
74 buffer,
75 self.lora_b_stacked,
76 + self.bias_stacked,
77 add_input=True)
78 # now have column partitioned output
79
@@ -131,27 +132,14 @@ def _mcp_apply(x, bias, layer: QKVParallelLinearWithLora):
132 layer.lora_a_stacked[idx], 1.0)
133
134 buffers = tensor_model_parallel_all_gather(buffers)
134 - left_offset = 0
135 - for idx in range(n):
136 - shard_size = layer.lora_b_stacked[idx].shape[2]
137 -
138 - if layer.bias_stacked is not None:
139 - bias = layer.bias_stacked[idx]
140 - if bias is not None:
141 - bias = bias.view(-1, bias.shape[-1])
142 - bias = bias[layer.punica_wrapper.token_lora_indices]
143 - bias[layer.punica_wrapper.token_lora_indices == -1] = 0
144 - output[:, left_offset:left_offset + shard_size] += bias
145 -
146 - layer.punica_wrapper.add_expand_slice(
147 - output,
148 - buffers[idx],
149 - layer.lora_b_stacked[idx],
150 - left_offset,
151 - shard_size,
152 - add_input=True,
153 - )
154 - left_offset += shard_size
135 + layer.punica_wrapper.add_expand_packed_nslice(
136 + output,
137 + buffers,
138 + layer.lora_b_stacked,
139 + layer.bias_stacked,
140 + 1.0,
141 + layer.output_slices,
142 + )
143
144 output = output.view(*out_orig_shape)
145 # now have column partitioned and packed output
@@ -234,6 +222,7 @@ class QKVParallelLinearWithShardedLora(QKVParallelLinearWithLora):
222 self.punica_wrapper.add_expand(output,
223 buffer,
224 self.lora_b_stacked,
225 + self.bias_all,
226 add_input=True)
227 # now have column partitioned output
228 output = output.view(*out_orig_shape)
@@ -350,15 +339,9 @@ class RowParallelLinearWithShardedLoRA(RowParallelLinearWithLoRA):
339 # reduced before being used
340 shard_size = self.lora_b_stacked.shape[2]
341 start_idx = self.tp_rank * shard_size
353 -
354 - if self.bias_stacked is not None:
355 - bias = self.bias_stacked.view(-1, self.bias_stacked.shape[-1])
356 - bias = bias[self.punica_wrapper.token_lora_indices]
357 - bias[self.punica_wrapper.token_lora_indices == -1] = 0
358 - output += bias
359 -
342 self.punica_wrapper.add_expand_slice(output, buffer,
361 - self.lora_b_stacked, start_idx,
343 + self.lora_b_stacked,
344 + self.bias_stacked, start_idx,
345 shard_size)
346 output = output.view(*out_orig_shape)
347 return output
vllm/lora/layers.py
+12 -101
@@ -67,63 +67,6 @@ def _not_fully_sharded_can_replace(can_replace):
67 return dec
68
69
70 -def apply_bias(
71 - indices: torch.Tensor,
72 - output: torch.Tensor,
73 - bias_stacked: torch.Tensor,
74 -):
75 - """Applies bias to output
76 -
77 - Input shapes:
78 - bias_stacked: (num_loras, output_dim)
79 - indices: (batch_size)
80 - output: (batch_size, output_dim)
81 - """
82 - org_output = output
83 - output = output.view(-1, output.shape[-1])
84 - indices = indices.view(-1)
85 -
86 - bias_stacked = bias_stacked.view(-1, bias_stacked.shape[-1])
87 - bias_stacked = bias_stacked[indices]
88 - bias_stacked[indices == -1] = 0
89 - output += bias_stacked
90 -
91 - return output.view_as(org_output)
92 -
93 -
94 -def apply_bias_packed_nslice(
95 - indices: torch.Tensor,
96 - output: torch.Tensor,
97 - output_slices: Tuple[int, ...],
98 - bias_stacked: Tuple[torch.Tensor, torch.Tensor, torch.Tensor],
99 -):
100 - """Applies bias to output
101 -
102 - Input shapes:
103 - bias_stacked: 3 element tuple of (num_loras, output_dim)
104 - indices: (batch_size)
105 - output: (batch_size, q_slice_size + 2*kv_slice_size)
106 - output_slices: n-1 element tuple of (slice_size...),
107 - where n is number of slices
108 - """
109 - org_output = output
110 - output = output.view(-1, output.shape[-1])
111 - indices = indices.view(-1)
112 -
113 - offset_left = 0
114 - for slice_idx, slice in enumerate(output_slices):
115 - bias = bias_stacked[slice_idx]
116 - if bias is not None:
117 - bias = bias.view(-1, bias.shape[-1])
118 - bias = bias[indices]
119 - bias[indices == -1] = 0
120 - output[:, offset_left:offset_left + slice] += bias
121 -
122 - offset_left += slice
123 -
124 - return output.view_as(org_output)
125 -
126 -
70 @dataclass
71 class LoRAMapping(AdapterMapping):
72 is_prefill: bool = False
@@ -311,6 +254,7 @@ class VocabParallelEmbeddingWithLoRA(BaseLayerWithLoRA):
254 self.punica_wrapper.add_expand(full_output,
255 full_lora_a_embeddings,
256 self.lora_b_stacked,
257 + bias_all=None,
258 add_input=True)
259 return full_output.view_as(full_output_org)
260
@@ -399,15 +343,9 @@ class ReplicatedLinearWithLoRA(BaseLayerWithLoRA):
343 def apply(self, x: torch.Tensor,
344 bias: Optional[torch.Tensor]) -> torch.Tensor:
345 output = self.base_layer.quant_method.apply(self.base_layer, x, bias)
402 - if self.bias_stacked is not None:
403 - self.indices = self.punica_wrapper.token_lora_indices
404 - output = apply_bias(
405 - self.indices,
406 - output,
407 - self.bias_stacked,
408 - )
346 self.punica_wrapper.add_lora(output, x, self.lora_a_stacked,
410 - self.lora_b_stacked, 1.0)
347 + self.lora_b_stacked, self.bias_stacked,
348 + 1.0)
349 return output
350
351 def forward(self, input_):
@@ -576,15 +514,9 @@ class ColumnParallelLinearWithLoRA(BaseLayerWithLoRA):
514 def apply(self, x: torch.Tensor,
515 bias: Optional[torch.Tensor]) -> torch.Tensor:
516 output = self.base_layer.quant_method.apply(self.base_layer, x, bias)
579 - if self.bias_stacked is not None:
580 - self.indices = self.punica_wrapper.token_lora_indices
581 - output = apply_bias(
582 - self.indices,
583 - output,
584 - self.bias_stacked,
585 - )
517 self.punica_wrapper.add_lora(output, x, self.lora_a_stacked,
587 - self.lora_b_stacked, 1.0)
518 + self.lora_b_stacked, self.bias_stacked,
519 + 1.0)
520 return output
521
522 def forward(self, input_):
@@ -687,8 +619,8 @@ class MergedColumnParallelLinearWithLoRA(ColumnParallelLinearWithLoRA):
619 ) for _ in range(n_slices))
620 else:
621 self.bias_stacked = None
690 -
622 self.output_dim = self.lora_b_stacked[0].shape[2]
623 + self.output_slices = (self.output_dim, self.output_dim)
624
625 def reset_lora(self, index: int):
626 self.lora_a_stacked[0][index] = 0
@@ -772,17 +704,9 @@ class MergedColumnParallelLinearWithLoRA(ColumnParallelLinearWithLoRA):
704 def apply(self, x: torch.Tensor,
705 bias: Optional[torch.Tensor]) -> torch.Tensor:
706 output = self.base_layer.quant_method.apply(self.base_layer, x, bias)
775 - if self.bias_stacked is not None:
776 - self.indices = self.punica_wrapper.token_lora_indices
777 - output = apply_bias_packed_nslice(
778 - self.indices,
779 - output,
780 - (self.output_dim, self.output_dim),
781 - self.bias_stacked,
782 - )
707 self.punica_wrapper.add_lora_packed_nslice(
784 - output, x, self.lora_a_stacked, self.lora_b_stacked, 1.0,
785 - (self.output_dim, self.output_dim))
708 + output, x, self.lora_a_stacked, self.lora_b_stacked,
709 + self.bias_stacked, 1.0, (self.output_dim, self.output_dim))
710 return output
711
712 @classmethod
@@ -1129,17 +1053,10 @@ class MergedQKVParallelLinearWithLora(ColumnParallelLinearWithLoRA):
1053 def apply(self, x: torch.Tensor,
1054 bias: Optional[torch.Tensor]) -> torch.Tensor:
1055 output = self.base_layer.quant_method.apply(self.base_layer, x, bias)
1132 - if self.bias_stacked is not None:
1133 - self.indices = self.punica_wrapper.token_lora_indices
1134 - output = apply_bias_packed_nslice(
1135 - self.indices,
1136 - output,
1137 - self.output_slices,
1138 - self.bias_stacked,
1139 - )
1056 self.punica_wrapper.add_lora_packed_nslice(output, x,
1057 self.lora_a_stacked,
1142 - self.lora_b_stacked, 1.0,
1058 + self.lora_b_stacked,
1059 + self.bias_stacked, 1.0,
1060 self.output_slices)
1061 return output
1062
@@ -1264,15 +1181,9 @@ class RowParallelLinearWithLoRA(BaseLayerWithLoRA):
1181
1182 def apply(self, x: torch.Tensor) -> torch.Tensor:
1183 output = self.base_layer.quant_method.apply(self.base_layer, x)
1267 - if self.bias_stacked is not None:
1268 - self.indices = self.punica_wrapper.token_lora_indices
1269 - output = apply_bias(
1270 - self.indices,
1271 - output,
1272 - self.bias_stacked,
1273 - )
1184 self.punica_wrapper.add_lora(output, x, self.lora_a_stacked,
1275 - self.lora_b_stacked, 1.0)
1185 + self.lora_b_stacked, self.bias_stacked,
1186 + 1.0)
1187 return output
1188
1189 def forward(self, input_):
vllm/lora/punica.py
+105 -12
@@ -450,6 +450,62 @@ class PunicaWrapper:
450 bgmv_expand_slice(x, w_t_all, y, self.token_lora_indices, y_offset,
451 y_slice_size, add_input)
452
453 + def apply_bias(
454 + self,
455 + indices: torch.Tensor,
456 + output: torch.Tensor,
457 + bias_stacked: torch.Tensor,
458 + ):
459 + """Applies bias to output
460 +
461 + Input shapes:
462 + bias_stacked: (num_loras, output_dim)
463 + indices: (batch_size)
464 + output: (batch_size, output_dim)
465 + """
466 + org_output = output
467 + output = output.view(-1, output.shape[-1])
468 + indices = indices.view(-1)
469 +
470 + bias_stacked = bias_stacked.view(-1, bias_stacked.shape[-1])
471 + bias_stacked = bias_stacked[indices]
472 + bias_stacked[indices == -1] = 0
473 + output += bias_stacked
474 +
475 + return output.view_as(org_output)
476 +
477 + def apply_bias_packed_nslice(
478 + self,
479 + indices: torch.Tensor,
480 + output: torch.Tensor,
481 + output_slices: Tuple[int, ...],
482 + bias_stacked: Tuple[Optional[torch.Tensor], ...],
483 + ):
484 + """Applies bias to output
485 +
486 + Input shapes:
487 + bias_stacked: 3 element tuple of (num_loras, output_dim)
488 + indices: (batch_size)
489 + output: (batch_size, q_slice_size + 2*kv_slice_size)
490 + output_slices: n-1 element tuple of (slice_size...),
491 + where n is number of slices
492 + """
493 + org_output = output
494 + output = output.view(-1, output.shape[-1])
495 + indices = indices.view(-1)
496 +
497 + offset_left = 0
498 + for slice_idx, slice in enumerate(output_slices):
499 + bias = bias_stacked[slice_idx]
500 + if bias is not None:
501 + bias = bias.view(-1, bias.shape[-1])
502 + bias = bias[indices]
503 + bias[indices == -1] = 0
504 + output[:, offset_left:offset_left + slice] += bias
505 + offset_left += slice
506 +
507 + return output.view_as(org_output)
508 +
509 def add_shrink(
510 self,
511 y: torch.Tensor,
@@ -474,16 +530,19 @@ class PunicaWrapper:
530 y: torch.Tensor,
531 x: torch.Tensor,
532 w_t_all: torch.Tensor,
533 + bias_all: Optional[torch.Tensor],
534 add_input: bool = True,
535 ):
536 """
480 - Perform the ` y+=x@w_t_all` computation, which is suitable for the
537 + Perform the ` y+=x@w_t_all+bias` computation, which is suitable for the
538 GEMM of lora'b.
539 When `is_prefill` is true, it indicates that it is currently the
540 prefill stage, and the `expand_prefill` function should be called.
541 Otherwise, it is the decode stage, and the expand_decode function
542 should be called.
543 """
544 + if bias_all is not None:
545 + y = self.apply_bias(self.token_lora_indices, y, bias_all)
546
547 expand_fun: Callable = (self.expand_prefill
548 if self.is_prefill else self.expand_decode)
@@ -493,23 +552,54 @@ class PunicaWrapper:
552 y: torch.Tensor,
553 x: torch.Tensor,
554 w_t_all: torch.Tensor,
555 + bias_all: Optional[torch.Tensor],
556 y_offset: Optional[int],
557 y_slice_size: Optional[int],
558 add_input: bool = True):
559 """
560 Similar to `add_expand`
561 """
562 + if bias_all is not None:
563 + y = self.apply_bias(self.token_lora_indices, y, bias_all)
564
565 expand_slice_fun: Callable = (self.expand_slice_prefill
566 if self.is_prefill else
567 self.expand_slice_decode)
568 expand_slice_fun(y, x, w_t_all, y_offset, y_slice_size, add_input)
569
570 + def add_expand_packed_nslice(self, y: torch.Tensor, x: torch.Tensor,
571 + lora_b_stacked: Tuple[torch.Tensor, ...],
572 + bias_stacked: Optional[Tuple[torch.Tensor,
573 + ...]],
574 + scale: float,
575 + output_slices: Tuple[int, ...]) -> None:
576 + """
577 + Similar to `add_expand`
578 + """
579 + y_org = y
580 + y = y.view(-1, y.shape[-1])
581 + offset_left = 0
582 + if bias_stacked is not None:
583 + self.apply_bias_packed_nslice(self.token_lora_indices, y,
584 + output_slices, bias_stacked)
585 + for slice_idx in range(len(lora_b_stacked)):
586 + self.add_expand_slice(y,
587 + x[slice_idx],
588 + lora_b_stacked[slice_idx],
589 + None,
590 + offset_left,
591 + output_slices[slice_idx],
592 + add_input=True)
593 + offset_left += output_slices[slice_idx]
594 +
595 + y = y.view_as(y_org)
596 +
597 def add_lora(self,
598 y: torch.Tensor,
599 x: torch.Tensor,
600 wa_t_all: torch.Tensor,
601 wb_t_all: torch.Tensor,
602 + bias_all: Optional[torch.Tensor],
603 scale: float,
604 y_offset: Optional[int] = None,
605 y_slice_size: Optional[int] = None,
@@ -522,12 +612,13 @@ class PunicaWrapper:
612 @ wa_t_all[indices[i], layer_idx, :, :].transpose(-1, -2)
613 @ wb_t_all[indices[i], layer_idx, :, :].transpose(-1, -2)
614 * scale
525 - ).squeeze(0)
615 + ).squeeze(0)+bias[i]
616 Args:
617 y (torch.Tensor): Output tensor. Will be changed in-place.
618 x (torch.Tensor): Input tensor
619 wa_t_all (torch.Tensor): lora_a's weight
620 wb_t_all (torch.Tensor): lora_b's weight
621 + bias_all: (torch.Tensor): lora's bias
622 scale (float): Scaling factor.
623 y_offset (Optional[int], optional): Offset to apply to the starting
624 column of y.
@@ -544,27 +635,26 @@ class PunicaWrapper:
635 buffer = torch.zeros((x.size(0), r),
636 dtype=torch.float32,
637 device=x.device)
547 -
638 + if bias_all is not None:
639 + y = self.apply_bias(self.token_lora_indices, y, bias_all)
640 self.add_shrink(buffer, x, wa_t_all, scale)
641 if y_offset is None and y_slice_size is None:
550 - self.add_expand(y, buffer, wb_t_all, add_input=True)
642 + self.add_expand(y, buffer, wb_t_all, bias_all=None, add_input=True)
643 else:
644 self.add_expand_slice(y,
645 buffer,
646 wb_t_all,
647 + None,
648 y_offset,
649 y_slice_size,
650 add_input=True)
651 y = y.view_as(y_org)
652
653 def add_lora_packed_nslice(self, y: torch.Tensor, x: torch.Tensor,
561 - lora_a_stacked: Tuple[torch.Tensor,
562 - torch.Tensor,
563 - torch.Tensor],
564 - lora_b_stacked: Tuple[torch.Tensor,
565 - torch.Tensor,
566 - torch.Tensor],
567 - scale: float,
654 + lora_a_stacked: Tuple[torch.Tensor, ...],
655 + lora_b_stacked: Tuple[torch.Tensor, ...],
656 + bias_all: Tuple[Optional[torch.Tensor],
657 + ...], scale: float,
658 output_slices: Tuple[int, ...]) -> None:
659 """
660 Applies lora to each input. Similar to add_lora, This method is
@@ -575,10 +665,13 @@ class PunicaWrapper:
665 x = x.view(-1, x.shape[-1])
666 y = y.view(-1, y.shape[-1])
667 offset_left = 0
668 + if bias_all is not None:
669 + y = self.apply_bias_packed_nslice(self.token_lora_indices, y,
670 + output_slices, bias_all)
671 # TODO fuse these kernels
672 for slice_idx in range(len(output_slices)):
673 self.add_lora(y, x, lora_a_stacked[slice_idx],
581 - lora_b_stacked[slice_idx], scale, offset_left,
674 + lora_b_stacked[slice_idx], None, scale, offset_left,
675 output_slices[slice_idx])
676 offset_left += output_slices[slice_idx]
677