@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 8af890a865
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 04:15 UTC

Enable more models to inference based on LoRA (#3382)

Co-authored-by: Antoni Baum <antoni.baum@protonmail.com>

Jee Li committed Mar 26, 2024 at 09:09 UTC 8af890a865bf9f744d7d9bd5515558b42224c744
10 files changed +402 -45
csrc/punica/bgmv/bgmv_config.h
+6
@@ -16,10 +16,13 @@ void bgmv_kernel(out_T *__restrict__ Y, const in_T *__restrict__ X,
16 f(in_T, out_T, W_T, narrow, 512) \
17 f(in_T, out_T, W_T, narrow, 768) \
18 f(in_T, out_T, W_T, narrow, 1024) \
19 + f(in_T, out_T, W_T, narrow, 1152) \
20 f(in_T, out_T, W_T, narrow, 1280) \
21 + f(in_T, out_T, W_T, narrow, 1536) \
22 f(in_T, out_T, W_T, narrow, 1728) \
23 f(in_T, out_T, W_T, narrow, 1792) \
24 f(in_T, out_T, W_T, narrow, 2048) \
25 + f(in_T, out_T, W_T, narrow, 2304) \
26 f(in_T, out_T, W_T, narrow, 2560) \
27 f(in_T, out_T, W_T, narrow, 2752) \
28 f(in_T, out_T, W_T, narrow, 2816) \
@@ -27,10 +30,12 @@ void bgmv_kernel(out_T *__restrict__ Y, const in_T *__restrict__ X,
30 f(in_T, out_T, W_T, narrow, 3456) \
31 f(in_T, out_T, W_T, narrow, 3584) \
32 f(in_T, out_T, W_T, narrow, 4096) \
33 + f(in_T, out_T, W_T, narrow, 4608) \
34 f(in_T, out_T, W_T, narrow, 5120) \
35 f(in_T, out_T, W_T, narrow, 5504) \
36 f(in_T, out_T, W_T, narrow, 5632) \
37 f(in_T, out_T, W_T, narrow, 6144) \
38 + f(in_T, out_T, W_T, narrow, 6848) \
39 f(in_T, out_T, W_T, narrow, 6912) \
40 f(in_T, out_T, W_T, narrow, 7168) \
41 f(in_T, out_T, W_T, narrow, 8192) \
@@ -45,6 +50,7 @@ void bgmv_kernel(out_T *__restrict__ Y, const in_T *__restrict__ X,
50 f(in_T, out_T, W_T, narrow, 20480) \
51 f(in_T, out_T, W_T, narrow, 22016) \
52 f(in_T, out_T, W_T, narrow, 24576) \
53 + f(in_T, out_T, W_T, narrow, 27392) \
54 f(in_T, out_T, W_T, narrow, 28672) \
55 f(in_T, out_T, W_T, narrow, 32000) \
56 f(in_T, out_T, W_T, narrow, 32256) \
tests/lora/conftest.py
+10
@@ -134,6 +134,16 @@ def gemma_lora_files():
134 return snapshot_download(repo_id="wskwon/gemma-7b-test-lora")
135
136
137 +@pytest.fixture(scope="session")
138 +def chatglm3_lora_files():
139 + return snapshot_download(repo_id="jeeejeee/chatglm3-text2sql-spider")
140 +
141 +
142 +@pytest.fixture(scope="session")
143 +def baichuan_lora_files():
144 + return snapshot_download(repo_id="jeeejeee/baichuan7b-text2sql-spider")
145 +
146 +
147 @pytest.fixture
148 def llama_2_7b_engine_extra_embeddings() -> nn.Module:
149 cleanup()
tests/lora/test_baichuan.py new
+108
@@ -0,0 +1,108 @@
1 +import pytest
2 +
3 +import vllm
4 +from vllm.lora.request import LoRARequest
5 +
6 +from .conftest import cleanup
7 +
8 +MODEL_PATH = "baichuan-inc/Baichuan-7B"
9 +
10 +PROMPT_TEMPLATE = """I want you to act as a SQL terminal in front of an example database, you need only to return the sql command to me.Below is an instruction that describes a task, Write a response that appropriately completes the request.\n"\n##Instruction:\nconcert_singer contains tables such as stadium, singer, concert, singer_in_concert. Table stadium has columns such as Stadium_ID, Location, Name, Capacity, Highest, Lowest, Average. Stadium_ID is the primary key.\nTable singer has columns such as Singer_ID, Name, Country, Song_Name, Song_release_year, Age, Is_male. Singer_ID is the primary key.\nTable concert has columns such as concert_ID, concert_Name, Theme, Stadium_ID, Year. concert_ID is the primary key.\nTable singer_in_concert has columns such as concert_ID, Singer_ID. concert_ID is the primary key.\nThe Stadium_ID of concert is the foreign key of Stadium_ID of stadium.\nThe Singer_ID of singer_in_concert is the foreign key of Singer_ID of singer.\nThe concert_ID of singer_in_concert is the foreign key of concert_ID of concert.\n\n###Input:\n{query}\n\n###Response:""" # noqa: E501
11 +
12 +
13 +def do_sample(llm, lora_path: str, lora_id: int) -> str:
14 + prompts = [
15 + PROMPT_TEMPLATE.format(query="How many singers do we have?"),
16 + PROMPT_TEMPLATE.format(
17 + query=
18 + "What is the average, minimum, and maximum age of all singers from France?" # noqa: E501
19 + ),
20 + PROMPT_TEMPLATE.format(
21 + query=
22 + "Show name, country, age for all singers ordered by age from the oldest to the youngest." # noqa: E501
23 + ),
24 + ]
25 + print(prompts)
26 + sampling_params = vllm.SamplingParams(temperature=0, max_tokens=256)
27 + outputs = llm.generate(
28 + prompts,
29 + sampling_params,
30 + lora_request=LoRARequest(str(lora_id), lora_id, lora_path)
31 + if lora_id else None)
32 + # Print the outputs.
33 + generated_texts = []
34 + for output in outputs:
35 + prompt = output.prompt
36 + generated_text = output.outputs[0].text.strip()
37 + generated_texts.append(generated_text)
38 + print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
39 + return generated_texts
40 +
41 +
42 +def test_baichuan_lora(baichuan_lora_files):
43 + llm = vllm.LLM(MODEL_PATH,
44 + max_model_len=1024,
45 + enable_lora=True,
46 + max_loras=4,
47 + max_lora_rank=64,
48 + trust_remote_code=True)
49 +
50 + expected_lora_output = [
51 + "SELECT count(*) FROM singer",
52 + "SELECT avg(age) , min(age) , max(age) FROM singer WHERE Country = 'France'", # noqa: E501
53 + "SELECT name , country , age FROM singer ORDER BY age ASC",
54 + ]
55 +
56 + output1 = do_sample(llm, baichuan_lora_files, lora_id=1)
57 + for i in range(len(expected_lora_output)):
58 + assert output1[i] == expected_lora_output[i]
59 + output2 = do_sample(llm, baichuan_lora_files, lora_id=2)
60 + for i in range(len(expected_lora_output)):
61 + assert output2[i] == expected_lora_output[i]
62 +
63 +
64 +@pytest.mark.skip("Requires multiple GPUs")
65 +def test_llama_tensor_parallel_equality(baichuan_lora_files):
66 + # Cannot use as it will initialize torch.cuda too early...
67 + # if torch.cuda.device_count() < 4:
68 + # pytest.skip(f"Not enough GPUs for tensor parallelism {4}")
69 +
70 + llm_tp1 = vllm.LLM(MODEL_PATH,
71 + enable_lora=True,
72 + max_num_seqs=16,
73 + max_loras=4,
74 + max_lora_rank=64,
75 + tensor_parallel_size=1,
76 + trust_remote_code=True)
77 + output_tp1 = do_sample(llm_tp1, baichuan_lora_files, lora_id=1)
78 +
79 + del llm_tp1
80 + cleanup()
81 +
82 + llm_tp2 = vllm.LLM(MODEL_PATH,
83 + enable_lora=True,
84 + max_num_seqs=16,
85 + max_loras=4,
86 + max_lora_rank=64,
87 + tensor_parallel_size=2,
88 + trust_remote_code=True)
89 + output_tp2 = do_sample(llm_tp2, baichuan_lora_files, lora_id=2)
90 +
91 + del llm_tp2
92 + cleanup()
93 +
94 + assert output_tp1 == output_tp2
95 +
96 + llm_tp4 = vllm.LLM(MODEL_PATH,
97 + enable_lora=True,
98 + max_num_seqs=16,
99 + max_loras=4,
100 + max_lora_rank=64,
101 + tensor_parallel_size=4,
102 + trust_remote_code=True)
103 + output_tp4 = do_sample(llm_tp4, baichuan_lora_files, lora_id=2)
104 +
105 + del llm_tp4
106 + cleanup()
107 +
108 + assert output_tp1 == output_tp4
\ No newline at end of file
tests/lora/test_chatglm3.py new
+57
@@ -0,0 +1,57 @@
1 +import vllm
2 +from vllm.lora.request import LoRARequest
3 +
4 +MODEL_PATH = "THUDM/chatglm3-6b"
5 +
6 +PROMPT_TEMPLATE = """I want you to act as a SQL terminal in front of an example database, you need only to return the sql command to me.Below is an instruction that describes a task, Write a response that appropriately completes the request.\n"\n##Instruction:\nconcert_singer contains tables such as stadium, singer, concert, singer_in_concert. Table stadium has columns such as Stadium_ID, Location, Name, Capacity, Highest, Lowest, Average. Stadium_ID is the primary key.\nTable singer has columns such as Singer_ID, Name, Country, Song_Name, Song_release_year, Age, Is_male. Singer_ID is the primary key.\nTable concert has columns such as concert_ID, concert_Name, Theme, Stadium_ID, Year. concert_ID is the primary key.\nTable singer_in_concert has columns such as concert_ID, Singer_ID. concert_ID is the primary key.\nThe Stadium_ID of concert is the foreign key of Stadium_ID of stadium.\nThe Singer_ID of singer_in_concert is the foreign key of Singer_ID of singer.\nThe concert_ID of singer_in_concert is the foreign key of concert_ID of concert.\n\n###Input:\n{query}\n\n###Response:""" # noqa: E501
7 +
8 +
9 +def do_sample(llm, lora_path: str, lora_id: int) -> str:
10 + prompts = [
11 + PROMPT_TEMPLATE.format(query="How many singers do we have?"),
12 + PROMPT_TEMPLATE.format(
13 + query=
14 + "What is the average, minimum, and maximum age of all singers from France?" # noqa: E501
15 + ),
16 + PROMPT_TEMPLATE.format(
17 + query=
18 + "Show name, country, age for all singers ordered by age from the oldest to the youngest." # noqa: E501
19 + ),
20 + ]
21 + print(prompts)
22 + sampling_params = vllm.SamplingParams(temperature=0, max_tokens=32)
23 + outputs = llm.generate(
24 + prompts,
25 + sampling_params,
26 + lora_request=LoRARequest(str(lora_id), lora_id, lora_path)
27 + if lora_id else None)
28 + # Print the outputs.
29 + generated_texts = []
30 + for output in outputs:
31 + prompt = output.prompt
32 + generated_text = output.outputs[0].text.strip()
33 + generated_texts.append(generated_text)
34 + print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
35 + return generated_texts
36 +
37 +
38 +def test_chatglm3_lora(chatglm3_lora_files):
39 + llm = vllm.LLM(MODEL_PATH,
40 + max_model_len=1024,
41 + enable_lora=True,
42 + max_loras=4,
43 + max_lora_rank=64,
44 + trust_remote_code=True)
45 +
46 + expected_lora_output = [
47 + "SELECT count(*) FROM singer",
48 + "SELECT avg(age) , min(age) , max(age) FROM singer WHERE country = 'France'", # noqa: E501
49 + "SELECT name , country , age FROM singer ORDER BY age",
50 + ]
51 +
52 + output1 = do_sample(llm, chatglm3_lora_files, lora_id=1)
53 + for i in range(len(expected_lora_output)):
54 + assert output1[i] == expected_lora_output[i]
55 + output2 = do_sample(llm, chatglm3_lora_files, lora_id=2)
56 + for i in range(len(expected_lora_output)):
57 + assert output2[i] == expected_lora_output[i]
tests/lora/test_layers.py
+10 -3
@@ -8,12 +8,16 @@ import torch
8 import torch.nn.functional as F
9
10 from vllm.config import LoRAConfig
11 +# yapf conflicts with isort for this block
12 +# yapf: disable
13 from vllm.lora.layers import (BaseLayerWithLoRA, ColumnParallelLinearWithLoRA,
14 LogitsProcessorWithLoRA, LoRAMapping,
15 MergedColumnParallelLinearWithLoRA,
16 + MergedQKVParallelLinearWithLora,
17 QKVParallelLinearWithLora,
18 RowParallelLinearWithLoRA,
19 VocabParallelEmbeddingWithLoRA)
20 +# yapf: enable
21 from vllm.lora.models import (LoRALayerWeights, PackedLoRALayerWeights,
22 convert_mapping)
23 from vllm.model_executor.layers.linear import (ColumnParallelLinear,
@@ -93,8 +97,7 @@ def populate_loras(
97 lora_dict: Dict[int, LoRALayerWeights] = dict()
98
99 # Dictionary that maps the lora ID to the
96 - # corresponding subloras. Only useful when
97 - # repeats > 1.
100 + # corresponding subloras.
101 sublora_dict: Dict[int, List[LoRALayerWeights]] = dict()
102
103 for slot_idx, lora_id in enumerate(id_to_index):
@@ -607,7 +610,7 @@ def test_linear_parallel(dist_init, num_loras, orientation, device) -> None:
610
611 @torch.inference_mode()
612 @pytest.mark.parametrize("num_loras", [1, 2, 4, 8])
610 -@pytest.mark.parametrize("repeats", [2, 3])
613 +@pytest.mark.parametrize("repeats", [1, 2, 3])
614 @pytest.mark.parametrize("device", CUDA_DEVICES)
615 def test_column_parallel_packed(dist_init, num_loras, repeats, device) -> None:
616
@@ -623,6 +626,10 @@ def test_column_parallel_packed(dist_init, num_loras, repeats, device) -> None:
626 bias=False)
627 linear.weight.data = torch.rand_like(linear.weight.data)
628 lora_linear = MergedColumnParallelLinearWithLoRA(linear)
629 + elif repeats == 3:
630 + linear = QKVParallelLinear(4096, 64, 32, bias=False)
631 + linear.weight.data = torch.rand_like(linear.weight.data)
632 + lora_linear = MergedQKVParallelLinearWithLora(linear)
633 else:
634 linear = QKVParallelLinear(4096, 64, 32, bias=False)
635 linear.weight.data = torch.rand_like(linear.weight.data)
tests/lora/test_punica.py
+4 -3
@@ -43,9 +43,10 @@ def _lora_ref_impl(
43
44
45 H1 = H2 = [
46 - 128, 256, 512, 1024, 1280, 2048, 2560, 2752, 3072, 3456, 3584, 4096, 5120,
47 - 5504, 5632, 6144, 6912, 7168, 8192, 9216, 10240, 11008, 13824, 14336,
48 - 22016, 24576, 32000, 32256, 32512, 32768, 33024
46 + 128, 256, 512, 1024, 1152, 1280, 1536, 2048, 2304, 2560, 2752, 3072, 3456,
47 + 3584, 4096, 4608, 5120, 5504, 5632, 6144, 6848, 6912, 7168, 8192, 9216,
48 + 10240, 11008, 13824, 14336, 22016, 24576, 27392, 32000, 32256, 32512,
49 + 32768, 33024
50 ]
51 SEED = [0xabcdabcd987]
52
vllm/lora/layers.py
+146 -22
@@ -1,7 +1,8 @@
1 # pylint: disable=unused-argument
2 +import inspect
3 import math
4 from dataclasses import dataclass
4 -from typing import TYPE_CHECKING, List, Optional, Tuple
5 +from typing import TYPE_CHECKING, List, Optional, Set, Tuple, Type
6
7 import torch
8 import torch.nn as nn
@@ -114,8 +115,11 @@ class LoRAMapping:
115
116 class BaseLayerWithLoRA(nn.Module):
117
117 - def create_lora_weights(self, max_loras: int, lora_config: LoRAConfig,
118 - model_config: PretrainedConfig) -> None:
118 + def create_lora_weights(
119 + self,
120 + max_loras: int,
121 + lora_config: LoRAConfig,
122 + model_config: Optional[PretrainedConfig] = None) -> None:
123 """Initializes lora matrices."""
124 ...
125
@@ -144,6 +148,13 @@ class BaseLayerWithLoRA(nn.Module):
148 """Sets the mapping indices."""
149 ...
150
151 + @classmethod
152 + def can_replace_layer(cls, source_layer: nn.Module,
153 + lora_config: LoRAConfig, packed_modules_list: List,
154 + model_config: Optional[PretrainedConfig]) -> bool:
155 + """Returns True if the layer can be replaced by this LoRA layer."""
156 + raise NotImplementedError
157 +
158
159 class VocabParallelEmbeddingWithLoRA(BaseLayerWithLoRA):
160
@@ -278,12 +289,19 @@ class VocabParallelEmbeddingWithLoRA(BaseLayerWithLoRA):
289 self.indices[:self.indices_len[0]], 0, 1.0)
290 return full_output.view_as(full_output_org)
291
292 + @classmethod
293 + def can_replace_layer(cls, source_layer: nn.Module,
294 + lora_config: LoRAConfig, packed_modules_list: List,
295 + model_config: Optional[PretrainedConfig]) -> bool:
296 + return type(source_layer) is VocabParallelEmbedding
297 +
298
299 class ColumnParallelLinearWithLoRA(BaseLayerWithLoRA):
300
301 def __init__(self, base_layer: ColumnParallelLinear) -> None:
302 super().__init__()
303 self.base_layer = base_layer
304 + self.tp_size = get_tensor_model_parallel_world_size()
305
306 def create_lora_weights(
307 self,
@@ -309,7 +327,7 @@ class ColumnParallelLinearWithLoRA(BaseLayerWithLoRA):
327
328 self.indices: Optional[torch.Tensor] = None
329 self.indices_len: Optional[List[int]] = None
312 - self.output_dim = self.lora_b_stacked.shape[1]
330 + self.output_dim = self.lora_b_stacked.shape[2]
331
332 def reset_lora(self, index: int):
333 self.lora_a_stacked[index] = 0
@@ -323,7 +341,12 @@ class ColumnParallelLinearWithLoRA(BaseLayerWithLoRA):
341 embeddings_tensor: Optional[torch.Tensor],
342 ):
343 self.reset_lora(index)
326 -
344 + if self.tp_size > 1:
345 + tensor_model_parallel_rank = get_tensor_model_parallel_rank()
346 + shard_size = self.output_dim
347 + start_idx = tensor_model_parallel_rank * shard_size
348 + end_idx = (tensor_model_parallel_rank + 1) * shard_size
349 + lora_b = lora_b[:, start_idx:end_idx]
350 self.lora_a_stacked[index,
351 0, :lora_a.shape[1], :lora_a.shape[0]].copy_(
352 lora_a.T, non_blocking=True)
@@ -383,6 +406,14 @@ class ColumnParallelLinearWithLoRA(BaseLayerWithLoRA):
406 def linear_weights(self):
407 return self.base_layer.linear_weights
408
409 + @classmethod
410 + def can_replace_layer(cls, source_layer: nn.Module,
411 + lora_config: LoRAConfig, packed_modules_list: List,
412 + model_config: Optional[PretrainedConfig]) -> bool:
413 + return type(source_layer) is ColumnParallelLinear or (
414 + type(source_layer) is MergedColumnParallelLinear
415 + and len(packed_modules_list) == 1)
416 +
417
418 class MergedColumnParallelLinearWithLoRA(ColumnParallelLinearWithLoRA):
419 """ColumnParallelLinear layer that is composed of 2 sublayers (slices)
@@ -485,8 +516,80 @@ class MergedColumnParallelLinearWithLoRA(ColumnParallelLinearWithLoRA):
516 )
517 return output
518
519 + @classmethod
520 + def can_replace_layer(cls, source_layer: nn.Module,
521 + lora_config: LoRAConfig, packed_modules_list: List,
522 + model_config: Optional[PretrainedConfig]) -> bool:
523 + return type(source_layer) is MergedColumnParallelLinear and len(
524 + packed_modules_list) == 2
525 +
526
527 class QKVParallelLinearWithLora(ColumnParallelLinearWithLoRA):
528 + """
529 + ColumnParallelLinear layer that is specifically designed for
530 + qkv_proj. Certain models, such as chtglm3 and baichuan-7b,
531 + only contains a single LoRA within their qkv_proj layer.
532 +
533 + During inference with Tensor Parallel, the weights of lora_b
534 + must be accurately partitioned according to the respective ranks.
535 +
536 + Q slice may have different shape than K and V slices (which both have
537 + the same shape).
538 + """
539 +
540 + def __init__(self, base_layer: QKVParallelLinear) -> None:
541 + super().__init__(base_layer)
542 + self.tp_size = get_tensor_model_parallel_world_size()
543 + self.q_proj_total_size = (self.base_layer.total_num_heads *
544 + self.base_layer.head_size)
545 + self.q_proj_shard_size = (self.base_layer.num_heads *
546 + self.base_layer.head_size)
547 + self.kv_proj_shard_size = (self.base_layer.num_kv_heads *
548 + self.base_layer.head_size)
549 + self.kv_proj_total_size = (self.base_layer.total_num_kv_heads *
550 + self.base_layer.head_size)
551 +
552 + def set_lora(
553 + self,
554 + index: int,
555 + lora_a: torch.Tensor,
556 + lora_b: torch.Tensor,
557 + embeddings_tensor: Optional[torch.Tensor],
558 + ):
559 + self.reset_lora(index)
560 + if self.tp_size > 1:
561 + tp_rank = get_tensor_model_parallel_rank()
562 + self.q_shard_id = tp_rank
563 + self.kv_shard_id = tp_rank // self.base_layer.num_kv_head_replicas
564 + lora_b_q = lora_b[:, self.q_proj_shard_size *
565 + self.q_shard_id:self.q_proj_shard_size *
566 + (self.q_shard_id + 1)]
567 + k_offset = self.q_proj_total_size
568 + lora_b_k = lora_b[:, k_offset + self.kv_proj_shard_size *
569 + self.kv_shard_id:k_offset +
570 + self.kv_proj_shard_size * (self.kv_shard_id + 1)]
571 + v_offset = k_offset + self.kv_proj_total_size
572 + lora_b_v = lora_b[:, v_offset + self.kv_proj_shard_size *
573 + self.kv_shard_id:v_offset +
574 + self.kv_proj_shard_size * (self.kv_shard_id + 1)]
575 + lora_b = torch.cat([lora_b_q, lora_b_k, lora_b_v], dim=1)
576 +
577 + self.lora_a_stacked[index,
578 + 0, :lora_a.shape[1], :lora_a.shape[0]].copy_(
579 + lora_a.T, non_blocking=True)
580 + self.lora_b_stacked[index,
581 + 0, :lora_b.shape[1], :lora_b.shape[0]].copy_(
582 + lora_b.T, non_blocking=True)
583 +
584 + @classmethod
585 + def can_replace_layer(cls, source_layer: nn.Module,
586 + lora_config: LoRAConfig, packed_modules_list: List,
587 + model_config: Optional[PretrainedConfig]) -> bool:
588 + return type(source_layer) is QKVParallelLinear and len(
589 + packed_modules_list) == 1
590 +
591 +
592 +class MergedQKVParallelLinearWithLora(ColumnParallelLinearWithLoRA):
593 """ColumnParallelLinear layer that is composed of 3 sublayers (slices)
594 packed together in qkv proj fashion
595 (q_proj + k_proj + v_proj -> qkv_proj).
@@ -654,6 +757,13 @@ class QKVParallelLinearWithLora(ColumnParallelLinearWithLoRA):
757 )
758 return output
759
760 + @classmethod
761 + def can_replace_layer(cls, source_layer: nn.Module,
762 + lora_config: LoRAConfig, packed_modules_list: List,
763 + model_config: Optional[PretrainedConfig]) -> bool:
764 + return type(source_layer) is QKVParallelLinear and len(
765 + packed_modules_list) == 3
766 +
767
768 class RowParallelLinearWithLoRA(BaseLayerWithLoRA):
769
@@ -780,6 +890,12 @@ class RowParallelLinearWithLoRA(BaseLayerWithLoRA):
890 def weight(self):
891 return self.base_layer.weight
892
893 + @classmethod
894 + def can_replace_layer(cls, source_layer: nn.Module,
895 + lora_config: LoRAConfig, packed_modules_list: List,
896 + model_config: Optional[PretrainedConfig]) -> bool:
897 + return type(source_layer) is RowParallelLinear
898 +
899
900 class LogitsProcessorWithLoRA(BaseLayerWithLoRA):
901
@@ -900,7 +1016,7 @@ class LogitsProcessorWithLoRA(BaseLayerWithLoRA):
1016 hidden_states: torch.Tensor,
1017 embedding: torch.Tensor,
1018 embedding_bias: Optional[torch.Tensor] = None,
903 - ) -> torch.Tensor:
1019 + ) -> Optional[torch.Tensor]:
1020 # Get the logits for the next tokens.
1021 logits = torch.matmul(hidden_states, embedding.t())
1022 if embedding_bias is not None:
@@ -949,22 +1065,30 @@ class LogitsProcessorWithLoRA(BaseLayerWithLoRA):
1065 def forward(self, *args, **kwargs):
1066 return type(self.base_layer).forward(self, *args, **kwargs)
1067
952 -
953 -def from_layer(
954 - layer: nn.Module,
955 - max_loras: int,
956 - lora_config: LoRAConfig,
957 - model_config: Optional[PretrainedConfig] = None) -> BaseLayerWithLoRA:
958 - supported_layer_types = {
959 - VocabParallelEmbedding: VocabParallelEmbeddingWithLoRA,
960 - ColumnParallelLinear: ColumnParallelLinearWithLoRA,
961 - QKVParallelLinear: QKVParallelLinearWithLora,
962 - MergedColumnParallelLinear: MergedColumnParallelLinearWithLoRA,
963 - RowParallelLinear: RowParallelLinearWithLoRA,
964 - }
965 - for src_layer_type, lora_layer_type in supported_layer_types.items():
966 - if type(layer) is src_layer_type: # pylint: disable=unidiomatic-typecheck
967 - ret = lora_layer_type(layer)
1068 + @classmethod
1069 + def can_replace_layer(cls, source_layer: nn.Module,
1070 + lora_config: LoRAConfig, packed_modules_list: List,
1071 + model_config: Optional[PretrainedConfig]) -> bool:
1072 + # Special handling for the LogitsProcessor.
1073 + return False
1074 +
1075 +
1076 +_all_lora_classes: Set[Type[BaseLayerWithLoRA]] = {
1077 + cls
1078 + for cls in globals().values() if inspect.isclass(cls)
1079 + and issubclass(cls, BaseLayerWithLoRA) and cls is not BaseLayerWithLoRA
1080 +}
1081 +
1082 +
1083 +def from_layer(layer: nn.Module,
1084 + max_loras: int,
1085 + lora_config: LoRAConfig,
1086 + packed_modules_list: List,
1087 + model_config: Optional[PretrainedConfig] = None) -> nn.Module:
1088 + for lora_cls in _all_lora_classes:
1089 + if lora_cls.can_replace_layer(layer, lora_config, packed_modules_list,
1090 + model_config):
1091 + ret = lora_cls(layer)
1092 ret.create_lora_weights(max_loras, lora_config, model_config)
1093 return ret
1094 return layer
vllm/lora/models.py
+7 -4
@@ -413,11 +413,12 @@ class LoRAModelManager:
413 for module_name, module in self.model.named_modules():
414 if not self._match_target_modules(module_name):
415 continue
416 -
416 + parts = module_name.split(".")[-1]
417 + packed_moduled_lst = self.packed_modules_mapping.get(parts, [])
418 new_module = replace_submodule(
419 self.model, module_name,
420 from_layer(module, self.lora_slots, self.lora_config,
420 - self.model.config))
421 + packed_moduled_lst, self.model.config))
422 # (yard1): TODO make this more robust
423 if "lm_head" in module_name:
424 logits_processor_module = self.model.get_submodule(
@@ -510,8 +511,10 @@ class LoRAModelManager:
511 def _register_packed_modules(self, module_full_name: str) -> None:
512 parts = module_full_name.split(".")
513 module_name = parts[-1]
513 - replacements = self.packed_modules_mapping.get(module_name)
514 - if not replacements:
514 + replacements = self.packed_modules_mapping.get(module_name, [])
515 + # When replacements is less than or equal to 1, it indicates that this
516 + # module is not a packed module.
517 + if len(replacements) <= 1:
518 return
519 prefix = ".".join(parts[:-1])
520 self.packed_modules[module_full_name] = [
vllm/model_executor/models/baichuan.py
+39 -13
@@ -26,6 +26,7 @@ from torch import nn
26 from transformers import PretrainedConfig
27
28 from vllm.attention import Attention, AttentionMetadata
29 +from vllm.config import LoRAConfig
30 from vllm.model_executor.layers.activation import SiluAndMul
31 from vllm.model_executor.layers.layernorm import RMSNorm
32 from vllm.model_executor.layers.linear import (LinearMethodBase,
@@ -282,11 +283,30 @@ class BaiChuanModel(nn.Module):
283
284
285 class BaiChuanBaseForCausalLM(nn.Module):
286 + packed_modules_mapping = {
287 + "W_pack": ["W_pack"],
288 + "gate_up_proj": [
289 + "gate_proj",
290 + "up_proj",
291 + ],
292 + }
293 + # LoRA specific attributes
294 + supported_lora_modules = [
295 + "W_pack",
296 + "o_proj",
297 + "gate_up_proj",
298 + "down_proj",
299 + ]
300 + embedding_modules = {}
301 + embedding_padding_modules = []
302
286 - def __init__(self,
287 - config,
288 - position_embedding: str,
289 - linear_method: Optional[LinearMethodBase] = None):
303 + def __init__(
304 + self,
305 + config,
306 + position_embedding: str,
307 + linear_method: Optional[LinearMethodBase] = None,
308 + lora_config: Optional[LoRAConfig] = None,
309 + ):
310 super().__init__()
311 self.config = config
312 self.linear_method = linear_method
@@ -371,19 +391,25 @@ class BaiChuanBaseForCausalLM(nn.Module):
391 class BaichuanForCausalLM(BaiChuanBaseForCausalLM):
392 """Baichuan 13B and Baichuan2 7B/13B."""
393
374 - def __init__(self,
375 - config,
376 - linear_method: Optional[LinearMethodBase] = None):
394 + def __init__(
395 + self,
396 + config,
397 + linear_method: Optional[LinearMethodBase] = None,
398 + lora_config: Optional[LoRAConfig] = None,
399 + ):
400 if config.hidden_size == 4096: # baichuan2 7b
378 - super().__init__(config, "ROPE", linear_method)
401 + super().__init__(config, "ROPE", linear_method, lora_config)
402 else: # baichuan 13b, baichuan2 13b
380 - super().__init__(config, "ALIBI", linear_method)
403 + super().__init__(config, "ALIBI", linear_method, lora_config)
404
405
406 class BaiChuanForCausalLM(BaiChuanBaseForCausalLM):
407 """Baichuan 7B."""
408
386 - def __init__(self,
387 - config,
388 - linear_method: Optional[LinearMethodBase] = None):
389 - super().__init__(config, "ROPE", linear_method)
409 + def __init__(
410 + self,
411 + config,
412 + linear_method: Optional[LinearMethodBase] = None,
413 + lora_config: Optional[LoRAConfig] = None,
414 + ):
415 + super().__init__(config, "ROPE", linear_method, lora_config)
vllm/model_executor/models/chatglm.py
+15
@@ -9,6 +9,7 @@ from torch import nn
9 from torch.nn import LayerNorm
10
11 from vllm.attention import Attention, AttentionMetadata
12 +from vllm.config import LoRAConfig
13 from vllm.model_executor.layers.activation import SiluAndMul
14 from vllm.model_executor.layers.layernorm import RMSNorm
15 from vllm.model_executor.layers.linear import (LinearMethodBase,
@@ -317,11 +318,25 @@ class ChatGLMModel(nn.Module):
318
319
320 class ChatGLMForCausalLM(nn.Module):
321 + packed_modules_mapping = {
322 + "query_key_value": ["query_key_value"],
323 + "dense_h_to_4h": ["dense_h_to_4h"]
324 + }
325 + # LoRA specific attributes
326 + supported_lora_modules = [
327 + "query_key_value",
328 + "dense",
329 + "dense_h_to_4h",
330 + "dense_4h_to_h",
331 + ]
332 + embedding_modules = {}
333 + embedding_padding_modules = []
334
335 def __init__(
336 self,
337 config: ChatGLMConfig,
338 linear_method: Optional[LinearMethodBase] = None,
339 + lora_config: Optional[LoRAConfig] = None,
340 ):
341 super().__init__()
342 self.config: ChatGLMConfig = config