A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 03:00 UTC
[Core] Support LoRA on quantized models (#4012)
Jee Li committed
Apr 12, 2024 at 12:02 UTC
1096717ae9e0b414ad625c1a12354dd1d949ffb1
4 files changed
+234
-26
tests/lora/conftest.py
+5
@@ -143,6 +143,11 @@ def baichuan_lora_files():
143
return snapshot_download(repo_id="jeeejeee/baichuan7b-text2sql-spider")
144
145
146
+@pytest.fixture(scope="session")
147
+def tinyllama_lora_files():
148
+ return snapshot_download(repo_id="jashing/tinyllama-colorist-lora")
149
+
150
+
151
@pytest.fixture
152
def llama_2_7b_engine_extra_embeddings() -> nn.Module:
153
cleanup()
tests/lora/test_quant_model.py
new
+179
@@ -0,0 +1,179 @@
1
+# Adapted from
2
+# https://github.com/fmmoret/vllm/blob/fm-support-lora-on-quantized-models/tests/lora/test_llama.py
3
+from dataclasses import dataclass
4
+from typing import List
5
+
6
+import pytest
7
+
8
+import vllm
9
+from vllm.lora.request import LoRARequest
10
+
11
+from .conftest import cleanup
12
+
13
+
14
+@dataclass
15
+class ModelWithQuantization:
16
+ model_path: str
17
+ quantization: str
18
+
19
+
20
+MODELS: List[ModelWithQuantization] = [
21
+ ModelWithQuantization(model_path="TheBloke/TinyLlama-1.1B-Chat-v0.3-AWQ",
22
+ quantization="AWQ"),
23
+ ModelWithQuantization(model_path="TheBloke/TinyLlama-1.1B-Chat-v0.3-GPTQ",
24
+ quantization="GPTQ"),
25
+]
26
+
27
+
28
+def do_sample(llm, lora_path: str, lora_id: int, max_tokens=256):
29
+ raw_prompts = [
30
+ "Give me an orange-ish brown color",
31
+ "Give me a neon pink color",
32
+ ]
33
+
34
+ def format_prompt_tuples(prompt):
35
+ return f"<|im_start|>user\n{prompt}<|im_end|>\n<|im_start|>assistant\n"
36
+
37
+ prompts = [format_prompt_tuples(p) for p in raw_prompts]
38
+
39
+ sampling_params = vllm.SamplingParams(temperature=0,
40
+ max_tokens=max_tokens,
41
+ stop=["<|im_end|>"])
42
+ outputs = llm.generate(
43
+ prompts,
44
+ sampling_params,
45
+ lora_request=LoRARequest(str(lora_id), lora_id, lora_path)
46
+ if lora_id else None)
47
+ # Print the outputs.
48
+ generated_texts = []
49
+ for output in outputs:
50
+ prompt = output.prompt
51
+ generated_text = output.outputs[0].text
52
+ generated_texts.append(generated_text)
53
+ print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
54
+ return generated_texts
55
+
56
+
57
+@pytest.mark.parametrize("model", MODELS)
58
+@pytest.mark.parametrize("tp_size", [1])
59
+def test_quant_model_lora(tinyllama_lora_files, model, tp_size):
60
+ # Cannot use as it will initialize torch.cuda too early...
61
+ # if torch.cuda.device_count() < tp_size:
62
+ # pytest.skip(f"Not enough GPUs for tensor parallelism {tp_size}")
63
+
64
+ llm = vllm.LLM(model=model.model_path,
65
+ enable_lora=True,
66
+ max_num_seqs=16,
67
+ max_loras=4,
68
+ max_model_len=400,
69
+ tensor_parallel_size=tp_size,
70
+ quantization=model.quantization,
71
+ trust_remote_code=True)
72
+
73
+ if model.quantization is None:
74
+ expected_no_lora_output = [
75
+ "Here are some examples of orange-brown colors",
76
+ "I'm sorry, I don't have"
77
+ ]
78
+ expected_lora_output = [
79
+ "#ff8050",
80
+ "#ff8080",
81
+ ]
82
+ elif model.quantization == "AWQ":
83
+ expected_no_lora_output = [
84
+ "I'm sorry, I don't understand",
85
+ "I'm sorry, I don't understand",
86
+ ]
87
+ expected_lora_output = [
88
+ "#f07700: A v",
89
+ "#f00000: A v",
90
+ ]
91
+ elif model.quantization == "GPTQ":
92
+ expected_no_lora_output = [
93
+ "I'm sorry, I don't have",
94
+ "I'm sorry, I don't have",
95
+ ]
96
+ expected_lora_output = [
97
+ "#f08800: This is",
98
+ "#f07788 \n#",
99
+ ]
100
+
101
+ def expect_match(output, expected_output):
102
+ # HACK: GPTQ lora outputs are just incredibly unstable.
103
+ # Assert that the outputs changed.
104
+ if (model.quantization == "GPTQ"
105
+ and expected_output is expected_lora_output):
106
+ assert output != expected_no_lora_output
107
+ for i, o in enumerate(output):
108
+ assert o.startswith(
109
+ '#'), f"Expected example {i} to start with # but got {o}"
110
+ return
111
+ assert output == expected_output
112
+
113
+ max_tokens = 10
114
+
115
+ print("lora adapter created")
116
+ output = do_sample(llm,
117
+ tinyllama_lora_files,
118
+ lora_id=0,
119
+ max_tokens=max_tokens)
120
+ expect_match(output, expected_no_lora_output)
121
+
122
+ print("lora 1")
123
+ output = do_sample(llm,
124
+ tinyllama_lora_files,
125
+ lora_id=1,
126
+ max_tokens=max_tokens)
127
+ expect_match(output, expected_lora_output)
128
+
129
+ print("no lora")
130
+ output = do_sample(llm,
131
+ tinyllama_lora_files,
132
+ lora_id=0,
133
+ max_tokens=max_tokens)
134
+ expect_match(output, expected_no_lora_output)
135
+
136
+ print("lora 2")
137
+ output = do_sample(llm,
138
+ tinyllama_lora_files,
139
+ lora_id=2,
140
+ max_tokens=max_tokens)
141
+ expect_match(output, expected_lora_output)
142
+
143
+ print("removing lora")
144
+
145
+ del llm
146
+ cleanup()
147
+
148
+
149
+@pytest.mark.parametrize("model", MODELS)
150
+@pytest.mark.skip("Requires multiple GPUs")
151
+def test_quant_model_tp_equality(tinyllama_lora_files, model):
152
+ # Cannot use as it will initialize torch.cuda too early...
153
+ # if torch.cuda.device_count() < 2:
154
+ # pytest.skip(f"Not enough GPUs for tensor parallelism {2}")
155
+
156
+ llm_tp1 = vllm.LLM(model=model.model_path,
157
+ enable_lora=True,
158
+ max_num_seqs=16,
159
+ max_loras=4,
160
+ tensor_parallel_size=1,
161
+ quantization=model.quantization,
162
+ trust_remote_code=True)
163
+ output_tp1 = do_sample(llm_tp1, tinyllama_lora_files, lora_id=1)
164
+
165
+ del llm_tp1
166
+ cleanup()
167
+
168
+ llm_tp2 = vllm.LLM(model=model.model_path,
169
+ enable_lora=True,
170
+ max_num_seqs=16,
171
+ max_loras=4,
172
+ tensor_parallel_size=2,
173
+ quantization=model.quantization)
174
+ output_tp2 = do_sample(llm_tp2, tinyllama_lora_files, lora_id=1)
175
+
176
+ del llm_tp2
177
+ cleanup()
178
+
179
+ assert output_tp1 == output_tp2
vllm/config.py
+6
-3
@@ -822,9 +822,12 @@ class LoRAConfig:
822
self.lora_dtype = model_config.dtype
823
elif isinstance(self.lora_dtype, str):
824
self.lora_dtype = getattr(torch, self.lora_dtype)
825
- if model_config.quantization is not None:
826
- raise ValueError(
827
- "LoRA is not supported with quantized models yet.")
825
+ if model_config.quantization and model_config.quantization not in [
826
+ "awq", "gptq"
827
+ ]:
828
+ # TODO support marlin and squeezellm
829
+ logger.warning(f"{model_config.quantization} quantization is not "
830
+ "tested with LoRA yet.")
831
832
def verify_with_scheduler_config(self, scheduler_config: SchedulerConfig):
833
if scheduler_config.max_num_batched_tokens > 65528:
vllm/lora/layers.py
+44
-23
@@ -29,6 +29,19 @@ if TYPE_CHECKING:
29
pass
30
31
32
+def _get_lora_device(base_layer: nn.Module) -> torch.device:
33
+ # code borrowed from https://github.com/fmmoret/vllm/blob/fm-support-lora-on-quantized-models/vllm/lora/layers.py#L34
34
+ """Returns the device for where to place the LoRA tensors."""
35
+ if hasattr(base_layer, "weight"):
36
+ return base_layer.weight.device
37
+ if hasattr(base_layer, "linear_weights") and isinstance(
38
+ base_layer.linear_weights, dict):
39
+ values = list(base_layer.linear_weights.values())
40
+ if len(values) and isinstance(values[0], torch.Tensor):
41
+ return values[0].device
42
+ raise ValueError(f"Unsupported base layer: {base_layer}")
43
+
44
+
45
def _apply_lora(
46
x: torch.Tensor,
47
lora_a_stacked: torch.Tensor,
@@ -302,6 +315,9 @@ class ColumnParallelLinearWithLoRA(BaseLayerWithLoRA):
315
super().__init__()
316
self.base_layer = base_layer
317
self.tp_size = get_tensor_model_parallel_world_size()
318
+ self.input_size = self.base_layer.input_size
319
+ self.output_size = self.base_layer.output_size_per_partition
320
+ self.device = _get_lora_device(self.base_layer)
321
322
def create_lora_weights(
323
self,
@@ -312,17 +328,17 @@ class ColumnParallelLinearWithLoRA(BaseLayerWithLoRA):
328
max_loras,
329
1,
330
lora_config.max_lora_rank,
315
- self.base_layer.weight.shape[1],
331
+ self.input_size,
332
dtype=lora_config.lora_dtype,
317
- device=self.base_layer.weight.device,
333
+ device=self.device,
334
)
335
self.lora_b_stacked = torch.zeros(
336
max_loras,
337
1,
322
- self.base_layer.weight.shape[0],
338
+ self.output_size,
339
lora_config.max_lora_rank,
340
dtype=lora_config.lora_dtype,
325
- device=self.base_layer.weight.device,
341
+ device=self.device,
342
)
343
344
self.indices: Optional[torch.Tensor] = None
@@ -442,18 +458,18 @@ class MergedColumnParallelLinearWithLoRA(ColumnParallelLinearWithLoRA):
458
max_loras,
459
1,
460
lora_config.max_lora_rank,
445
- self.base_layer.weight.shape[1],
461
+ self.input_size,
462
dtype=lora_config.lora_dtype,
447
- device=self.base_layer.weight.device,
463
+ device=self.device,
464
) for _ in range(n_slices))
465
self.lora_b_stacked = tuple(
466
torch.zeros(
467
max_loras,
468
1,
453
- self.base_layer.weight.shape[0] // 2,
469
+ self.output_size // 2,
470
lora_config.max_lora_rank,
471
dtype=lora_config.lora_dtype,
456
- device=self.base_layer.weight.device,
472
+ device=self.device,
473
) for _ in range(n_slices))
474
475
self.indices: Optional[torch.Tensor] = None
@@ -619,25 +635,25 @@ class MergedQKVParallelLinearWithLora(ColumnParallelLinearWithLoRA):
635
max_loras,
636
1,
637
lora_config.max_lora_rank,
622
- self.base_layer.weight.shape[1],
638
+ self.input_size,
639
dtype=lora_config.lora_dtype,
624
- device=self.base_layer.weight.device,
640
+ device=self.device,
641
),
642
torch.zeros(
643
max_loras,
644
1,
645
lora_config.max_lora_rank,
630
- self.base_layer.weight.shape[1],
646
+ self.input_size,
647
dtype=lora_config.lora_dtype,
632
- device=self.base_layer.weight.device,
648
+ device=self.device,
649
),
650
torch.zeros(
651
max_loras,
652
1,
653
lora_config.max_lora_rank,
638
- self.base_layer.weight.shape[1],
654
+ self.input_size,
655
dtype=lora_config.lora_dtype,
640
- device=self.base_layer.weight.device,
656
+ device=self.device,
657
),
658
)
659
self.lora_b_stacked = (
@@ -647,7 +663,7 @@ class MergedQKVParallelLinearWithLora(ColumnParallelLinearWithLoRA):
663
self.q_proj_shard_size,
664
lora_config.max_lora_rank,
665
dtype=lora_config.lora_dtype,
650
- device=self.base_layer.weight.device,
666
+ device=self.device,
667
),
668
torch.zeros(
669
max_loras,
@@ -655,7 +671,7 @@ class MergedQKVParallelLinearWithLora(ColumnParallelLinearWithLoRA):
671
self.kv_proj_shard_size,
672
lora_config.max_lora_rank,
673
dtype=lora_config.lora_dtype,
658
- device=self.base_layer.weight.device,
674
+ device=self.device,
675
),
676
torch.zeros(
677
max_loras,
@@ -663,7 +679,7 @@ class MergedQKVParallelLinearWithLora(ColumnParallelLinearWithLoRA):
679
self.kv_proj_shard_size,
680
lora_config.max_lora_rank,
681
dtype=lora_config.lora_dtype,
666
- device=self.base_layer.weight.device,
682
+ device=self.device,
683
),
684
)
685
@@ -766,6 +782,9 @@ class RowParallelLinearWithLoRA(BaseLayerWithLoRA):
782
def __init__(self, base_layer: RowParallelLinear) -> None:
783
super().__init__()
784
self.base_layer = base_layer
785
+ self.input_size = self.base_layer.input_size_per_partition
786
+ self.output_size = self.base_layer.output_size
787
+ self.device = _get_lora_device(self.base_layer)
788
789
def create_lora_weights(
790
self,
@@ -777,20 +796,20 @@ class RowParallelLinearWithLoRA(BaseLayerWithLoRA):
796
max_loras,
797
1,
798
lora_config.max_lora_rank,
780
- self.base_layer.weight.shape[1],
799
+ self.input_size,
800
),
801
dtype=lora_config.lora_dtype,
783
- device=self.base_layer.weight.device,
802
+ device=self.device,
803
)
804
self.lora_b_stacked = torch.zeros(
805
(
806
max_loras,
807
1,
789
- self.base_layer.weight.shape[0],
808
+ self.output_size,
809
lora_config.max_lora_rank,
810
),
811
dtype=lora_config.lora_dtype,
793
- device=self.base_layer.weight.device,
812
+ device=self.device,
813
)
814
self.indices: Optional[torch.Tensor] = None
815
self.indices_len: Optional[List[int]] = None
@@ -809,7 +828,7 @@ class RowParallelLinearWithLoRA(BaseLayerWithLoRA):
828
self.reset_lora(index)
829
if self.base_layer.tp_size > 1:
830
tensor_model_parallel_rank = get_tensor_model_parallel_rank()
812
- shard_size = self.base_layer.weight.shape[1]
831
+ shard_size = self.input_size
832
start_idx = tensor_model_parallel_rank * shard_size
833
end_idx = (tensor_model_parallel_rank + 1) * shard_size
834
lora_a = lora_a[start_idx:end_idx, :]
@@ -884,7 +903,9 @@ class RowParallelLinearWithLoRA(BaseLayerWithLoRA):
903
904
@property
905
def weight(self):
887
- return self.base_layer.weight
906
+
907
+ return self.base_layer.weight if hasattr(
908
+ self.base_layer, "weight") else self.base_layer.qweight
909
910
@classmethod
911
def can_replace_layer(cls, source_layer: nn.Module,