@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 1096717ae9
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 03:00 UTC

[Core] Support LoRA on quantized models (#4012)

Jee Li committed Apr 12, 2024 at 12:02 UTC 1096717ae9e0b414ad625c1a12354dd1d949ffb1
4 files changed +234 -26
tests/lora/conftest.py
+5
@@ -143,6 +143,11 @@ def baichuan_lora_files():
143 return snapshot_download(repo_id="jeeejeee/baichuan7b-text2sql-spider")
144
145
146 +@pytest.fixture(scope="session")
147 +def tinyllama_lora_files():
148 + return snapshot_download(repo_id="jashing/tinyllama-colorist-lora")
149 +
150 +
151 @pytest.fixture
152 def llama_2_7b_engine_extra_embeddings() -> nn.Module:
153 cleanup()
tests/lora/test_quant_model.py new
+179
@@ -0,0 +1,179 @@
1 +# Adapted from
2 +# https://github.com/fmmoret/vllm/blob/fm-support-lora-on-quantized-models/tests/lora/test_llama.py
3 +from dataclasses import dataclass
4 +from typing import List
5 +
6 +import pytest
7 +
8 +import vllm
9 +from vllm.lora.request import LoRARequest
10 +
11 +from .conftest import cleanup
12 +
13 +
14 +@dataclass
15 +class ModelWithQuantization:
16 + model_path: str
17 + quantization: str
18 +
19 +
20 +MODELS: List[ModelWithQuantization] = [
21 + ModelWithQuantization(model_path="TheBloke/TinyLlama-1.1B-Chat-v0.3-AWQ",
22 + quantization="AWQ"),
23 + ModelWithQuantization(model_path="TheBloke/TinyLlama-1.1B-Chat-v0.3-GPTQ",
24 + quantization="GPTQ"),
25 +]
26 +
27 +
28 +def do_sample(llm, lora_path: str, lora_id: int, max_tokens=256):
29 + raw_prompts = [
30 + "Give me an orange-ish brown color",
31 + "Give me a neon pink color",
32 + ]
33 +
34 + def format_prompt_tuples(prompt):
35 + return f"<|im_start|>user\n{prompt}<|im_end|>\n<|im_start|>assistant\n"
36 +
37 + prompts = [format_prompt_tuples(p) for p in raw_prompts]
38 +
39 + sampling_params = vllm.SamplingParams(temperature=0,
40 + max_tokens=max_tokens,
41 + stop=["<|im_end|>"])
42 + outputs = llm.generate(
43 + prompts,
44 + sampling_params,
45 + lora_request=LoRARequest(str(lora_id), lora_id, lora_path)
46 + if lora_id else None)
47 + # Print the outputs.
48 + generated_texts = []
49 + for output in outputs:
50 + prompt = output.prompt
51 + generated_text = output.outputs[0].text
52 + generated_texts.append(generated_text)
53 + print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
54 + return generated_texts
55 +
56 +
57 +@pytest.mark.parametrize("model", MODELS)
58 +@pytest.mark.parametrize("tp_size", [1])
59 +def test_quant_model_lora(tinyllama_lora_files, model, tp_size):
60 + # Cannot use as it will initialize torch.cuda too early...
61 + # if torch.cuda.device_count() < tp_size:
62 + # pytest.skip(f"Not enough GPUs for tensor parallelism {tp_size}")
63 +
64 + llm = vllm.LLM(model=model.model_path,
65 + enable_lora=True,
66 + max_num_seqs=16,
67 + max_loras=4,
68 + max_model_len=400,
69 + tensor_parallel_size=tp_size,
70 + quantization=model.quantization,
71 + trust_remote_code=True)
72 +
73 + if model.quantization is None:
74 + expected_no_lora_output = [
75 + "Here are some examples of orange-brown colors",
76 + "I'm sorry, I don't have"
77 + ]
78 + expected_lora_output = [
79 + "#ff8050",
80 + "#ff8080",
81 + ]
82 + elif model.quantization == "AWQ":
83 + expected_no_lora_output = [
84 + "I'm sorry, I don't understand",
85 + "I'm sorry, I don't understand",
86 + ]
87 + expected_lora_output = [
88 + "#f07700: A v",
89 + "#f00000: A v",
90 + ]
91 + elif model.quantization == "GPTQ":
92 + expected_no_lora_output = [
93 + "I'm sorry, I don't have",
94 + "I'm sorry, I don't have",
95 + ]
96 + expected_lora_output = [
97 + "#f08800: This is",
98 + "#f07788 \n#",
99 + ]
100 +
101 + def expect_match(output, expected_output):
102 + # HACK: GPTQ lora outputs are just incredibly unstable.
103 + # Assert that the outputs changed.
104 + if (model.quantization == "GPTQ"
105 + and expected_output is expected_lora_output):
106 + assert output != expected_no_lora_output
107 + for i, o in enumerate(output):
108 + assert o.startswith(
109 + '#'), f"Expected example {i} to start with # but got {o}"
110 + return
111 + assert output == expected_output
112 +
113 + max_tokens = 10
114 +
115 + print("lora adapter created")
116 + output = do_sample(llm,
117 + tinyllama_lora_files,
118 + lora_id=0,
119 + max_tokens=max_tokens)
120 + expect_match(output, expected_no_lora_output)
121 +
122 + print("lora 1")
123 + output = do_sample(llm,
124 + tinyllama_lora_files,
125 + lora_id=1,
126 + max_tokens=max_tokens)
127 + expect_match(output, expected_lora_output)
128 +
129 + print("no lora")
130 + output = do_sample(llm,
131 + tinyllama_lora_files,
132 + lora_id=0,
133 + max_tokens=max_tokens)
134 + expect_match(output, expected_no_lora_output)
135 +
136 + print("lora 2")
137 + output = do_sample(llm,
138 + tinyllama_lora_files,
139 + lora_id=2,
140 + max_tokens=max_tokens)
141 + expect_match(output, expected_lora_output)
142 +
143 + print("removing lora")
144 +
145 + del llm
146 + cleanup()
147 +
148 +
149 +@pytest.mark.parametrize("model", MODELS)
150 +@pytest.mark.skip("Requires multiple GPUs")
151 +def test_quant_model_tp_equality(tinyllama_lora_files, model):
152 + # Cannot use as it will initialize torch.cuda too early...
153 + # if torch.cuda.device_count() < 2:
154 + # pytest.skip(f"Not enough GPUs for tensor parallelism {2}")
155 +
156 + llm_tp1 = vllm.LLM(model=model.model_path,
157 + enable_lora=True,
158 + max_num_seqs=16,
159 + max_loras=4,
160 + tensor_parallel_size=1,
161 + quantization=model.quantization,
162 + trust_remote_code=True)
163 + output_tp1 = do_sample(llm_tp1, tinyllama_lora_files, lora_id=1)
164 +
165 + del llm_tp1
166 + cleanup()
167 +
168 + llm_tp2 = vllm.LLM(model=model.model_path,
169 + enable_lora=True,
170 + max_num_seqs=16,
171 + max_loras=4,
172 + tensor_parallel_size=2,
173 + quantization=model.quantization)
174 + output_tp2 = do_sample(llm_tp2, tinyllama_lora_files, lora_id=1)
175 +
176 + del llm_tp2
177 + cleanup()
178 +
179 + assert output_tp1 == output_tp2
vllm/config.py
+6 -3
@@ -822,9 +822,12 @@ class LoRAConfig:
822 self.lora_dtype = model_config.dtype
823 elif isinstance(self.lora_dtype, str):
824 self.lora_dtype = getattr(torch, self.lora_dtype)
825 - if model_config.quantization is not None:
826 - raise ValueError(
827 - "LoRA is not supported with quantized models yet.")
825 + if model_config.quantization and model_config.quantization not in [
826 + "awq", "gptq"
827 + ]:
828 + # TODO support marlin and squeezellm
829 + logger.warning(f"{model_config.quantization} quantization is not "
830 + "tested with LoRA yet.")
831
832 def verify_with_scheduler_config(self, scheduler_config: SchedulerConfig):
833 if scheduler_config.max_num_batched_tokens > 65528:
vllm/lora/layers.py
+44 -23
@@ -29,6 +29,19 @@ if TYPE_CHECKING:
29 pass
30
31
32 +def _get_lora_device(base_layer: nn.Module) -> torch.device:
33 + # code borrowed from https://github.com/fmmoret/vllm/blob/fm-support-lora-on-quantized-models/vllm/lora/layers.py#L34
34 + """Returns the device for where to place the LoRA tensors."""
35 + if hasattr(base_layer, "weight"):
36 + return base_layer.weight.device
37 + if hasattr(base_layer, "linear_weights") and isinstance(
38 + base_layer.linear_weights, dict):
39 + values = list(base_layer.linear_weights.values())
40 + if len(values) and isinstance(values[0], torch.Tensor):
41 + return values[0].device
42 + raise ValueError(f"Unsupported base layer: {base_layer}")
43 +
44 +
45 def _apply_lora(
46 x: torch.Tensor,
47 lora_a_stacked: torch.Tensor,
@@ -302,6 +315,9 @@ class ColumnParallelLinearWithLoRA(BaseLayerWithLoRA):
315 super().__init__()
316 self.base_layer = base_layer
317 self.tp_size = get_tensor_model_parallel_world_size()
318 + self.input_size = self.base_layer.input_size
319 + self.output_size = self.base_layer.output_size_per_partition
320 + self.device = _get_lora_device(self.base_layer)
321
322 def create_lora_weights(
323 self,
@@ -312,17 +328,17 @@ class ColumnParallelLinearWithLoRA(BaseLayerWithLoRA):
328 max_loras,
329 1,
330 lora_config.max_lora_rank,
315 - self.base_layer.weight.shape[1],
331 + self.input_size,
332 dtype=lora_config.lora_dtype,
317 - device=self.base_layer.weight.device,
333 + device=self.device,
334 )
335 self.lora_b_stacked = torch.zeros(
336 max_loras,
337 1,
322 - self.base_layer.weight.shape[0],
338 + self.output_size,
339 lora_config.max_lora_rank,
340 dtype=lora_config.lora_dtype,
325 - device=self.base_layer.weight.device,
341 + device=self.device,
342 )
343
344 self.indices: Optional[torch.Tensor] = None
@@ -442,18 +458,18 @@ class MergedColumnParallelLinearWithLoRA(ColumnParallelLinearWithLoRA):
458 max_loras,
459 1,
460 lora_config.max_lora_rank,
445 - self.base_layer.weight.shape[1],
461 + self.input_size,
462 dtype=lora_config.lora_dtype,
447 - device=self.base_layer.weight.device,
463 + device=self.device,
464 ) for _ in range(n_slices))
465 self.lora_b_stacked = tuple(
466 torch.zeros(
467 max_loras,
468 1,
453 - self.base_layer.weight.shape[0] // 2,
469 + self.output_size // 2,
470 lora_config.max_lora_rank,
471 dtype=lora_config.lora_dtype,
456 - device=self.base_layer.weight.device,
472 + device=self.device,
473 ) for _ in range(n_slices))
474
475 self.indices: Optional[torch.Tensor] = None
@@ -619,25 +635,25 @@ class MergedQKVParallelLinearWithLora(ColumnParallelLinearWithLoRA):
635 max_loras,
636 1,
637 lora_config.max_lora_rank,
622 - self.base_layer.weight.shape[1],
638 + self.input_size,
639 dtype=lora_config.lora_dtype,
624 - device=self.base_layer.weight.device,
640 + device=self.device,
641 ),
642 torch.zeros(
643 max_loras,
644 1,
645 lora_config.max_lora_rank,
630 - self.base_layer.weight.shape[1],
646 + self.input_size,
647 dtype=lora_config.lora_dtype,
632 - device=self.base_layer.weight.device,
648 + device=self.device,
649 ),
650 torch.zeros(
651 max_loras,
652 1,
653 lora_config.max_lora_rank,
638 - self.base_layer.weight.shape[1],
654 + self.input_size,
655 dtype=lora_config.lora_dtype,
640 - device=self.base_layer.weight.device,
656 + device=self.device,
657 ),
658 )
659 self.lora_b_stacked = (
@@ -647,7 +663,7 @@ class MergedQKVParallelLinearWithLora(ColumnParallelLinearWithLoRA):
663 self.q_proj_shard_size,
664 lora_config.max_lora_rank,
665 dtype=lora_config.lora_dtype,
650 - device=self.base_layer.weight.device,
666 + device=self.device,
667 ),
668 torch.zeros(
669 max_loras,
@@ -655,7 +671,7 @@ class MergedQKVParallelLinearWithLora(ColumnParallelLinearWithLoRA):
671 self.kv_proj_shard_size,
672 lora_config.max_lora_rank,
673 dtype=lora_config.lora_dtype,
658 - device=self.base_layer.weight.device,
674 + device=self.device,
675 ),
676 torch.zeros(
677 max_loras,
@@ -663,7 +679,7 @@ class MergedQKVParallelLinearWithLora(ColumnParallelLinearWithLoRA):
679 self.kv_proj_shard_size,
680 lora_config.max_lora_rank,
681 dtype=lora_config.lora_dtype,
666 - device=self.base_layer.weight.device,
682 + device=self.device,
683 ),
684 )
685
@@ -766,6 +782,9 @@ class RowParallelLinearWithLoRA(BaseLayerWithLoRA):
782 def __init__(self, base_layer: RowParallelLinear) -> None:
783 super().__init__()
784 self.base_layer = base_layer
785 + self.input_size = self.base_layer.input_size_per_partition
786 + self.output_size = self.base_layer.output_size
787 + self.device = _get_lora_device(self.base_layer)
788
789 def create_lora_weights(
790 self,
@@ -777,20 +796,20 @@ class RowParallelLinearWithLoRA(BaseLayerWithLoRA):
796 max_loras,
797 1,
798 lora_config.max_lora_rank,
780 - self.base_layer.weight.shape[1],
799 + self.input_size,
800 ),
801 dtype=lora_config.lora_dtype,
783 - device=self.base_layer.weight.device,
802 + device=self.device,
803 )
804 self.lora_b_stacked = torch.zeros(
805 (
806 max_loras,
807 1,
789 - self.base_layer.weight.shape[0],
808 + self.output_size,
809 lora_config.max_lora_rank,
810 ),
811 dtype=lora_config.lora_dtype,
793 - device=self.base_layer.weight.device,
812 + device=self.device,
813 )
814 self.indices: Optional[torch.Tensor] = None
815 self.indices_len: Optional[List[int]] = None
@@ -809,7 +828,7 @@ class RowParallelLinearWithLoRA(BaseLayerWithLoRA):
828 self.reset_lora(index)
829 if self.base_layer.tp_size > 1:
830 tensor_model_parallel_rank = get_tensor_model_parallel_rank()
812 - shard_size = self.base_layer.weight.shape[1]
831 + shard_size = self.input_size
832 start_idx = tensor_model_parallel_rank * shard_size
833 end_idx = (tensor_model_parallel_rank + 1) * shard_size
834 lora_a = lora_a[start_idx:end_idx, :]
@@ -884,7 +903,9 @@ class RowParallelLinearWithLoRA(BaseLayerWithLoRA):
903
904 @property
905 def weight(self):
887 - return self.base_layer.weight
906 +
907 + return self.base_layer.weight if hasattr(
908 + self.base_layer, "weight") else self.base_layer.qweight
909
910 @classmethod
911 def can_replace_layer(cls, source_layer: nn.Module,