@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 07934cc237
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 02:00 UTC

[Misc][LoRA] Improve the readability of LoRA error messages (#12102)

Signed-off-by: Jee Jee Li <pandaleefree@gmail.com>

Jee Jee Li committed Jan 17, 2025 at 19:32 UTC 07934cc237d16427d705e5abc3c83e4eb0f9b7f4
10 files changed +245 -116
tests/entrypoints/openai/test_lora_adapters.py
+50 -19
@@ -17,6 +17,33 @@ MODEL_NAME = "HuggingFaceH4/zephyr-7b-beta"
17 # generation quality here
18 LORA_NAME = "typeof/zephyr-7b-beta-lora"
19
20 +BADREQUEST_CASES = [
21 + (
22 + "test_rank",
23 + {
24 + "r": 1024
25 + },
26 + "is greater than max_lora_rank",
27 + ),
28 + (
29 + "test_bias",
30 + {
31 + "bias": "all"
32 + },
33 + "Adapter bias cannot be used without bias_enabled",
34 + ),
35 + ("test_dora", {
36 + "use_dora": True
37 + }, "does not yet support DoRA"),
38 + (
39 + "test_modules_to_save",
40 + {
41 + "modules_to_save": ["lm_head"]
42 + },
43 + "only supports modules_to_save being None",
44 + ),
45 +]
46 +
47
48 @pytest.fixture(scope="module")
49 def zephyr_lora_files():
@@ -138,32 +165,36 @@ async def test_dynamic_lora_invalid_files(client: openai.AsyncOpenAI,
165
166
167 @pytest.mark.asyncio
141 -async def test_dynamic_lora_invalid_lora_rank(client: openai.AsyncOpenAI,
142 - tmp_path, zephyr_lora_files):
143 - invalid_rank = tmp_path / "invalid_rank"
144 -
145 - # Copy adapter from zephyr_lora_files to invalid_rank
146 - shutil.copytree(zephyr_lora_files, invalid_rank)
147 -
148 - with open(invalid_rank / "adapter_config.json") as f:
168 +@pytest.mark.parametrize("test_name,config_change,expected_error",
169 + BADREQUEST_CASES)
170 +async def test_dynamic_lora_badrequests(client: openai.AsyncOpenAI, tmp_path,
171 + zephyr_lora_files, test_name: str,
172 + config_change: dict,
173 + expected_error: str):
174 + # Create test directory
175 + test_dir = tmp_path / test_name
176 +
177 + # Copy adapter files
178 + shutil.copytree(zephyr_lora_files, test_dir)
179 +
180 + # Load and modify configuration
181 + config_path = test_dir / "adapter_config.json"
182 + with open(config_path) as f:
183 adapter_config = json.load(f)
184 + # Apply configuration changes
185 + adapter_config.update(config_change)
186
151 - print(adapter_config)
152 -
153 - # assert False
154 -
155 - # Change rank to invalid value
156 - adapter_config["r"] = 1024
157 - with open(invalid_rank / "adapter_config.json", "w") as f:
187 + # Save modified configuration
188 + with open(config_path, "w") as f:
189 json.dump(adapter_config, f)
190
160 - with pytest.raises(openai.BadRequestError,
161 - match="is greater than max_lora_rank"):
191 + # Test loading the adapter
192 + with pytest.raises(openai.BadRequestError, match=expected_error):
193 await client.post("load_lora_adapter",
194 cast_to=str,
195 body={
165 - "lora_name": "invalid-json",
166 - "lora_path": str(invalid_rank)
196 + "lora_name": test_name,
197 + "lora_path": str(test_dir)
198 })
199
200
tests/lora/test_lora_checkpoints.py
+16
@@ -3,6 +3,7 @@ from typing import List
3 import pytest
4
5 from vllm.lora.models import LoRAModel
6 +from vllm.lora.peft_helper import PEFTHelper
7 from vllm.model_executor.models.baichuan import BaiChuanBaseForCausalLM
8 from vllm.model_executor.models.utils import WeightsMapper
9
@@ -30,11 +31,14 @@ def test_load_checkpoints(
31 else:
32 expected_lora_modules.append(module)
33 if lora_name == "baichuan7B":
34 + peft_helper = PEFTHelper.from_local_dir(baichuan_lora_files,
35 + max_position_embeddings=4096)
36 # For the baichuan7B model, load it's LoRA,
37 # and the test should pass.
38 LoRAModel.from_local_checkpoint(
39 baichuan_lora_files,
40 expected_lora_modules,
41 + peft_helper=peft_helper,
42 lora_model_id=1,
43 device="cpu",
44 embedding_modules=embedding_modules,
@@ -43,9 +47,12 @@ def test_load_checkpoints(
47 # Test that the target_modules contain prefix
48 # such as "model.layers.0.self_atten.W_pack", and
49 # the test should pass.
50 + peft_helper = PEFTHelper.from_local_dir(baichuan_zero_lora_files,
51 + max_position_embeddings=4096)
52 LoRAModel.from_local_checkpoint(
53 baichuan_zero_lora_files,
54 expected_lora_modules,
55 + peft_helper=peft_helper,
56 lora_model_id=1,
57 device="cpu",
58 embedding_modules=embedding_modules,
@@ -53,9 +60,12 @@ def test_load_checkpoints(
60 elif lora_name == "baichuan7B-zero-regex":
61 # Test that the `target_modules` in the form of regular expressions,
62 # such as `model\\..*(W_pack|o_proj)`, and the test should pass.
63 + peft_helper = PEFTHelper.from_local_dir(baichuan_regex_lora_files,
64 + max_position_embeddings=4096)
65 LoRAModel.from_local_checkpoint(
66 baichuan_regex_lora_files,
67 expected_lora_modules,
68 + peft_helper=peft_helper,
69 lora_model_id=1,
70 device="cpu",
71 embedding_modules=embedding_modules,
@@ -64,10 +74,13 @@ def test_load_checkpoints(
74 # For the baichuan7B model, load chatglm3-6b's LoRA,
75 # and the test should raise the following error.
76 expected_error = "Please verify that the loaded LoRA module is correct" # noqa: E501
77 + peft_helper = PEFTHelper.from_local_dir(chatglm3_lora_files,
78 + max_position_embeddings=4096)
79 with pytest.raises(ValueError, match=expected_error):
80 LoRAModel.from_local_checkpoint(
81 chatglm3_lora_files,
82 expected_lora_modules,
83 + peft_helper=peft_helper,
84 lora_model_id=1,
85 device="cpu",
86 embedding_modules=embedding_modules,
@@ -94,9 +107,12 @@ def test_lora_weights_mapping(baichuan_lora_files):
107 ".layers.": ".baichuan_layers.",
108 },
109 )
110 + peft_helper = PEFTHelper.from_local_dir(baichuan_lora_files,
111 + max_position_embeddings=4096)
112 lora_model = LoRAModel.from_local_checkpoint(
113 baichuan_lora_files,
114 expected_lora_modules,
115 + peft_helper=peft_helper,
116 lora_model_id=1,
117 device="cpu",
118 embedding_modules=embedding_modules,
tests/lora/test_lora_huggingface.py
+3
@@ -3,6 +3,7 @@ from typing import List
3 import pytest
4
5 from vllm.lora.models import LoRAModel
6 +from vllm.lora.peft_helper import PEFTHelper
7 from vllm.lora.utils import get_adapter_absolute_path
8 from vllm.model_executor.models.llama import LlamaForCausalLM
9
@@ -27,9 +28,11 @@ def test_load_checkpoints_from_huggingface(lora_fixture_name, request):
28 lora_path = get_adapter_absolute_path(lora_name)
29
30 # lora loading should work for either absolute path and hugggingface id.
31 + peft_helper = PEFTHelper.from_local_dir(lora_path, 4096)
32 lora_model = LoRAModel.from_local_checkpoint(
33 lora_path,
34 expected_lora_modules,
35 + peft_helper=peft_helper,
36 lora_model_id=1,
37 device="cpu",
38 embedding_modules=embedding_modules,
tests/lora/test_lora_manager.py
+2 -57
@@ -1,5 +1,3 @@
1 -import json
2 -import math
1 import os
2 from typing import Dict, List
3
@@ -34,56 +32,6 @@ DEVICES = ([
32 ] if current_platform.is_cuda_alike() else ["cpu"])
33
34
37 -def test_peft_helper(sql_lora_files):
38 - lora_config_path = os.path.join(sql_lora_files, "adapter_config.json")
39 - with open(lora_config_path) as f:
40 - config = json.load(f)
41 - peft_helper = PEFTHelper.from_dict(config)
42 - assert peft_helper.r == 8
43 - assert peft_helper.lora_alpha == 16
44 - assert peft_helper.target_modules == [
45 - "q_proj",
46 - "v_proj",
47 - "k_proj",
48 - "o_proj",
49 - "gate_proj",
50 - "up_proj",
51 - "down_proj",
52 - "embed_tokens",
53 - "lm_head",
54 - ]
55 - scaling = peft_helper.lora_alpha / peft_helper.r
56 - assert abs(peft_helper.vllm_lora_scaling_factor - scaling) < 1e-3
57 -
58 - # test RSLoRA
59 - config = dict(r=8,
60 - lora_alpha=16,
61 - target_modules=["gate_proj"],
62 - use_rslora=True)
63 - peft_helper = PEFTHelper.from_dict(config)
64 -
65 - scaling = peft_helper.lora_alpha / math.sqrt(peft_helper.r)
66 - assert abs(peft_helper.vllm_lora_scaling_factor - scaling) < 1e-3
67 -
68 - expected_error = "vLLM only supports modules_to_save being None."
69 - with pytest.raises(ValueError, match=expected_error):
70 - config = dict(
71 - r=8,
72 - lora_alpha=16,
73 - target_modules=["gate_proj"],
74 - modules_to_save=["lm_head"],
75 - )
76 - PEFTHelper.from_dict(config)
77 -
78 - expected_error = "vLLM does not yet support DoRA."
79 - with pytest.raises(ValueError, match=expected_error):
80 - config = dict(r=8,
81 - lora_alpha=16,
82 - target_modules=["gate_proj"],
83 - use_dora=True)
84 - PEFTHelper.from_dict(config)
85 -
86 -
35 @pytest.mark.parametrize("device", DEVICES)
36 def test_from_lora_tensors(sql_lora_files, device):
37 tensors = load_file(
@@ -91,11 +39,8 @@ def test_from_lora_tensors(sql_lora_files, device):
39 new_embeddings = load_file(
40 os.path.join(sql_lora_files, "new_embeddings.safetensors"))
41
94 - lora_config_path = os.path.join(sql_lora_files, "adapter_config.json")
95 - with open(lora_config_path) as f:
96 - config = json.load(f)
97 -
98 - peft_helper = PEFTHelper.from_dict(config)
42 + peft_helper = PEFTHelper.from_local_dir(sql_lora_files,
43 + max_position_embeddings=4096)
44 lora_model = LoRAModel.from_lora_tensors(
45 1,
46 tensors,
tests/lora/test_peft_helper.py new
+109
@@ -0,0 +1,109 @@
1 +import json
2 +import math
3 +import shutil
4 +
5 +import pytest
6 +
7 +from vllm.config import LoRAConfig
8 +from vllm.lora.peft_helper import PEFTHelper
9 +
10 +ERROR_CASES = [
11 + (
12 + "test_rank",
13 + {
14 + "r": 1024
15 + },
16 + "is greater than max_lora_rank",
17 + ),
18 + (
19 + "test_bias",
20 + {
21 + "bias": "all"
22 + },
23 + "Adapter bias cannot be used without bias_enabled",
24 + ),
25 + ("test_dora", {
26 + "use_dora": True
27 + }, "does not yet support DoRA"),
28 + (
29 + "test_modules_to_save",
30 + {
31 + "modules_to_save": ["lm_head"]
32 + },
33 + "only supports modules_to_save being None",
34 + ),
35 +]
36 +
37 +
38 +def test_peft_helper_pass(long_context_lora_files_16k_1, tmp_path):
39 + peft_helper = PEFTHelper.from_local_dir(long_context_lora_files_16k_1,
40 + max_position_embeddings=4096)
41 + lora_config = LoRAConfig(max_lora_rank=16, max_cpu_loras=3, max_loras=2)
42 + peft_helper.validate_legal(lora_config)
43 + assert peft_helper.r == 8
44 + assert peft_helper.lora_alpha == 16
45 + assert peft_helper.target_modules == [
46 + "q_proj",
47 + "v_proj",
48 + "k_proj",
49 + "o_proj",
50 + "gate_proj",
51 + "up_proj",
52 + "down_proj",
53 + "embed_tokens",
54 + "lm_head",
55 + ]
56 + assert peft_helper.context_length == 16384
57 + assert peft_helper.vllm_max_position_embeddings == 4096
58 + assert peft_helper.vllm_long_context_scaling_factor == float(
59 + math.ceil(peft_helper.context_length /
60 + peft_helper.vllm_max_position_embeddings))
61 + # test RSLoRA
62 + rslora_config = dict(use_rslora=True)
63 + test_dir = tmp_path / "test_rslora"
64 + shutil.copytree(long_context_lora_files_16k_1, test_dir)
65 +
66 + # Load and modify configuration
67 + config_path = test_dir / "adapter_config.json"
68 + with open(config_path) as f:
69 + adapter_config = json.load(f)
70 + # Apply configuration changes
71 + adapter_config.update(rslora_config)
72 +
73 + # Save modified configuration
74 + with open(config_path, "w") as f:
75 + json.dump(adapter_config, f)
76 +
77 + peft_helper = PEFTHelper.from_local_dir(test_dir,
78 + max_position_embeddings=4096)
79 + peft_helper.validate_legal(lora_config)
80 + scaling = peft_helper.lora_alpha / math.sqrt(peft_helper.r)
81 + assert abs(peft_helper.vllm_lora_scaling_factor - scaling) < 1e-3
82 +
83 +
84 +@pytest.mark.parametrize("test_name,config_change,expected_error", ERROR_CASES)
85 +def test_peft_helper_error(
86 + sql_lora_files,
87 + tmp_path,
88 + test_name: str,
89 + config_change: dict,
90 + expected_error: str,
91 +):
92 + test_dir = tmp_path / test_name
93 + shutil.copytree(sql_lora_files, test_dir)
94 +
95 + # Load and modify configuration
96 + config_path = test_dir / "adapter_config.json"
97 + with open(config_path) as f:
98 + adapter_config = json.load(f)
99 + # Apply configuration changes
100 + adapter_config.update(config_change)
101 +
102 + # Save modified configuration
103 + with open(config_path, "w") as f:
104 + json.dump(adapter_config, f)
105 + lora_config = LoRAConfig(max_lora_rank=16, max_cpu_loras=3, max_loras=2)
106 + # Test loading the adapter
107 + with pytest.raises(ValueError, match=expected_error):
108 + PEFTHelper.from_local_dir(
109 + test_dir, max_position_embeddings=4096).validate_legal(lora_config)
vllm/engine/multiprocessing/engine.py
+1
@@ -296,6 +296,7 @@ class MQLLMEngine:
296 is_engine_errored=False,
297 exception=e)
298 self._send_outputs(rpc_err)
299 + return
300 # Otherwise, send back the successful load message
301 self._send_outputs(
302 RPCAdapterLoadedResponse(request_id=request.request_id))
vllm/entrypoints/openai/serving_models.py
+8 -16
@@ -157,24 +157,16 @@ class OpenAIServingModels:
157 # This will also pre-load it for incoming requests
158 try:
159 await self.engine_client.add_lora(lora_request)
160 - except ValueError as e:
161 - # Adapter not found or lora configuration errors
162 - if "No adapter found" in str(e):
163 - return create_error_response(message=str(e),
164 - err_type="NotFoundError",
165 - status_code=HTTPStatus.NOT_FOUND)
166 - else:
167 - return create_error_response(
168 - message=str(e),
169 - err_type="BadRequestError",
170 - status_code=HTTPStatus.BAD_REQUEST)
160 except BaseException as e:
172 - # Some other unexpected problem loading the adapter, e.g. malformed
173 - # input files.
174 - # More detailed error messages for the user would be nicer here
161 + error_type = "BadRequestError"
162 + status_code = HTTPStatus.BAD_REQUEST
163 + if isinstance(e, ValueError) and "No adapter found" in str(e):
164 + error_type = "NotFoundError"
165 + status_code = HTTPStatus.NOT_FOUND
166 +
167 return create_error_response(message=str(e),
176 - err_type="BadRequestError",
177 - status_code=HTTPStatus.BAD_REQUEST)
168 + err_type=error_type,
169 + status_code=status_code)
170
171 self.lora_requests.append(lora_request)
172 logger.info("Loaded new LoRA adapter: name '%s', path '%s'", lora_name,
vllm/lora/models.py
+2 -10
@@ -1,5 +1,4 @@
1 import copy
2 -import json
2 import math
3 import os
4 import re
@@ -180,8 +179,8 @@ class LoRAModel(AdapterModel):
179 cls,
180 lora_dir: str,
181 expected_lora_modules: List[str],
182 + peft_helper: PEFTHelper,
183 *,
184 - max_position_embeddings: Optional[int] = None,
184 lora_model_id: Optional[int] = None,
185 device: str = "cuda",
186 dtype: Optional[torch.dtype] = None,
@@ -196,9 +195,7 @@ class LoRAModel(AdapterModel):
195 lora_dir: The local path that has lora data.
196 expected_lora_modules: Name of modules that are expected to be
197 replaced by lora.
199 - max_position_embeddings: Max position embedding length. Used to
200 - scaling the largest context length. If None, the lora model's
201 - context length is not scaled.
198 + peft_helper: Loaded lora configuration information.
199 lora_model_id: Lora model id. If not given, automatically set by
200 a global counter.
201 device: Device where the lora model is loaded.
@@ -207,18 +204,13 @@ class LoRAModel(AdapterModel):
204 Returns:
205 Loaded LoRA Model.
206 """
210 - lora_config_path = os.path.join(lora_dir, "adapter_config.json")
207 lora_tensor_path = os.path.join(lora_dir, "adapter_model.safetensors")
208 lora_bin_file_path = os.path.join(lora_dir, "adapter_model.bin")
209 new_embeddings_tensor_path = os.path.join(
210 lora_dir, "new_embeddings.safetensors")
211 new_embeddings_bin_file_path = os.path.join(lora_dir,
212 "new_embeddings.bin")
217 - with open(lora_config_path) as f:
218 - config = json.load(f)
213
220 - config["vllm_max_position_embeddings"] = max_position_embeddings
221 - peft_helper = PEFTHelper.from_dict(config)
214 unexpected_modules: List[Union[list[str], str]]
215 if os.path.isfile(lora_tensor_path):
216 tensors: Dict[str, torch.Tensor] = {}
vllm/lora/peft_helper.py
+41 -8
@@ -1,9 +1,12 @@
1 # Adapted from: https://github.com/huggingface/peft/blob/main/src/peft/tuners/lora/config.py
2
3 +import json
4 import math
5 +import os
6 from dataclasses import MISSING, dataclass, field, fields
5 -from typing import Literal, Optional, Union
7 +from typing import List, Literal, Optional, Union
8
9 +from vllm.config import LoRAConfig
10 from vllm.logger import init_logger
11
12 logger = init_logger(__name__)
@@ -11,6 +14,12 @@ logger = init_logger(__name__)
14
15 @dataclass
16 class PEFTHelper:
17 + """
18 + A helper class for PEFT configurations, specifically designed for LoRA.
19 + This class handles configuration validation, compatibility checks for
20 + various LoRA implementations.
21 + """
22 +
23 # Required fields
24 r: int
25 lora_alpha: int
@@ -29,20 +38,18 @@ class PEFTHelper:
38 vllm_max_position_embeddings: Optional[int] = field(default=False)
39 vllm_long_context_scaling_factor: Optional[float] = field(default=None)
40
32 - def _validate_features(self):
41 + def _validate_features(self) -> List[str]:
42 + """
43 + Check if there are any unsupported Lora features.
44 + """
45 error_msg = []
34 -
46 if self.modules_to_save:
47 error_msg.append("vLLM only supports modules_to_save being None.")
37 -
48 if self.use_dora:
49 error_msg.append("vLLM does not yet support DoRA.")
40 -
41 - if error_msg:
42 - raise ValueError(f"{', '.join(error_msg)}")
50 + return error_msg
51
52 def __post_init__(self):
45 - self._validate_features()
53 if self.use_rslora:
54 logger.info_once("Loading LoRA weights trained with rsLoRA.")
55 self.vllm_lora_scaling_factor = self.lora_alpha / math.sqrt(self.r)
@@ -78,3 +85,29 @@ class PEFTHelper:
85 for k, v in config_dict.items() if k in class_fields
86 }
87 return cls(**filtered_dict)
88 +
89 + @classmethod
90 + def from_local_dir(cls, lora_path: str,
91 + max_position_embeddings: Optional[int]) -> "PEFTHelper":
92 + lora_config_path = os.path.join(lora_path, "adapter_config.json")
93 +
94 + with open(lora_config_path) as f:
95 + config = json.load(f)
96 + config["vllm_max_position_embeddings"] = max_position_embeddings
97 + return cls.from_dict(config)
98 +
99 + def validate_legal(self, lora_config: LoRAConfig) -> None:
100 + """
101 + Validates the LoRA configuration settings against application
102 + constraints and requirements.
103 + """
104 + error_msg = self._validate_features()
105 + if self.r > lora_config.max_lora_rank:
106 + error_msg.append(
107 + f"LoRA rank {self.r} is greater than max_lora_rank"
108 + f" {lora_config.max_lora_rank}.")
109 + if self.bias != "none" and not lora_config.bias_enabled:
110 + error_msg.append(
111 + "Adapter bias cannot be used without bias_enabled.")
112 + if error_msg:
113 + raise ValueError(f"{' '.join(error_msg)}")
vllm/lora/worker_manager.py
+13 -6
@@ -12,6 +12,7 @@ from vllm.config import LoRAConfig
12 from vllm.logger import init_logger
13 from vllm.lora.models import (LoRAModel, LoRAModelManager,
14 LRUCacheLoRAModelManager, create_lora_manager)
15 +from vllm.lora.peft_helper import PEFTHelper
16 from vllm.lora.request import LoRARequest
17 from vllm.lora.utils import get_adapter_absolute_path
18
@@ -95,6 +96,13 @@ class WorkerLoRAManager(AbstractWorkerManager):
96 expected_lora_modules = list(set(expected_lora_modules))
97 lora_path = get_adapter_absolute_path(lora_request.lora_path)
98
99 + peft_helper = PEFTHelper.from_local_dir(
100 + lora_path, self.max_position_embeddings)
101 +
102 + # Validates the LoRA configuration against requirements before
103 + # loading weights, throwing an exception if validation fails.
104 + peft_helper.validate_legal(self.lora_config)
105 +
106 # For some models like Qwen2VL, we need to use hf_to_vllm_mapper
107 # to ensure correct loading of lora weights.
108 hf_to_vllm_mapper = None
@@ -105,7 +113,7 @@ class WorkerLoRAManager(AbstractWorkerManager):
113 lora = self._lora_model_cls.from_local_checkpoint(
114 lora_path,
115 expected_lora_modules,
108 - max_position_embeddings=self.max_position_embeddings,
116 + peft_helper=peft_helper,
117 lora_model_id=lora_request.lora_int_id,
118 device="cpu",
119 dtype=self.lora_config.lora_dtype,
@@ -120,15 +128,14 @@ class WorkerLoRAManager(AbstractWorkerManager):
128 # - No adapter found to download from huggingface (or in
129 # offline mode)
130 # - No local adapter files found at `lora_request.lora_path`
131 + # For NotFoundError
132 raise ValueError(
133 f"Loading lora {lora_request.lora_name} failed: No adapter "
134 f"found for {lora_path}") from e
135 except Exception as e:
127 - raise RuntimeError(f"Loading lora {lora_path} failed") from e
128 - if lora.rank > self.lora_config.max_lora_rank:
129 - raise ValueError(
130 - f"LoRA rank {lora.rank} is greater than max_lora_rank "
131 - f"{self.lora_config.max_lora_rank}.")
136 + # For BadRequestError
137 + raise e
138 +
139 if lora.extra_vocab_size > self.lora_config.lora_extra_vocab_size:
140 raise ValueError(f"LoRA added vocab size {lora.extra_vocab_size} "
141 f"is greater than lora_extra_vocab_size "