A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 02:00 UTC
[Misc][LoRA] Improve the readability of LoRA error messages (#12102)
Signed-off-by: Jee Jee Li <pandaleefree@gmail.com>
Jee Jee Li committed
Jan 17, 2025 at 19:32 UTC
07934cc237d16427d705e5abc3c83e4eb0f9b7f4
10 files changed
+245
-116
tests/entrypoints/openai/test_lora_adapters.py
+50
-19
@@ -17,6 +17,33 @@ MODEL_NAME = "HuggingFaceH4/zephyr-7b-beta"
17
# generation quality here
18
LORA_NAME = "typeof/zephyr-7b-beta-lora"
19
20
+BADREQUEST_CASES = [
21
+ (
22
+ "test_rank",
23
+ {
24
+ "r": 1024
25
+ },
26
+ "is greater than max_lora_rank",
27
+ ),
28
+ (
29
+ "test_bias",
30
+ {
31
+ "bias": "all"
32
+ },
33
+ "Adapter bias cannot be used without bias_enabled",
34
+ ),
35
+ ("test_dora", {
36
+ "use_dora": True
37
+ }, "does not yet support DoRA"),
38
+ (
39
+ "test_modules_to_save",
40
+ {
41
+ "modules_to_save": ["lm_head"]
42
+ },
43
+ "only supports modules_to_save being None",
44
+ ),
45
+]
46
+
47
48
@pytest.fixture(scope="module")
49
def zephyr_lora_files():
@@ -138,32 +165,36 @@ async def test_dynamic_lora_invalid_files(client: openai.AsyncOpenAI,
165
166
167
@pytest.mark.asyncio
141
-async def test_dynamic_lora_invalid_lora_rank(client: openai.AsyncOpenAI,
142
- tmp_path, zephyr_lora_files):
143
- invalid_rank = tmp_path / "invalid_rank"
144
-
145
- # Copy adapter from zephyr_lora_files to invalid_rank
146
- shutil.copytree(zephyr_lora_files, invalid_rank)
147
-
148
- with open(invalid_rank / "adapter_config.json") as f:
168
+@pytest.mark.parametrize("test_name,config_change,expected_error",
169
+ BADREQUEST_CASES)
170
+async def test_dynamic_lora_badrequests(client: openai.AsyncOpenAI, tmp_path,
171
+ zephyr_lora_files, test_name: str,
172
+ config_change: dict,
173
+ expected_error: str):
174
+ # Create test directory
175
+ test_dir = tmp_path / test_name
176
+
177
+ # Copy adapter files
178
+ shutil.copytree(zephyr_lora_files, test_dir)
179
+
180
+ # Load and modify configuration
181
+ config_path = test_dir / "adapter_config.json"
182
+ with open(config_path) as f:
183
adapter_config = json.load(f)
184
+ # Apply configuration changes
185
+ adapter_config.update(config_change)
186
151
- print(adapter_config)
152
-
153
- # assert False
154
-
155
- # Change rank to invalid value
156
- adapter_config["r"] = 1024
157
- with open(invalid_rank / "adapter_config.json", "w") as f:
187
+ # Save modified configuration
188
+ with open(config_path, "w") as f:
189
json.dump(adapter_config, f)
190
160
- with pytest.raises(openai.BadRequestError,
161
- match="is greater than max_lora_rank"):
191
+ # Test loading the adapter
192
+ with pytest.raises(openai.BadRequestError, match=expected_error):
193
await client.post("load_lora_adapter",
194
cast_to=str,
195
body={
165
- "lora_name": "invalid-json",
166
- "lora_path": str(invalid_rank)
196
+ "lora_name": test_name,
197
+ "lora_path": str(test_dir)
198
})
199
200
tests/lora/test_lora_checkpoints.py
+16
@@ -3,6 +3,7 @@ from typing import List
3
import pytest
4
5
from vllm.lora.models import LoRAModel
6
+from vllm.lora.peft_helper import PEFTHelper
7
from vllm.model_executor.models.baichuan import BaiChuanBaseForCausalLM
8
from vllm.model_executor.models.utils import WeightsMapper
9
@@ -30,11 +31,14 @@ def test_load_checkpoints(
31
else:
32
expected_lora_modules.append(module)
33
if lora_name == "baichuan7B":
34
+ peft_helper = PEFTHelper.from_local_dir(baichuan_lora_files,
35
+ max_position_embeddings=4096)
36
# For the baichuan7B model, load it's LoRA,
37
# and the test should pass.
38
LoRAModel.from_local_checkpoint(
39
baichuan_lora_files,
40
expected_lora_modules,
41
+ peft_helper=peft_helper,
42
lora_model_id=1,
43
device="cpu",
44
embedding_modules=embedding_modules,
@@ -43,9 +47,12 @@ def test_load_checkpoints(
47
# Test that the target_modules contain prefix
48
# such as "model.layers.0.self_atten.W_pack", and
49
# the test should pass.
50
+ peft_helper = PEFTHelper.from_local_dir(baichuan_zero_lora_files,
51
+ max_position_embeddings=4096)
52
LoRAModel.from_local_checkpoint(
53
baichuan_zero_lora_files,
54
expected_lora_modules,
55
+ peft_helper=peft_helper,
56
lora_model_id=1,
57
device="cpu",
58
embedding_modules=embedding_modules,
@@ -53,9 +60,12 @@ def test_load_checkpoints(
60
elif lora_name == "baichuan7B-zero-regex":
61
# Test that the `target_modules` in the form of regular expressions,
62
# such as `model\\..*(W_pack|o_proj)`, and the test should pass.
63
+ peft_helper = PEFTHelper.from_local_dir(baichuan_regex_lora_files,
64
+ max_position_embeddings=4096)
65
LoRAModel.from_local_checkpoint(
66
baichuan_regex_lora_files,
67
expected_lora_modules,
68
+ peft_helper=peft_helper,
69
lora_model_id=1,
70
device="cpu",
71
embedding_modules=embedding_modules,
@@ -64,10 +74,13 @@ def test_load_checkpoints(
74
# For the baichuan7B model, load chatglm3-6b's LoRA,
75
# and the test should raise the following error.
76
expected_error = "Please verify that the loaded LoRA module is correct" # noqa: E501
77
+ peft_helper = PEFTHelper.from_local_dir(chatglm3_lora_files,
78
+ max_position_embeddings=4096)
79
with pytest.raises(ValueError, match=expected_error):
80
LoRAModel.from_local_checkpoint(
81
chatglm3_lora_files,
82
expected_lora_modules,
83
+ peft_helper=peft_helper,
84
lora_model_id=1,
85
device="cpu",
86
embedding_modules=embedding_modules,
@@ -94,9 +107,12 @@ def test_lora_weights_mapping(baichuan_lora_files):
107
".layers.": ".baichuan_layers.",
108
},
109
)
110
+ peft_helper = PEFTHelper.from_local_dir(baichuan_lora_files,
111
+ max_position_embeddings=4096)
112
lora_model = LoRAModel.from_local_checkpoint(
113
baichuan_lora_files,
114
expected_lora_modules,
115
+ peft_helper=peft_helper,
116
lora_model_id=1,
117
device="cpu",
118
embedding_modules=embedding_modules,
tests/lora/test_lora_huggingface.py
+3
@@ -3,6 +3,7 @@ from typing import List
3
import pytest
4
5
from vllm.lora.models import LoRAModel
6
+from vllm.lora.peft_helper import PEFTHelper
7
from vllm.lora.utils import get_adapter_absolute_path
8
from vllm.model_executor.models.llama import LlamaForCausalLM
9
@@ -27,9 +28,11 @@ def test_load_checkpoints_from_huggingface(lora_fixture_name, request):
28
lora_path = get_adapter_absolute_path(lora_name)
29
30
# lora loading should work for either absolute path and hugggingface id.
31
+ peft_helper = PEFTHelper.from_local_dir(lora_path, 4096)
32
lora_model = LoRAModel.from_local_checkpoint(
33
lora_path,
34
expected_lora_modules,
35
+ peft_helper=peft_helper,
36
lora_model_id=1,
37
device="cpu",
38
embedding_modules=embedding_modules,
tests/lora/test_lora_manager.py
+2
-57
@@ -1,5 +1,3 @@
1
-import json
2
-import math
1
import os
2
from typing import Dict, List
3
@@ -34,56 +32,6 @@ DEVICES = ([
32
] if current_platform.is_cuda_alike() else ["cpu"])
33
34
37
-def test_peft_helper(sql_lora_files):
38
- lora_config_path = os.path.join(sql_lora_files, "adapter_config.json")
39
- with open(lora_config_path) as f:
40
- config = json.load(f)
41
- peft_helper = PEFTHelper.from_dict(config)
42
- assert peft_helper.r == 8
43
- assert peft_helper.lora_alpha == 16
44
- assert peft_helper.target_modules == [
45
- "q_proj",
46
- "v_proj",
47
- "k_proj",
48
- "o_proj",
49
- "gate_proj",
50
- "up_proj",
51
- "down_proj",
52
- "embed_tokens",
53
- "lm_head",
54
- ]
55
- scaling = peft_helper.lora_alpha / peft_helper.r
56
- assert abs(peft_helper.vllm_lora_scaling_factor - scaling) < 1e-3
57
-
58
- # test RSLoRA
59
- config = dict(r=8,
60
- lora_alpha=16,
61
- target_modules=["gate_proj"],
62
- use_rslora=True)
63
- peft_helper = PEFTHelper.from_dict(config)
64
-
65
- scaling = peft_helper.lora_alpha / math.sqrt(peft_helper.r)
66
- assert abs(peft_helper.vllm_lora_scaling_factor - scaling) < 1e-3
67
-
68
- expected_error = "vLLM only supports modules_to_save being None."
69
- with pytest.raises(ValueError, match=expected_error):
70
- config = dict(
71
- r=8,
72
- lora_alpha=16,
73
- target_modules=["gate_proj"],
74
- modules_to_save=["lm_head"],
75
- )
76
- PEFTHelper.from_dict(config)
77
-
78
- expected_error = "vLLM does not yet support DoRA."
79
- with pytest.raises(ValueError, match=expected_error):
80
- config = dict(r=8,
81
- lora_alpha=16,
82
- target_modules=["gate_proj"],
83
- use_dora=True)
84
- PEFTHelper.from_dict(config)
85
-
86
-
35
@pytest.mark.parametrize("device", DEVICES)
36
def test_from_lora_tensors(sql_lora_files, device):
37
tensors = load_file(
@@ -91,11 +39,8 @@ def test_from_lora_tensors(sql_lora_files, device):
39
new_embeddings = load_file(
40
os.path.join(sql_lora_files, "new_embeddings.safetensors"))
41
94
- lora_config_path = os.path.join(sql_lora_files, "adapter_config.json")
95
- with open(lora_config_path) as f:
96
- config = json.load(f)
97
-
98
- peft_helper = PEFTHelper.from_dict(config)
42
+ peft_helper = PEFTHelper.from_local_dir(sql_lora_files,
43
+ max_position_embeddings=4096)
44
lora_model = LoRAModel.from_lora_tensors(
45
1,
46
tensors,
tests/lora/test_peft_helper.py
new
+109
@@ -0,0 +1,109 @@
1
+import json
2
+import math
3
+import shutil
4
+
5
+import pytest
6
+
7
+from vllm.config import LoRAConfig
8
+from vllm.lora.peft_helper import PEFTHelper
9
+
10
+ERROR_CASES = [
11
+ (
12
+ "test_rank",
13
+ {
14
+ "r": 1024
15
+ },
16
+ "is greater than max_lora_rank",
17
+ ),
18
+ (
19
+ "test_bias",
20
+ {
21
+ "bias": "all"
22
+ },
23
+ "Adapter bias cannot be used without bias_enabled",
24
+ ),
25
+ ("test_dora", {
26
+ "use_dora": True
27
+ }, "does not yet support DoRA"),
28
+ (
29
+ "test_modules_to_save",
30
+ {
31
+ "modules_to_save": ["lm_head"]
32
+ },
33
+ "only supports modules_to_save being None",
34
+ ),
35
+]
36
+
37
+
38
+def test_peft_helper_pass(long_context_lora_files_16k_1, tmp_path):
39
+ peft_helper = PEFTHelper.from_local_dir(long_context_lora_files_16k_1,
40
+ max_position_embeddings=4096)
41
+ lora_config = LoRAConfig(max_lora_rank=16, max_cpu_loras=3, max_loras=2)
42
+ peft_helper.validate_legal(lora_config)
43
+ assert peft_helper.r == 8
44
+ assert peft_helper.lora_alpha == 16
45
+ assert peft_helper.target_modules == [
46
+ "q_proj",
47
+ "v_proj",
48
+ "k_proj",
49
+ "o_proj",
50
+ "gate_proj",
51
+ "up_proj",
52
+ "down_proj",
53
+ "embed_tokens",
54
+ "lm_head",
55
+ ]
56
+ assert peft_helper.context_length == 16384
57
+ assert peft_helper.vllm_max_position_embeddings == 4096
58
+ assert peft_helper.vllm_long_context_scaling_factor == float(
59
+ math.ceil(peft_helper.context_length /
60
+ peft_helper.vllm_max_position_embeddings))
61
+ # test RSLoRA
62
+ rslora_config = dict(use_rslora=True)
63
+ test_dir = tmp_path / "test_rslora"
64
+ shutil.copytree(long_context_lora_files_16k_1, test_dir)
65
+
66
+ # Load and modify configuration
67
+ config_path = test_dir / "adapter_config.json"
68
+ with open(config_path) as f:
69
+ adapter_config = json.load(f)
70
+ # Apply configuration changes
71
+ adapter_config.update(rslora_config)
72
+
73
+ # Save modified configuration
74
+ with open(config_path, "w") as f:
75
+ json.dump(adapter_config, f)
76
+
77
+ peft_helper = PEFTHelper.from_local_dir(test_dir,
78
+ max_position_embeddings=4096)
79
+ peft_helper.validate_legal(lora_config)
80
+ scaling = peft_helper.lora_alpha / math.sqrt(peft_helper.r)
81
+ assert abs(peft_helper.vllm_lora_scaling_factor - scaling) < 1e-3
82
+
83
+
84
+@pytest.mark.parametrize("test_name,config_change,expected_error", ERROR_CASES)
85
+def test_peft_helper_error(
86
+ sql_lora_files,
87
+ tmp_path,
88
+ test_name: str,
89
+ config_change: dict,
90
+ expected_error: str,
91
+):
92
+ test_dir = tmp_path / test_name
93
+ shutil.copytree(sql_lora_files, test_dir)
94
+
95
+ # Load and modify configuration
96
+ config_path = test_dir / "adapter_config.json"
97
+ with open(config_path) as f:
98
+ adapter_config = json.load(f)
99
+ # Apply configuration changes
100
+ adapter_config.update(config_change)
101
+
102
+ # Save modified configuration
103
+ with open(config_path, "w") as f:
104
+ json.dump(adapter_config, f)
105
+ lora_config = LoRAConfig(max_lora_rank=16, max_cpu_loras=3, max_loras=2)
106
+ # Test loading the adapter
107
+ with pytest.raises(ValueError, match=expected_error):
108
+ PEFTHelper.from_local_dir(
109
+ test_dir, max_position_embeddings=4096).validate_legal(lora_config)
vllm/engine/multiprocessing/engine.py
+1
@@ -296,6 +296,7 @@ class MQLLMEngine:
296
is_engine_errored=False,
297
exception=e)
298
self._send_outputs(rpc_err)
299
+ return
300
# Otherwise, send back the successful load message
301
self._send_outputs(
302
RPCAdapterLoadedResponse(request_id=request.request_id))
vllm/entrypoints/openai/serving_models.py
+8
-16
@@ -157,24 +157,16 @@ class OpenAIServingModels:
157
# This will also pre-load it for incoming requests
158
try:
159
await self.engine_client.add_lora(lora_request)
160
- except ValueError as e:
161
- # Adapter not found or lora configuration errors
162
- if "No adapter found" in str(e):
163
- return create_error_response(message=str(e),
164
- err_type="NotFoundError",
165
- status_code=HTTPStatus.NOT_FOUND)
166
- else:
167
- return create_error_response(
168
- message=str(e),
169
- err_type="BadRequestError",
170
- status_code=HTTPStatus.BAD_REQUEST)
160
except BaseException as e:
172
- # Some other unexpected problem loading the adapter, e.g. malformed
173
- # input files.
174
- # More detailed error messages for the user would be nicer here
161
+ error_type = "BadRequestError"
162
+ status_code = HTTPStatus.BAD_REQUEST
163
+ if isinstance(e, ValueError) and "No adapter found" in str(e):
164
+ error_type = "NotFoundError"
165
+ status_code = HTTPStatus.NOT_FOUND
166
+
167
return create_error_response(message=str(e),
176
- err_type="BadRequestError",
177
- status_code=HTTPStatus.BAD_REQUEST)
168
+ err_type=error_type,
169
+ status_code=status_code)
170
171
self.lora_requests.append(lora_request)
172
logger.info("Loaded new LoRA adapter: name '%s', path '%s'", lora_name,
vllm/lora/models.py
+2
-10
@@ -1,5 +1,4 @@
1
import copy
2
-import json
2
import math
3
import os
4
import re
@@ -180,8 +179,8 @@ class LoRAModel(AdapterModel):
179
cls,
180
lora_dir: str,
181
expected_lora_modules: List[str],
182
+ peft_helper: PEFTHelper,
183
*,
184
- max_position_embeddings: Optional[int] = None,
184
lora_model_id: Optional[int] = None,
185
device: str = "cuda",
186
dtype: Optional[torch.dtype] = None,
@@ -196,9 +195,7 @@ class LoRAModel(AdapterModel):
195
lora_dir: The local path that has lora data.
196
expected_lora_modules: Name of modules that are expected to be
197
replaced by lora.
199
- max_position_embeddings: Max position embedding length. Used to
200
- scaling the largest context length. If None, the lora model's
201
- context length is not scaled.
198
+ peft_helper: Loaded lora configuration information.
199
lora_model_id: Lora model id. If not given, automatically set by
200
a global counter.
201
device: Device where the lora model is loaded.
@@ -207,18 +204,13 @@ class LoRAModel(AdapterModel):
204
Returns:
205
Loaded LoRA Model.
206
"""
210
- lora_config_path = os.path.join(lora_dir, "adapter_config.json")
207
lora_tensor_path = os.path.join(lora_dir, "adapter_model.safetensors")
208
lora_bin_file_path = os.path.join(lora_dir, "adapter_model.bin")
209
new_embeddings_tensor_path = os.path.join(
210
lora_dir, "new_embeddings.safetensors")
211
new_embeddings_bin_file_path = os.path.join(lora_dir,
212
"new_embeddings.bin")
217
- with open(lora_config_path) as f:
218
- config = json.load(f)
213
220
- config["vllm_max_position_embeddings"] = max_position_embeddings
221
- peft_helper = PEFTHelper.from_dict(config)
214
unexpected_modules: List[Union[list[str], str]]
215
if os.path.isfile(lora_tensor_path):
216
tensors: Dict[str, torch.Tensor] = {}
vllm/lora/peft_helper.py
+41
-8
@@ -1,9 +1,12 @@
1
# Adapted from: https://github.com/huggingface/peft/blob/main/src/peft/tuners/lora/config.py
2
3
+import json
4
import math
5
+import os
6
from dataclasses import MISSING, dataclass, field, fields
5
-from typing import Literal, Optional, Union
7
+from typing import List, Literal, Optional, Union
8
9
+from vllm.config import LoRAConfig
10
from vllm.logger import init_logger
11
12
logger = init_logger(__name__)
@@ -11,6 +14,12 @@ logger = init_logger(__name__)
14
15
@dataclass
16
class PEFTHelper:
17
+ """
18
+ A helper class for PEFT configurations, specifically designed for LoRA.
19
+ This class handles configuration validation, compatibility checks for
20
+ various LoRA implementations.
21
+ """
22
+
23
# Required fields
24
r: int
25
lora_alpha: int
@@ -29,20 +38,18 @@ class PEFTHelper:
38
vllm_max_position_embeddings: Optional[int] = field(default=False)
39
vllm_long_context_scaling_factor: Optional[float] = field(default=None)
40
32
- def _validate_features(self):
41
+ def _validate_features(self) -> List[str]:
42
+ """
43
+ Check if there are any unsupported Lora features.
44
+ """
45
error_msg = []
34
-
46
if self.modules_to_save:
47
error_msg.append("vLLM only supports modules_to_save being None.")
37
-
48
if self.use_dora:
49
error_msg.append("vLLM does not yet support DoRA.")
40
-
41
- if error_msg:
42
- raise ValueError(f"{', '.join(error_msg)}")
50
+ return error_msg
51
52
def __post_init__(self):
45
- self._validate_features()
53
if self.use_rslora:
54
logger.info_once("Loading LoRA weights trained with rsLoRA.")
55
self.vllm_lora_scaling_factor = self.lora_alpha / math.sqrt(self.r)
@@ -78,3 +85,29 @@ class PEFTHelper:
85
for k, v in config_dict.items() if k in class_fields
86
}
87
return cls(**filtered_dict)
88
+
89
+ @classmethod
90
+ def from_local_dir(cls, lora_path: str,
91
+ max_position_embeddings: Optional[int]) -> "PEFTHelper":
92
+ lora_config_path = os.path.join(lora_path, "adapter_config.json")
93
+
94
+ with open(lora_config_path) as f:
95
+ config = json.load(f)
96
+ config["vllm_max_position_embeddings"] = max_position_embeddings
97
+ return cls.from_dict(config)
98
+
99
+ def validate_legal(self, lora_config: LoRAConfig) -> None:
100
+ """
101
+ Validates the LoRA configuration settings against application
102
+ constraints and requirements.
103
+ """
104
+ error_msg = self._validate_features()
105
+ if self.r > lora_config.max_lora_rank:
106
+ error_msg.append(
107
+ f"LoRA rank {self.r} is greater than max_lora_rank"
108
+ f" {lora_config.max_lora_rank}.")
109
+ if self.bias != "none" and not lora_config.bias_enabled:
110
+ error_msg.append(
111
+ "Adapter bias cannot be used without bias_enabled.")
112
+ if error_msg:
113
+ raise ValueError(f"{' '.join(error_msg)}")
vllm/lora/worker_manager.py
+13
-6
@@ -12,6 +12,7 @@ from vllm.config import LoRAConfig
12
from vllm.logger import init_logger
13
from vllm.lora.models import (LoRAModel, LoRAModelManager,
14
LRUCacheLoRAModelManager, create_lora_manager)
15
+from vllm.lora.peft_helper import PEFTHelper
16
from vllm.lora.request import LoRARequest
17
from vllm.lora.utils import get_adapter_absolute_path
18
@@ -95,6 +96,13 @@ class WorkerLoRAManager(AbstractWorkerManager):
96
expected_lora_modules = list(set(expected_lora_modules))
97
lora_path = get_adapter_absolute_path(lora_request.lora_path)
98
99
+ peft_helper = PEFTHelper.from_local_dir(
100
+ lora_path, self.max_position_embeddings)
101
+
102
+ # Validates the LoRA configuration against requirements before
103
+ # loading weights, throwing an exception if validation fails.
104
+ peft_helper.validate_legal(self.lora_config)
105
+
106
# For some models like Qwen2VL, we need to use hf_to_vllm_mapper
107
# to ensure correct loading of lora weights.
108
hf_to_vllm_mapper = None
@@ -105,7 +113,7 @@ class WorkerLoRAManager(AbstractWorkerManager):
113
lora = self._lora_model_cls.from_local_checkpoint(
114
lora_path,
115
expected_lora_modules,
108
- max_position_embeddings=self.max_position_embeddings,
116
+ peft_helper=peft_helper,
117
lora_model_id=lora_request.lora_int_id,
118
device="cpu",
119
dtype=self.lora_config.lora_dtype,
@@ -120,15 +128,14 @@ class WorkerLoRAManager(AbstractWorkerManager):
128
# - No adapter found to download from huggingface (or in
129
# offline mode)
130
# - No local adapter files found at `lora_request.lora_path`
131
+ # For NotFoundError
132
raise ValueError(
133
f"Loading lora {lora_request.lora_name} failed: No adapter "
134
f"found for {lora_path}") from e
135
except Exception as e:
127
- raise RuntimeError(f"Loading lora {lora_path} failed") from e
128
- if lora.rank > self.lora_config.max_lora_rank:
129
- raise ValueError(
130
- f"LoRA rank {lora.rank} is greater than max_lora_rank "
131
- f"{self.lora_config.max_lora_rank}.")
136
+ # For BadRequestError
137
+ raise e
138
+
139
if lora.extra_vocab_size > self.lora_config.lora_extra_vocab_size:
140
raise ValueError(f"LoRA added vocab size {lora.extra_vocab_size} "
141
f"is greater than lora_extra_vocab_size "