A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 20:45 UTC
[Core] Support dynamically loading Lora adapter from HuggingFace (#6234)
Co-authored-by: Antoni Baum <antoni.baum@protonmail.com>
Jiaxin Shan committed
Jul 22, 2024 at 15:42 UTC
42c7f66a386b2243dcd313feed7dec4e1d508167
11 files changed
+201
-18
tests/core/test_scheduler.py
+2
-2
@@ -462,7 +462,7 @@ def test_prefill_schedule_max_lora():
462
lora_request=LoRARequest(
463
lora_name=str(i),
464
lora_int_id=i + 1,
465
- lora_local_path="abc"))
465
+ lora_path="abc"))
466
waiting.append(seq_group)
467
# Add two more requests to verify lora is prioritized.
468
# 0: Lora, 1: Lora, 2: regular, 3: regular
@@ -760,7 +760,7 @@ def test_schedule_swapped_max_loras():
760
lora_request=LoRARequest(
761
lora_name=str(i),
762
lora_int_id=i + 1,
763
- lora_local_path="abc"))
763
+ lora_path="abc"))
764
scheduler._allocate_and_set_running(seq_group)
765
append_new_token_seq_group(60, seq_group, 1)
766
scheduler._swap_out(seq_group, blocks_to_swap_out)
tests/lora/conftest.py
+8
-2
@@ -159,8 +159,14 @@ def dummy_model_gate_up() -> nn.Module:
159
160
161
@pytest.fixture(scope="session")
162
-def sql_lora_files():
163
- return snapshot_download(repo_id="yard1/llama-2-7b-sql-lora-test")
162
+def sql_lora_huggingface_id():
163
+ # huggingface repo id is used to test lora runtime downloading.
164
+ return "yard1/llama-2-7b-sql-lora-test"
165
+
166
+
167
+@pytest.fixture(scope="session")
168
+def sql_lora_files(sql_lora_huggingface_id):
169
+ return snapshot_download(repo_id=sql_lora_huggingface_id)
170
171
172
@pytest.fixture(scope="session")
tests/lora/test_long_context.py
+1
-1
@@ -29,7 +29,7 @@ def _create_lora_request(lora_id, long_context_infos):
29
context_len = long_context_infos[lora_id]["context_length"]
30
scaling_factor = context_len_to_scaling_factor[context_len]
31
return LoRARequest(context_len, lora_id,
32
- long_context_infos[lora_id]["lora"],
32
+ long_context_infos[lora_id]["lora"], None,
33
4096 * scaling_factor)
34
35
tests/lora/test_lora_huggingface.py
new
+39
@@ -0,0 +1,39 @@
1
+from typing import List
2
+
3
+import pytest
4
+
5
+from vllm.lora.models import LoRAModel
6
+from vllm.lora.utils import get_adapter_absolute_path
7
+from vllm.model_executor.models.llama import LlamaForCausalLM
8
+
9
+# Provide absolute path and huggingface lora ids
10
+lora_fixture_name = ["sql_lora_files", "sql_lora_huggingface_id"]
11
+
12
+
13
+@pytest.mark.parametrize("lora_fixture_name", lora_fixture_name)
14
+def test_load_checkpoints_from_huggingface(lora_fixture_name, request):
15
+ lora_name = request.getfixturevalue(lora_fixture_name)
16
+ supported_lora_modules = LlamaForCausalLM.supported_lora_modules
17
+ packed_modules_mapping = LlamaForCausalLM.packed_modules_mapping
18
+ embedding_modules = LlamaForCausalLM.embedding_modules
19
+ embed_padding_modules = LlamaForCausalLM.embedding_padding_modules
20
+ expected_lora_modules: List[str] = []
21
+ for module in supported_lora_modules:
22
+ if module in packed_modules_mapping:
23
+ expected_lora_modules.extend(packed_modules_mapping[module])
24
+ else:
25
+ expected_lora_modules.append(module)
26
+
27
+ lora_path = get_adapter_absolute_path(lora_name)
28
+
29
+ # lora loading should work for either absolute path and hugggingface id.
30
+ lora_model = LoRAModel.from_local_checkpoint(
31
+ lora_path,
32
+ expected_lora_modules,
33
+ lora_model_id=1,
34
+ device="cpu",
35
+ embedding_modules=embedding_modules,
36
+ embedding_padding_modules=embed_padding_modules)
37
+
38
+ # Assertions to ensure the model is loaded correctly
39
+ assert lora_model is not None, "LoRAModel is not loaded correctly"
tests/lora/test_utils.py
+56
-1
@@ -1,9 +1,12 @@
1
from collections import OrderedDict
2
+from unittest.mock import patch
3
4
import pytest
5
+from huggingface_hub.utils import HfHubHTTPError
6
from torch import nn
7
6
-from vllm.lora.utils import parse_fine_tuned_lora_name, replace_submodule
8
+from vllm.lora.utils import (get_adapter_absolute_path,
9
+ parse_fine_tuned_lora_name, replace_submodule)
10
from vllm.utils import LRUCache
11
12
@@ -182,3 +185,55 @@ def test_lru_cache():
185
assert 2 in cache
186
assert 4 in cache
187
assert 6 in cache
188
+
189
+
190
+# Unit tests for get_adapter_absolute_path
191
+@patch('os.path.isabs')
192
+def test_get_adapter_absolute_path_absolute(mock_isabs):
193
+ path = '/absolute/path/to/lora'
194
+ mock_isabs.return_value = True
195
+ assert get_adapter_absolute_path(path) == path
196
+
197
+
198
+@patch('os.path.expanduser')
199
+def test_get_adapter_absolute_path_expanduser(mock_expanduser):
200
+ # Path with ~ that needs to be expanded
201
+ path = '~/relative/path/to/lora'
202
+ absolute_path = '/home/user/relative/path/to/lora'
203
+ mock_expanduser.return_value = absolute_path
204
+ assert get_adapter_absolute_path(path) == absolute_path
205
+
206
+
207
+@patch('os.path.exists')
208
+@patch('os.path.abspath')
209
+def test_get_adapter_absolute_path_local_existing(mock_abspath, mock_exist):
210
+ # Relative path that exists locally
211
+ path = 'relative/path/to/lora'
212
+ absolute_path = '/absolute/path/to/lora'
213
+ mock_exist.return_value = True
214
+ mock_abspath.return_value = absolute_path
215
+ assert get_adapter_absolute_path(path) == absolute_path
216
+
217
+
218
+@patch('huggingface_hub.snapshot_download')
219
+@patch('os.path.exists')
220
+def test_get_adapter_absolute_path_huggingface(mock_exist,
221
+ mock_snapshot_download):
222
+ # Hugging Face model identifier
223
+ path = 'org/repo'
224
+ absolute_path = '/mock/snapshot/path'
225
+ mock_exist.return_value = False
226
+ mock_snapshot_download.return_value = absolute_path
227
+ assert get_adapter_absolute_path(path) == absolute_path
228
+
229
+
230
+@patch('huggingface_hub.snapshot_download')
231
+@patch('os.path.exists')
232
+def test_get_adapter_absolute_path_huggingface_error(mock_exist,
233
+ mock_snapshot_download):
234
+ # Hugging Face model identifier with download error
235
+ path = 'org/repo'
236
+ mock_exist.return_value = False
237
+ mock_snapshot_download.side_effect = HfHubHTTPError(
238
+ "failed to query model info")
239
+ assert get_adapter_absolute_path(path) == path
vllm/entrypoints/openai/serving_engine.py
+2
-2
@@ -43,7 +43,7 @@ class PromptAdapterPath:
43
@dataclass
44
class LoRAModulePath:
45
name: str
46
- local_path: str
46
+ path: str
47
48
49
AnyRequest = Union[ChatCompletionRequest, CompletionRequest, DetokenizeRequest,
@@ -83,7 +83,7 @@ class OpenAIServing:
83
LoRARequest(
84
lora_name=lora.name,
85
lora_int_id=i,
86
- lora_local_path=lora.local_path,
86
+ lora_path=lora.path,
87
) for i, lora in enumerate(lora_modules, start=1)
88
]
89
vllm/lora/request.py
+39
-3
@@ -1,4 +1,5 @@
1
-from dataclasses import dataclass
1
+import warnings
2
+from dataclasses import dataclass, field
3
from typing import Optional
4
5
from vllm.adapter_commons.request import AdapterRequest
@@ -20,10 +21,25 @@ class LoRARequest(AdapterRequest):
21
22
lora_name: str
23
lora_int_id: int
23
- lora_local_path: str
24
+ lora_path: str = ""
25
+ lora_local_path: Optional[str] = field(default=None, repr=False)
26
long_lora_max_len: Optional[int] = None
27
__hash__ = AdapterRequest.__hash__
28
29
+ def __post_init__(self):
30
+ if 'lora_local_path' in self.__dict__:
31
+ warnings.warn(
32
+ "The 'lora_local_path' attribute is deprecated "
33
+ "and will be removed in a future version. "
34
+ "Please use 'lora_path' instead.",
35
+ DeprecationWarning,
36
+ stacklevel=2)
37
+ if not self.lora_path:
38
+ self.lora_path = self.lora_local_path or ""
39
+
40
+ # Ensure lora_path is not empty
41
+ assert self.lora_path, "lora_path cannot be empty"
42
+
43
@property
44
def adapter_id(self):
45
return self.lora_int_id
@@ -32,6 +48,26 @@ class LoRARequest(AdapterRequest):
48
def name(self):
49
return self.lora_name
50
51
+ @property
52
+ def path(self):
53
+ return self.lora_path
54
+
55
@property
56
def local_path(self):
37
- return self.lora_local_path
57
+ warnings.warn(
58
+ "The 'local_path' attribute is deprecated "
59
+ "and will be removed in a future version. "
60
+ "Please use 'path' instead.",
61
+ DeprecationWarning,
62
+ stacklevel=2)
63
+ return self.lora_path
64
+
65
+ @local_path.setter
66
+ def local_path(self, value):
67
+ warnings.warn(
68
+ "The 'local_path' attribute is deprecated "
69
+ "and will be removed in a future version. "
70
+ "Please use 'path' instead.",
71
+ DeprecationWarning,
72
+ stacklevel=2)
73
+ self.lora_path = value
vllm/lora/utils.py
+47
@@ -1,5 +1,9 @@
1
+import os
2
from typing import List, Optional, Set, Tuple, Type
3
4
+import huggingface_hub
5
+from huggingface_hub.utils import (EntryNotFoundError, HfHubHTTPError,
6
+ HFValidationError, RepositoryNotFoundError)
7
from torch import nn
8
from transformers import PretrainedConfig
9
@@ -105,3 +109,46 @@ def parse_fine_tuned_lora_name(name: str) -> Tuple[str, bool]:
109
return ".".join(parts[2:-1]), parts[-1] == "lora_embedding_A"
110
111
raise ValueError(f"{name} is unsupported LoRA weight")
112
+
113
+
114
+def get_adapter_absolute_path(lora_path: str) -> str:
115
+ """
116
+ Resolves the given lora_path to an absolute local path.
117
+
118
+ If the lora_path is identified as a Hugging Face model identifier,
119
+ it will download the model and return the local snapshot path.
120
+ Otherwise, it treats the lora_path as a local file path and
121
+ converts it to an absolute path.
122
+
123
+ Parameters:
124
+ lora_path (str): The path to the lora model, which can be an absolute path,
125
+ a relative path, or a Hugging Face model identifier.
126
+
127
+ Returns:
128
+ str: The resolved absolute local path to the lora model.
129
+ """
130
+
131
+ # Check if the path is an absolute path. Return it no matter exists or not.
132
+ if os.path.isabs(lora_path):
133
+ return lora_path
134
+
135
+ # If the path starts with ~, expand the user home directory.
136
+ if lora_path.startswith('~'):
137
+ return os.path.expanduser(lora_path)
138
+
139
+ # Check if the expanded relative path exists locally.
140
+ if os.path.exists(lora_path):
141
+ return os.path.abspath(lora_path)
142
+
143
+ # If the path does not exist locally, assume it's a Hugging Face repo.
144
+ try:
145
+ local_snapshot_path = huggingface_hub.snapshot_download(
146
+ repo_id=lora_path)
147
+ except (HfHubHTTPError, RepositoryNotFoundError, EntryNotFoundError,
148
+ HFValidationError):
149
+ # Handle errors that may occur during the download
150
+ # Return original path instead instead of throwing error here
151
+ logger.exception("Error downloading the HuggingFace model")
152
+ return lora_path
153
+
154
+ return local_snapshot_path
vllm/lora/worker_manager.py
+4
-3
@@ -13,6 +13,7 @@ from vllm.logger import init_logger
13
from vllm.lora.models import (LoRAModel, LoRAModelManager,
14
LRUCacheLoRAModelManager, create_lora_manager)
15
from vllm.lora.request import LoRARequest
16
+from vllm.lora.utils import get_adapter_absolute_path
17
18
logger = init_logger(__name__)
19
@@ -89,8 +90,9 @@ class WorkerLoRAManager(AbstractWorkerManager):
90
packed_modules_mapping[module])
91
else:
92
expected_lora_modules.append(module)
93
+ lora_path = get_adapter_absolute_path(lora_request.lora_path)
94
lora = self._lora_model_cls.from_local_checkpoint(
93
- lora_request.lora_local_path,
95
+ lora_path,
96
expected_lora_modules,
97
max_position_embeddings=self.max_position_embeddings,
98
lora_model_id=lora_request.lora_int_id,
@@ -102,8 +104,7 @@ class WorkerLoRAManager(AbstractWorkerManager):
104
embedding_padding_modules=self.embedding_padding_modules,
105
)
106
except Exception as e:
105
- raise RuntimeError(
106
- f"Loading lora {lora_request.lora_local_path} failed") from e
107
+ raise RuntimeError(f"Loading lora {lora_path} failed") from e
108
if lora.rank > self.lora_config.max_lora_rank:
109
raise ValueError(
110
f"LoRA rank {lora.rank} is greater than max_lora_rank "
vllm/transformers_utils/tokenizer.py
+2
-3
@@ -137,14 +137,13 @@ def get_lora_tokenizer(lora_request: LoRARequest, *args,
137
if lora_request is None:
138
return None
139
try:
140
- tokenizer = get_tokenizer(lora_request.lora_local_path, *args,
141
- **kwargs)
140
+ tokenizer = get_tokenizer(lora_request.lora_path, *args, **kwargs)
141
except OSError as e:
142
# No tokenizer was found in the LoRA folder,
143
# use base model tokenizer
144
logger.warning(
145
"No tokenizer found in %s, using base model tokenizer instead. "
147
- "(Exception: %s)", lora_request.lora_local_path, e)
146
+ "(Exception: %s)", lora_request.lora_path, e)
147
tokenizer = None
148
return tokenizer
149
vllm/worker/model_runner.py
+1
-1
@@ -691,7 +691,7 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
691
dummy_lora_request = LoRARequest(
692
lora_name=f"warmup_{lora_id}",
693
lora_int_id=lora_id,
694
- lora_local_path="/not/a/real/path",
694
+ lora_path="/not/a/real/path",
695
)
696
self.lora_manager.add_dummy_lora(dummy_lora_request,
697
rank=LORA_WARMUP_RANK)