A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 15:45 UTC
[Core] Support Lora lineage and base model metadata management (#6315)
Jiaxin Shan committed
Sep 19, 2024 at 23:20 UTC
260d40b5ea48df9421325388abcc8d907a560fc5
15 files changed
+337
-45
docs/source/models/lora.rst
+64
@@ -159,3 +159,67 @@ Example request to unload a LoRA adapter:
159
-d '{
160
"lora_name": "sql_adapter"
161
}'
162
+
163
+
164
+New format for `--lora-modules`
165
+-------------------------------
166
+
167
+In the previous version, users would provide LoRA modules via the following format, either as a key-value pair or in JSON format. For example:
168
+
169
+.. code-block:: bash
170
+
171
+ --lora-modules sql-lora=$HOME/.cache/huggingface/hub/models--yard1--llama-2-7b-sql-lora-test/snapshots/0dfa347e8877a4d4ed19ee56c140fa518470028c/
172
+
173
+This would only include the `name` and `path` for each LoRA module, but did not provide a way to specify a `base_model_name`.
174
+Now, you can specify a base_model_name alongside the name and path using JSON format. For example:
175
+
176
+.. code-block:: bash
177
+
178
+ --lora-modules '{"name": "sql-lora", "path": "/path/to/lora", "base_model_name": "meta-llama/Llama-2-7b"}'
179
+
180
+To provide the backward compatibility support, you can still use the old key-value format (name=path), but the `base_model_name` will remain unspecified in that case.
181
+
182
+
183
+Lora model lineage in model card
184
+--------------------------------
185
+
186
+The new format of `--lora-modules` is mainly to support the display of parent model information in the model card. Here's an explanation of how your current response supports this:
187
+
188
+- The `parent` field of LoRA model `sql-lora` now links to its base model `meta-llama/Llama-2-7b-hf`. This correctly reflects the hierarchical relationship between the base model and the LoRA adapter.
189
+- The `root` field points to the artifact location of the lora adapter.
190
+
191
+.. code-block:: bash
192
+
193
+ $ curl http://localhost:8000/v1/models
194
+
195
+ {
196
+ "object": "list",
197
+ "data": [
198
+ {
199
+ "id": "meta-llama/Llama-2-7b-hf",
200
+ "object": "model",
201
+ "created": 1715644056,
202
+ "owned_by": "vllm",
203
+ "root": "~/.cache/huggingface/hub/models--meta-llama--Llama-2-7b-hf/snapshots/01c7f73d771dfac7d292323805ebc428287df4f9/",
204
+ "parent": null,
205
+ "permission": [
206
+ {
207
+ .....
208
+ }
209
+ ]
210
+ },
211
+ {
212
+ "id": "sql-lora",
213
+ "object": "model",
214
+ "created": 1715644056,
215
+ "owned_by": "vllm",
216
+ "root": "~/.cache/huggingface/hub/models--yard1--llama-2-7b-sql-lora-test/snapshots/0dfa347e8877a4d4ed19ee56c140fa518470028c/",
217
+ "parent": meta-llama/Llama-2-7b-hf,
218
+ "permission": [
219
+ {
220
+ ....
221
+ }
222
+ ]
223
+ }
224
+ ]
225
+ }
tests/entrypoints/openai/test_cli_args.py
new
+91
@@ -0,0 +1,91 @@
1
+import json
2
+import unittest
3
+
4
+from vllm.entrypoints.openai.cli_args import make_arg_parser
5
+from vllm.entrypoints.openai.serving_engine import LoRAModulePath
6
+from vllm.utils import FlexibleArgumentParser
7
+
8
+LORA_MODULE = {
9
+ "name": "module2",
10
+ "path": "/path/to/module2",
11
+ "base_model_name": "llama"
12
+}
13
+
14
+
15
+class TestLoraParserAction(unittest.TestCase):
16
+
17
+ def setUp(self):
18
+ # Setting up argparse parser for tests
19
+ parser = FlexibleArgumentParser(
20
+ description="vLLM's remote OpenAI server.")
21
+ self.parser = make_arg_parser(parser)
22
+
23
+ def test_valid_key_value_format(self):
24
+ # Test old format: name=path
25
+ args = self.parser.parse_args([
26
+ '--lora-modules',
27
+ 'module1=/path/to/module1',
28
+ ])
29
+ expected = [LoRAModulePath(name='module1', path='/path/to/module1')]
30
+ self.assertEqual(args.lora_modules, expected)
31
+
32
+ def test_valid_json_format(self):
33
+ # Test valid JSON format input
34
+ args = self.parser.parse_args([
35
+ '--lora-modules',
36
+ json.dumps(LORA_MODULE),
37
+ ])
38
+ expected = [
39
+ LoRAModulePath(name='module2',
40
+ path='/path/to/module2',
41
+ base_model_name='llama')
42
+ ]
43
+ self.assertEqual(args.lora_modules, expected)
44
+
45
+ def test_invalid_json_format(self):
46
+ # Test invalid JSON format input, missing closing brace
47
+ with self.assertRaises(SystemExit):
48
+ self.parser.parse_args([
49
+ '--lora-modules',
50
+ '{"name": "module3", "path": "/path/to/module3"'
51
+ ])
52
+
53
+ def test_invalid_type_error(self):
54
+ # Test type error when values are not JSON or key=value
55
+ with self.assertRaises(SystemExit):
56
+ self.parser.parse_args([
57
+ '--lora-modules',
58
+ 'invalid_format' # This is not JSON or key=value format
59
+ ])
60
+
61
+ def test_invalid_json_field(self):
62
+ # Test valid JSON format but missing required fields
63
+ with self.assertRaises(SystemExit):
64
+ self.parser.parse_args([
65
+ '--lora-modules',
66
+ '{"name": "module4"}' # Missing required 'path' field
67
+ ])
68
+
69
+ def test_empty_values(self):
70
+ # Test when no LoRA modules are provided
71
+ args = self.parser.parse_args(['--lora-modules', ''])
72
+ self.assertEqual(args.lora_modules, [])
73
+
74
+ def test_multiple_valid_inputs(self):
75
+ # Test multiple valid inputs (both old and JSON format)
76
+ args = self.parser.parse_args([
77
+ '--lora-modules',
78
+ 'module1=/path/to/module1',
79
+ json.dumps(LORA_MODULE),
80
+ ])
81
+ expected = [
82
+ LoRAModulePath(name='module1', path='/path/to/module1'),
83
+ LoRAModulePath(name='module2',
84
+ path='/path/to/module2',
85
+ base_model_name='llama')
86
+ ]
87
+ self.assertEqual(args.lora_modules, expected)
88
+
89
+
90
+if __name__ == '__main__':
91
+ unittest.main()
tests/entrypoints/openai/test_lora_lineage.py
new
+83
@@ -0,0 +1,83 @@
1
+import json
2
+
3
+import openai # use the official client for correctness check
4
+import pytest
5
+import pytest_asyncio
6
+# downloading lora to test lora requests
7
+from huggingface_hub import snapshot_download
8
+
9
+from ...utils import RemoteOpenAIServer
10
+
11
+# any model with a chat template should work here
12
+MODEL_NAME = "HuggingFaceH4/zephyr-7b-beta"
13
+# technically this needs Mistral-7B-v0.1 as base, but we're not testing
14
+# generation quality here
15
+LORA_NAME = "typeof/zephyr-7b-beta-lora"
16
+
17
+
18
+@pytest.fixture(scope="module")
19
+def zephyr_lora_files():
20
+ return snapshot_download(repo_id=LORA_NAME)
21
+
22
+
23
+@pytest.fixture(scope="module")
24
+def server_with_lora_modules_json(zephyr_lora_files):
25
+ # Define the json format LoRA module configurations
26
+ lora_module_1 = {
27
+ "name": "zephyr-lora",
28
+ "path": zephyr_lora_files,
29
+ "base_model_name": MODEL_NAME
30
+ }
31
+
32
+ lora_module_2 = {
33
+ "name": "zephyr-lora2",
34
+ "path": zephyr_lora_files,
35
+ "base_model_name": MODEL_NAME
36
+ }
37
+
38
+ args = [
39
+ # use half precision for speed and memory savings in CI environment
40
+ "--dtype",
41
+ "bfloat16",
42
+ "--max-model-len",
43
+ "8192",
44
+ "--enforce-eager",
45
+ # lora config below
46
+ "--enable-lora",
47
+ "--lora-modules",
48
+ json.dumps(lora_module_1),
49
+ json.dumps(lora_module_2),
50
+ "--max-lora-rank",
51
+ "64",
52
+ "--max-cpu-loras",
53
+ "2",
54
+ "--max-num-seqs",
55
+ "64",
56
+ ]
57
+
58
+ with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
59
+ yield remote_server
60
+
61
+
62
+@pytest_asyncio.fixture
63
+async def client_for_lora_lineage(server_with_lora_modules_json):
64
+ async with server_with_lora_modules_json.get_async_client(
65
+ ) as async_client:
66
+ yield async_client
67
+
68
+
69
+@pytest.mark.asyncio
70
+async def test_check_lora_lineage(client_for_lora_lineage: openai.AsyncOpenAI,
71
+ zephyr_lora_files):
72
+ models = await client_for_lora_lineage.models.list()
73
+ models = models.data
74
+ served_model = models[0]
75
+ lora_models = models[1:]
76
+ assert served_model.id == MODEL_NAME
77
+ assert served_model.root == MODEL_NAME
78
+ assert served_model.parent is None
79
+ assert all(lora_model.root == zephyr_lora_files
80
+ for lora_model in lora_models)
81
+ assert all(lora_model.parent == MODEL_NAME for lora_model in lora_models)
82
+ assert lora_models[0].id == "zephyr-lora"
83
+ assert lora_models[1].id == "zephyr-lora2"
tests/entrypoints/openai/test_models.py
+4
-2
@@ -51,12 +51,14 @@ async def client(server):
51
52
53
@pytest.mark.asyncio
54
-async def test_check_models(client: openai.AsyncOpenAI):
54
+async def test_check_models(client: openai.AsyncOpenAI, zephyr_lora_files):
55
models = await client.models.list()
56
models = models.data
57
served_model = models[0]
58
lora_models = models[1:]
59
assert served_model.id == MODEL_NAME
60
- assert all(model.root == MODEL_NAME for model in models)
60
+ assert served_model.root == MODEL_NAME
61
+ assert all(lora_model.root == zephyr_lora_files
62
+ for lora_model in lora_models)
63
assert lora_models[0].id == "zephyr-lora"
64
assert lora_models[1].id == "zephyr-lora2"
tests/entrypoints/openai/test_serving_chat.py
+4
-2
@@ -7,10 +7,12 @@ from vllm.config import MultiModalConfig
7
from vllm.engine.multiprocessing.client import MQLLMEngineClient
8
from vllm.entrypoints.openai.protocol import ChatCompletionRequest
9
from vllm.entrypoints.openai.serving_chat import OpenAIServingChat
10
+from vllm.entrypoints.openai.serving_engine import BaseModelPath
11
from vllm.transformers_utils.tokenizer import get_tokenizer
12
13
MODEL_NAME = "openai-community/gpt2"
14
CHAT_TEMPLATE = "Dummy chat template for testing {}"
15
+BASE_MODEL_PATHS = [BaseModelPath(name=MODEL_NAME, model_path=MODEL_NAME)]
16
17
18
@dataclass
@@ -37,7 +39,7 @@ async def _async_serving_chat_init():
39
40
serving_completion = OpenAIServingChat(engine,
41
model_config,
40
- served_model_names=[MODEL_NAME],
42
+ BASE_MODEL_PATHS,
43
response_role="assistant",
44
chat_template=CHAT_TEMPLATE,
45
lora_modules=None,
@@ -58,7 +60,7 @@ def test_serving_chat_should_set_correct_max_tokens():
60
61
serving_chat = OpenAIServingChat(mock_engine,
62
MockModelConfig(),
61
- served_model_names=[MODEL_NAME],
63
+ BASE_MODEL_PATHS,
64
response_role="assistant",
65
chat_template=CHAT_TEMPLATE,
66
lora_modules=None,
tests/entrypoints/openai/test_serving_engine.py
+3
-2
@@ -8,9 +8,10 @@ from vllm.engine.protocol import EngineClient
8
from vllm.entrypoints.openai.protocol import (ErrorResponse,
9
LoadLoraAdapterRequest,
10
UnloadLoraAdapterRequest)
11
-from vllm.entrypoints.openai.serving_engine import OpenAIServing
11
+from vllm.entrypoints.openai.serving_engine import BaseModelPath, OpenAIServing
12
13
MODEL_NAME = "meta-llama/Llama-2-7b"
14
+BASE_MODEL_PATHS = [BaseModelPath(name=MODEL_NAME, model_path=MODEL_NAME)]
15
LORA_LOADING_SUCCESS_MESSAGE = (
16
"Success: LoRA adapter '{lora_name}' added successfully.")
17
LORA_UNLOADING_SUCCESS_MESSAGE = (
@@ -25,7 +26,7 @@ async def _async_serving_engine_init():
26
27
serving_engine = OpenAIServing(mock_engine_client,
28
mock_model_config,
28
- served_model_names=[MODEL_NAME],
29
+ BASE_MODEL_PATHS,
30
lora_modules=None,
31
prompt_adapters=None,
32
request_logger=None)
vllm/entrypoints/openai/api_server.py
+10
-4
@@ -50,6 +50,7 @@ from vllm.entrypoints.openai.protocol import (ChatCompletionRequest,
50
from vllm.entrypoints.openai.serving_chat import OpenAIServingChat
51
from vllm.entrypoints.openai.serving_completion import OpenAIServingCompletion
52
from vllm.entrypoints.openai.serving_embedding import OpenAIServingEmbedding
53
+from vllm.entrypoints.openai.serving_engine import BaseModelPath
54
from vllm.entrypoints.openai.serving_tokenization import (
55
OpenAIServingTokenization)
56
from vllm.logger import init_logger
@@ -476,13 +477,18 @@ def init_app_state(
477
else:
478
request_logger = RequestLogger(max_log_len=args.max_log_len)
479
480
+ base_model_paths = [
481
+ BaseModelPath(name=name, model_path=args.model)
482
+ for name in served_model_names
483
+ ]
484
+
485
state.engine_client = engine_client
486
state.log_stats = not args.disable_log_stats
487
488
state.openai_serving_chat = OpenAIServingChat(
489
engine_client,
490
model_config,
485
- served_model_names,
491
+ base_model_paths,
492
args.response_role,
493
lora_modules=args.lora_modules,
494
prompt_adapters=args.prompt_adapters,
@@ -494,7 +500,7 @@ def init_app_state(
500
state.openai_serving_completion = OpenAIServingCompletion(
501
engine_client,
502
model_config,
497
- served_model_names,
503
+ base_model_paths,
504
lora_modules=args.lora_modules,
505
prompt_adapters=args.prompt_adapters,
506
request_logger=request_logger,
@@ -503,13 +509,13 @@ def init_app_state(
509
state.openai_serving_embedding = OpenAIServingEmbedding(
510
engine_client,
511
model_config,
506
- served_model_names,
512
+ base_model_paths,
513
request_logger=request_logger,
514
)
515
state.openai_serving_tokenization = OpenAIServingTokenization(
516
engine_client,
517
model_config,
512
- served_model_names,
518
+ base_model_paths,
519
lora_modules=args.lora_modules,
520
request_logger=request_logger,
521
chat_template=args.chat_template,
vllm/entrypoints/openai/cli_args.py
+23
-4
@@ -31,8 +31,23 @@ class LoRAParserAction(argparse.Action):
31
32
lora_list: List[LoRAModulePath] = []
33
for item in values:
34
- name, path = item.split('=')
35
- lora_list.append(LoRAModulePath(name, path))
34
+ if item in [None, '']: # Skip if item is None or empty string
35
+ continue
36
+ if '=' in item and ',' not in item: # Old format: name=path
37
+ name, path = item.split('=')
38
+ lora_list.append(LoRAModulePath(name, path))
39
+ else: # Assume JSON format
40
+ try:
41
+ lora_dict = json.loads(item)
42
+ lora = LoRAModulePath(**lora_dict)
43
+ lora_list.append(lora)
44
+ except json.JSONDecodeError:
45
+ parser.error(
46
+ f"Invalid JSON format for --lora-modules: {item}")
47
+ except TypeError as e:
48
+ parser.error(
49
+ f"Invalid fields for --lora-modules: {item} - {str(e)}"
50
+ )
51
setattr(namespace, self.dest, lora_list)
52
53
@@ -95,8 +110,12 @@ def make_arg_parser(parser: FlexibleArgumentParser) -> FlexibleArgumentParser:
110
default=None,
111
nargs='+',
112
action=LoRAParserAction,
98
- help="LoRA module configurations in the format name=path. "
99
- "Multiple modules can be specified.")
113
+ help="LoRA module configurations in either 'name=path' format"
114
+ "or JSON format. "
115
+ "Example (old format): 'name=path' "
116
+ "Example (new format): "
117
+ "'{\"name\": \"name\", \"local_path\": \"path\", "
118
+ "\"base_model_name\": \"id\"}'")
119
parser.add_argument(
120
"--prompt-adapters",
121
type=nullable_str,
vllm/entrypoints/openai/run_batch.py
+7
-2
@@ -20,6 +20,7 @@ from vllm.entrypoints.openai.protocol import (BatchRequestInput,
20
# yapf: enable
21
from vllm.entrypoints.openai.serving_chat import OpenAIServingChat
22
from vllm.entrypoints.openai.serving_embedding import OpenAIServingEmbedding
23
+from vllm.entrypoints.openai.serving_engine import BaseModelPath
24
from vllm.usage.usage_lib import UsageContext
25
from vllm.utils import FlexibleArgumentParser, random_uuid
26
from vllm.version import __version__ as VLLM_VERSION
@@ -196,6 +197,10 @@ async def main(args):
197
engine_args, usage_context=UsageContext.OPENAI_BATCH_RUNNER)
198
199
model_config = await engine.get_model_config()
200
+ base_model_paths = [
201
+ BaseModelPath(name=name, model_path=args.model)
202
+ for name in served_model_names
203
+ ]
204
205
if args.disable_log_requests:
206
request_logger = None
@@ -206,7 +211,7 @@ async def main(args):
211
openai_serving_chat = OpenAIServingChat(
212
engine,
213
model_config,
209
- served_model_names,
214
+ base_model_paths,
215
args.response_role,
216
lora_modules=None,
217
prompt_adapters=None,
@@ -216,7 +221,7 @@ async def main(args):
221
openai_serving_embedding = OpenAIServingEmbedding(
222
engine,
223
model_config,
219
- served_model_names,
224
+ base_model_paths,
225
request_logger=request_logger,
226
)
227
vllm/entrypoints/openai/serving_chat.py
+6
-5
@@ -23,7 +23,8 @@ from vllm.entrypoints.openai.protocol import (
23
ChatCompletionResponseChoice, ChatCompletionResponseStreamChoice,
24
ChatCompletionStreamResponse, ChatMessage, DeltaFunctionCall, DeltaMessage,
25
DeltaToolCall, ErrorResponse, FunctionCall, ToolCall, UsageInfo)
26
-from vllm.entrypoints.openai.serving_engine import (LoRAModulePath,
26
+from vllm.entrypoints.openai.serving_engine import (BaseModelPath,
27
+ LoRAModulePath,
28
OpenAIServing,
29
PromptAdapterPath,
30
TextTokensPrompt)
@@ -47,7 +48,7 @@ class OpenAIServingChat(OpenAIServing):
48
def __init__(self,
49
engine_client: EngineClient,
50
model_config: ModelConfig,
50
- served_model_names: List[str],
51
+ base_model_paths: List[BaseModelPath],
52
response_role: str,
53
*,
54
lora_modules: Optional[List[LoRAModulePath]],
@@ -59,7 +60,7 @@ class OpenAIServingChat(OpenAIServing):
60
tool_parser: Optional[str] = None):
61
super().__init__(engine_client=engine_client,
62
model_config=model_config,
62
- served_model_names=served_model_names,
63
+ base_model_paths=base_model_paths,
64
lora_modules=lora_modules,
65
prompt_adapters=prompt_adapters,
66
request_logger=request_logger,
@@ -262,7 +263,7 @@ class OpenAIServingChat(OpenAIServing):
263
conversation: List[ConversationMessage],
264
tokenizer: AnyTokenizer,
265
) -> AsyncGenerator[str, None]:
265
- model_name = self.served_model_names[0]
266
+ model_name = self.base_model_paths[0].name
267
created_time = int(time.time())
268
chunk_object_type: Final = "chat.completion.chunk"
269
first_iteration = True
@@ -596,7 +597,7 @@ class OpenAIServingChat(OpenAIServing):
597
tokenizer: AnyTokenizer,
598
) -> Union[ErrorResponse, ChatCompletionResponse]:
599
599
- model_name = self.served_model_names[0]
600
+ model_name = self.base_model_paths[0].name
601
created_time = int(time.time())
602
final_res: Optional[RequestOutput] = None
603
vllm/entrypoints/openai/serving_completion.py
+5
-4
@@ -20,7 +20,8 @@ from vllm.entrypoints.openai.protocol import (CompletionLogProbs,
20
CompletionStreamResponse,
21
ErrorResponse, UsageInfo)
22
# yapf: enable
23
-from vllm.entrypoints.openai.serving_engine import (LoRAModulePath,
23
+from vllm.entrypoints.openai.serving_engine import (BaseModelPath,
24
+ LoRAModulePath,
25
OpenAIServing,
26
PromptAdapterPath)
27
from vllm.logger import init_logger
@@ -45,7 +46,7 @@ class OpenAIServingCompletion(OpenAIServing):
46
self,
47
engine_client: EngineClient,
48
model_config: ModelConfig,
48
- served_model_names: List[str],
49
+ base_model_paths: List[BaseModelPath],
50
*,
51
lora_modules: Optional[List[LoRAModulePath]],
52
prompt_adapters: Optional[List[PromptAdapterPath]],
@@ -54,7 +55,7 @@ class OpenAIServingCompletion(OpenAIServing):
55
):
56
super().__init__(engine_client=engine_client,
57
model_config=model_config,
57
- served_model_names=served_model_names,
58
+ base_model_paths=base_model_paths,
59
lora_modules=lora_modules,
60
prompt_adapters=prompt_adapters,
61
request_logger=request_logger,
@@ -89,7 +90,7 @@ class OpenAIServingCompletion(OpenAIServing):
90
return self.create_error_response(
91
"suffix is not currently supported")
92
92
- model_name = self.served_model_names[0]
93
+ model_name = self.base_model_paths[0].name
94
request_id = f"cmpl-{random_uuid()}"
95
created_time = int(time.time())
96
vllm/entrypoints/openai/serving_embedding.py
+3
-3
@@ -14,7 +14,7 @@ from vllm.entrypoints.openai.protocol import (EmbeddingRequest,
14
EmbeddingResponse,
15
EmbeddingResponseData,
16
ErrorResponse, UsageInfo)
17
-from vllm.entrypoints.openai.serving_engine import OpenAIServing
17
+from vllm.entrypoints.openai.serving_engine import BaseModelPath, OpenAIServing
18
from vllm.logger import init_logger
19
from vllm.outputs import EmbeddingOutput, EmbeddingRequestOutput
20
from vllm.utils import merge_async_iterators, random_uuid
@@ -73,13 +73,13 @@ class OpenAIServingEmbedding(OpenAIServing):
73
self,
74
engine_client: EngineClient,
75
model_config: ModelConfig,
76
- served_model_names: List[str],
76
+ base_model_paths: List[BaseModelPath],
77
*,
78
request_logger: Optional[RequestLogger],
79
):
80
super().__init__(engine_client=engine_client,
81
model_config=model_config,
82
- served_model_names=served_model_names,
82
+ base_model_paths=base_model_paths,
83
lora_modules=None,
84
prompt_adapters=None,
85
request_logger=request_logger)
vllm/entrypoints/openai/serving_engine.py
+29
-14
@@ -39,6 +39,12 @@ from vllm.utils import AtomicCounter
39
logger = init_logger(__name__)
40
41
42
+@dataclass
43
+class BaseModelPath:
44
+ name: str
45
+ model_path: str
46
+
47
+
48
@dataclass
49
class PromptAdapterPath:
50
name: str
@@ -49,6 +55,7 @@ class PromptAdapterPath:
55
class LoRAModulePath:
56
name: str
57
path: str
58
+ base_model_name: Optional[str] = None
59
60
61
AnyRequest = Union[ChatCompletionRequest, CompletionRequest, DetokenizeRequest,
@@ -66,7 +73,7 @@ class OpenAIServing:
73
self,
74
engine_client: EngineClient,
75
model_config: ModelConfig,
69
- served_model_names: List[str],
76
+ base_model_paths: List[BaseModelPath],
77
*,
78
lora_modules: Optional[List[LoRAModulePath]],
79
prompt_adapters: Optional[List[PromptAdapterPath]],
@@ -79,17 +86,20 @@ class OpenAIServing:
86
self.model_config = model_config
87
self.max_model_len = model_config.max_model_len
88
82
- self.served_model_names = served_model_names
89
+ self.base_model_paths = base_model_paths
90
91
self.lora_id_counter = AtomicCounter(0)
92
self.lora_requests = []
93
if lora_modules is not None:
94
self.lora_requests = [
88
- LoRARequest(
89
- lora_name=lora.name,
90
- lora_int_id=i,
91
- lora_path=lora.path,
92
- ) for i, lora in enumerate(lora_modules, start=1)
95
+ LoRARequest(lora_name=lora.name,
96
+ lora_int_id=i,
97
+ lora_path=lora.path,
98
+ base_model_name=lora.base_model_name
99
+ if lora.base_model_name
100
+ and self._is_model_supported(lora.base_model_name)
101
+ else self.base_model_paths[0].name)
102
+ for i, lora in enumerate(lora_modules, start=1)
103
]
104
105
self.prompt_adapter_requests = []
@@ -112,21 +122,23 @@ class OpenAIServing:
122
async def show_available_models(self) -> ModelList:
123
"""Show available models. Right now we only have one model."""
124
model_cards = [
115
- ModelCard(id=served_model_name,
125
+ ModelCard(id=base_model.name,
126
max_model_len=self.max_model_len,
117
- root=self.served_model_names[0],
127
+ root=base_model.model_path,
128
permission=[ModelPermission()])
119
- for served_model_name in self.served_model_names
129
+ for base_model in self.base_model_paths
130
]
131
lora_cards = [
132
ModelCard(id=lora.lora_name,
123
- root=self.served_model_names[0],
133
+ root=lora.local_path,
134
+ parent=lora.base_model_name if lora.base_model_name else
135
+ self.base_model_paths[0].name,
136
permission=[ModelPermission()])
137
for lora in self.lora_requests
138
]
139
prompt_adapter_cards = [
140
ModelCard(id=prompt_adapter.prompt_adapter_name,
129
- root=self.served_model_names[0],
141
+ root=self.base_model_paths[0].name,
142
permission=[ModelPermission()])
143
for prompt_adapter in self.prompt_adapter_requests
144
]
@@ -169,7 +181,7 @@ class OpenAIServing:
181
self,
182
request: AnyRequest,
183
) -> Optional[ErrorResponse]:
172
- if request.model in self.served_model_names:
184
+ if self._is_model_supported(request.model):
185
return None
186
if request.model in [lora.lora_name for lora in self.lora_requests]:
187
return None
@@ -187,7 +199,7 @@ class OpenAIServing:
199
self, request: AnyRequest
200
) -> Union[Tuple[None, None], Tuple[LoRARequest, None], Tuple[
201
None, PromptAdapterRequest]]:
190
- if request.model in self.served_model_names:
202
+ if self._is_model_supported(request.model):
203
return None, None
204
for lora in self.lora_requests:
205
if request.model == lora.lora_name:
@@ -480,3 +492,6 @@ class OpenAIServing:
492
if lora_request.lora_name != lora_name
493
]
494
return f"Success: LoRA adapter '{lora_name}' removed successfully."
495
+
496
+ def _is_model_supported(self, model_name):
497
+ return any(model.name == model_name for model in self.base_model_paths)
vllm/entrypoints/openai/serving_tokenization.py
+4
-3
@@ -16,7 +16,8 @@ from vllm.entrypoints.openai.protocol import (DetokenizeRequest,
16
TokenizeRequest,
17
TokenizeResponse)
18
# yapf: enable
19
-from vllm.entrypoints.openai.serving_engine import (LoRAModulePath,
19
+from vllm.entrypoints.openai.serving_engine import (BaseModelPath,
20
+ LoRAModulePath,
21
OpenAIServing)
22
from vllm.logger import init_logger
23
from vllm.transformers_utils.tokenizer import MistralTokenizer
@@ -31,7 +32,7 @@ class OpenAIServingTokenization(OpenAIServing):
32
self,
33
engine_client: EngineClient,
34
model_config: ModelConfig,
34
- served_model_names: List[str],
35
+ base_model_paths: List[BaseModelPath],
36
*,
37
lora_modules: Optional[List[LoRAModulePath]],
38
request_logger: Optional[RequestLogger],
@@ -39,7 +40,7 @@ class OpenAIServingTokenization(OpenAIServing):
40
):
41
super().__init__(engine_client=engine_client,
42
model_config=model_config,
42
- served_model_names=served_model_names,
43
+ base_model_paths=base_model_paths,
44
lora_modules=lora_modules,
45
prompt_adapters=None,
46
request_logger=request_logger)
vllm/lora/request.py
+1
@@ -28,6 +28,7 @@ class LoRARequest(
28
lora_path: str = ""
29
lora_local_path: Optional[str] = msgspec.field(default=None)
30
long_lora_max_len: Optional[int] = None
31
+ base_model_name: Optional[str] = msgspec.field(default=None)
32
33
def __post_init__(self):
34
if 'lora_local_path' in self.__struct_fields__: