@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 260d40b5ea
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 15:45 UTC

[Core] Support Lora lineage and base model metadata management (#6315)

Jiaxin Shan committed Sep 19, 2024 at 23:20 UTC 260d40b5ea48df9421325388abcc8d907a560fc5
15 files changed +337 -45
docs/source/models/lora.rst
+64
@@ -159,3 +159,67 @@ Example request to unload a LoRA adapter:
159 -d '{
160 "lora_name": "sql_adapter"
161 }'
162 +
163 +
164 +New format for `--lora-modules`
165 +-------------------------------
166 +
167 +In the previous version, users would provide LoRA modules via the following format, either as a key-value pair or in JSON format. For example:
168 +
169 +.. code-block:: bash
170 +
171 + --lora-modules sql-lora=$HOME/.cache/huggingface/hub/models--yard1--llama-2-7b-sql-lora-test/snapshots/0dfa347e8877a4d4ed19ee56c140fa518470028c/
172 +
173 +This would only include the `name` and `path` for each LoRA module, but did not provide a way to specify a `base_model_name`.
174 +Now, you can specify a base_model_name alongside the name and path using JSON format. For example:
175 +
176 +.. code-block:: bash
177 +
178 + --lora-modules '{"name": "sql-lora", "path": "/path/to/lora", "base_model_name": "meta-llama/Llama-2-7b"}'
179 +
180 +To provide the backward compatibility support, you can still use the old key-value format (name=path), but the `base_model_name` will remain unspecified in that case.
181 +
182 +
183 +Lora model lineage in model card
184 +--------------------------------
185 +
186 +The new format of `--lora-modules` is mainly to support the display of parent model information in the model card. Here's an explanation of how your current response supports this:
187 +
188 +- The `parent` field of LoRA model `sql-lora` now links to its base model `meta-llama/Llama-2-7b-hf`. This correctly reflects the hierarchical relationship between the base model and the LoRA adapter.
189 +- The `root` field points to the artifact location of the lora adapter.
190 +
191 +.. code-block:: bash
192 +
193 + $ curl http://localhost:8000/v1/models
194 +
195 + {
196 + "object": "list",
197 + "data": [
198 + {
199 + "id": "meta-llama/Llama-2-7b-hf",
200 + "object": "model",
201 + "created": 1715644056,
202 + "owned_by": "vllm",
203 + "root": "~/.cache/huggingface/hub/models--meta-llama--Llama-2-7b-hf/snapshots/01c7f73d771dfac7d292323805ebc428287df4f9/",
204 + "parent": null,
205 + "permission": [
206 + {
207 + .....
208 + }
209 + ]
210 + },
211 + {
212 + "id": "sql-lora",
213 + "object": "model",
214 + "created": 1715644056,
215 + "owned_by": "vllm",
216 + "root": "~/.cache/huggingface/hub/models--yard1--llama-2-7b-sql-lora-test/snapshots/0dfa347e8877a4d4ed19ee56c140fa518470028c/",
217 + "parent": meta-llama/Llama-2-7b-hf,
218 + "permission": [
219 + {
220 + ....
221 + }
222 + ]
223 + }
224 + ]
225 + }
tests/entrypoints/openai/test_cli_args.py new
+91
@@ -0,0 +1,91 @@
1 +import json
2 +import unittest
3 +
4 +from vllm.entrypoints.openai.cli_args import make_arg_parser
5 +from vllm.entrypoints.openai.serving_engine import LoRAModulePath
6 +from vllm.utils import FlexibleArgumentParser
7 +
8 +LORA_MODULE = {
9 + "name": "module2",
10 + "path": "/path/to/module2",
11 + "base_model_name": "llama"
12 +}
13 +
14 +
15 +class TestLoraParserAction(unittest.TestCase):
16 +
17 + def setUp(self):
18 + # Setting up argparse parser for tests
19 + parser = FlexibleArgumentParser(
20 + description="vLLM's remote OpenAI server.")
21 + self.parser = make_arg_parser(parser)
22 +
23 + def test_valid_key_value_format(self):
24 + # Test old format: name=path
25 + args = self.parser.parse_args([
26 + '--lora-modules',
27 + 'module1=/path/to/module1',
28 + ])
29 + expected = [LoRAModulePath(name='module1', path='/path/to/module1')]
30 + self.assertEqual(args.lora_modules, expected)
31 +
32 + def test_valid_json_format(self):
33 + # Test valid JSON format input
34 + args = self.parser.parse_args([
35 + '--lora-modules',
36 + json.dumps(LORA_MODULE),
37 + ])
38 + expected = [
39 + LoRAModulePath(name='module2',
40 + path='/path/to/module2',
41 + base_model_name='llama')
42 + ]
43 + self.assertEqual(args.lora_modules, expected)
44 +
45 + def test_invalid_json_format(self):
46 + # Test invalid JSON format input, missing closing brace
47 + with self.assertRaises(SystemExit):
48 + self.parser.parse_args([
49 + '--lora-modules',
50 + '{"name": "module3", "path": "/path/to/module3"'
51 + ])
52 +
53 + def test_invalid_type_error(self):
54 + # Test type error when values are not JSON or key=value
55 + with self.assertRaises(SystemExit):
56 + self.parser.parse_args([
57 + '--lora-modules',
58 + 'invalid_format' # This is not JSON or key=value format
59 + ])
60 +
61 + def test_invalid_json_field(self):
62 + # Test valid JSON format but missing required fields
63 + with self.assertRaises(SystemExit):
64 + self.parser.parse_args([
65 + '--lora-modules',
66 + '{"name": "module4"}' # Missing required 'path' field
67 + ])
68 +
69 + def test_empty_values(self):
70 + # Test when no LoRA modules are provided
71 + args = self.parser.parse_args(['--lora-modules', ''])
72 + self.assertEqual(args.lora_modules, [])
73 +
74 + def test_multiple_valid_inputs(self):
75 + # Test multiple valid inputs (both old and JSON format)
76 + args = self.parser.parse_args([
77 + '--lora-modules',
78 + 'module1=/path/to/module1',
79 + json.dumps(LORA_MODULE),
80 + ])
81 + expected = [
82 + LoRAModulePath(name='module1', path='/path/to/module1'),
83 + LoRAModulePath(name='module2',
84 + path='/path/to/module2',
85 + base_model_name='llama')
86 + ]
87 + self.assertEqual(args.lora_modules, expected)
88 +
89 +
90 +if __name__ == '__main__':
91 + unittest.main()
tests/entrypoints/openai/test_lora_lineage.py new
+83
@@ -0,0 +1,83 @@
1 +import json
2 +
3 +import openai # use the official client for correctness check
4 +import pytest
5 +import pytest_asyncio
6 +# downloading lora to test lora requests
7 +from huggingface_hub import snapshot_download
8 +
9 +from ...utils import RemoteOpenAIServer
10 +
11 +# any model with a chat template should work here
12 +MODEL_NAME = "HuggingFaceH4/zephyr-7b-beta"
13 +# technically this needs Mistral-7B-v0.1 as base, but we're not testing
14 +# generation quality here
15 +LORA_NAME = "typeof/zephyr-7b-beta-lora"
16 +
17 +
18 +@pytest.fixture(scope="module")
19 +def zephyr_lora_files():
20 + return snapshot_download(repo_id=LORA_NAME)
21 +
22 +
23 +@pytest.fixture(scope="module")
24 +def server_with_lora_modules_json(zephyr_lora_files):
25 + # Define the json format LoRA module configurations
26 + lora_module_1 = {
27 + "name": "zephyr-lora",
28 + "path": zephyr_lora_files,
29 + "base_model_name": MODEL_NAME
30 + }
31 +
32 + lora_module_2 = {
33 + "name": "zephyr-lora2",
34 + "path": zephyr_lora_files,
35 + "base_model_name": MODEL_NAME
36 + }
37 +
38 + args = [
39 + # use half precision for speed and memory savings in CI environment
40 + "--dtype",
41 + "bfloat16",
42 + "--max-model-len",
43 + "8192",
44 + "--enforce-eager",
45 + # lora config below
46 + "--enable-lora",
47 + "--lora-modules",
48 + json.dumps(lora_module_1),
49 + json.dumps(lora_module_2),
50 + "--max-lora-rank",
51 + "64",
52 + "--max-cpu-loras",
53 + "2",
54 + "--max-num-seqs",
55 + "64",
56 + ]
57 +
58 + with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
59 + yield remote_server
60 +
61 +
62 +@pytest_asyncio.fixture
63 +async def client_for_lora_lineage(server_with_lora_modules_json):
64 + async with server_with_lora_modules_json.get_async_client(
65 + ) as async_client:
66 + yield async_client
67 +
68 +
69 +@pytest.mark.asyncio
70 +async def test_check_lora_lineage(client_for_lora_lineage: openai.AsyncOpenAI,
71 + zephyr_lora_files):
72 + models = await client_for_lora_lineage.models.list()
73 + models = models.data
74 + served_model = models[0]
75 + lora_models = models[1:]
76 + assert served_model.id == MODEL_NAME
77 + assert served_model.root == MODEL_NAME
78 + assert served_model.parent is None
79 + assert all(lora_model.root == zephyr_lora_files
80 + for lora_model in lora_models)
81 + assert all(lora_model.parent == MODEL_NAME for lora_model in lora_models)
82 + assert lora_models[0].id == "zephyr-lora"
83 + assert lora_models[1].id == "zephyr-lora2"
tests/entrypoints/openai/test_models.py
+4 -2
@@ -51,12 +51,14 @@ async def client(server):
51
52
53 @pytest.mark.asyncio
54 -async def test_check_models(client: openai.AsyncOpenAI):
54 +async def test_check_models(client: openai.AsyncOpenAI, zephyr_lora_files):
55 models = await client.models.list()
56 models = models.data
57 served_model = models[0]
58 lora_models = models[1:]
59 assert served_model.id == MODEL_NAME
60 - assert all(model.root == MODEL_NAME for model in models)
60 + assert served_model.root == MODEL_NAME
61 + assert all(lora_model.root == zephyr_lora_files
62 + for lora_model in lora_models)
63 assert lora_models[0].id == "zephyr-lora"
64 assert lora_models[1].id == "zephyr-lora2"
tests/entrypoints/openai/test_serving_chat.py
+4 -2
@@ -7,10 +7,12 @@ from vllm.config import MultiModalConfig
7 from vllm.engine.multiprocessing.client import MQLLMEngineClient
8 from vllm.entrypoints.openai.protocol import ChatCompletionRequest
9 from vllm.entrypoints.openai.serving_chat import OpenAIServingChat
10 +from vllm.entrypoints.openai.serving_engine import BaseModelPath
11 from vllm.transformers_utils.tokenizer import get_tokenizer
12
13 MODEL_NAME = "openai-community/gpt2"
14 CHAT_TEMPLATE = "Dummy chat template for testing {}"
15 +BASE_MODEL_PATHS = [BaseModelPath(name=MODEL_NAME, model_path=MODEL_NAME)]
16
17
18 @dataclass
@@ -37,7 +39,7 @@ async def _async_serving_chat_init():
39
40 serving_completion = OpenAIServingChat(engine,
41 model_config,
40 - served_model_names=[MODEL_NAME],
42 + BASE_MODEL_PATHS,
43 response_role="assistant",
44 chat_template=CHAT_TEMPLATE,
45 lora_modules=None,
@@ -58,7 +60,7 @@ def test_serving_chat_should_set_correct_max_tokens():
60
61 serving_chat = OpenAIServingChat(mock_engine,
62 MockModelConfig(),
61 - served_model_names=[MODEL_NAME],
63 + BASE_MODEL_PATHS,
64 response_role="assistant",
65 chat_template=CHAT_TEMPLATE,
66 lora_modules=None,
tests/entrypoints/openai/test_serving_engine.py
+3 -2
@@ -8,9 +8,10 @@ from vllm.engine.protocol import EngineClient
8 from vllm.entrypoints.openai.protocol import (ErrorResponse,
9 LoadLoraAdapterRequest,
10 UnloadLoraAdapterRequest)
11 -from vllm.entrypoints.openai.serving_engine import OpenAIServing
11 +from vllm.entrypoints.openai.serving_engine import BaseModelPath, OpenAIServing
12
13 MODEL_NAME = "meta-llama/Llama-2-7b"
14 +BASE_MODEL_PATHS = [BaseModelPath(name=MODEL_NAME, model_path=MODEL_NAME)]
15 LORA_LOADING_SUCCESS_MESSAGE = (
16 "Success: LoRA adapter '{lora_name}' added successfully.")
17 LORA_UNLOADING_SUCCESS_MESSAGE = (
@@ -25,7 +26,7 @@ async def _async_serving_engine_init():
26
27 serving_engine = OpenAIServing(mock_engine_client,
28 mock_model_config,
28 - served_model_names=[MODEL_NAME],
29 + BASE_MODEL_PATHS,
30 lora_modules=None,
31 prompt_adapters=None,
32 request_logger=None)
vllm/entrypoints/openai/api_server.py
+10 -4
@@ -50,6 +50,7 @@ from vllm.entrypoints.openai.protocol import (ChatCompletionRequest,
50 from vllm.entrypoints.openai.serving_chat import OpenAIServingChat
51 from vllm.entrypoints.openai.serving_completion import OpenAIServingCompletion
52 from vllm.entrypoints.openai.serving_embedding import OpenAIServingEmbedding
53 +from vllm.entrypoints.openai.serving_engine import BaseModelPath
54 from vllm.entrypoints.openai.serving_tokenization import (
55 OpenAIServingTokenization)
56 from vllm.logger import init_logger
@@ -476,13 +477,18 @@ def init_app_state(
477 else:
478 request_logger = RequestLogger(max_log_len=args.max_log_len)
479
480 + base_model_paths = [
481 + BaseModelPath(name=name, model_path=args.model)
482 + for name in served_model_names
483 + ]
484 +
485 state.engine_client = engine_client
486 state.log_stats = not args.disable_log_stats
487
488 state.openai_serving_chat = OpenAIServingChat(
489 engine_client,
490 model_config,
485 - served_model_names,
491 + base_model_paths,
492 args.response_role,
493 lora_modules=args.lora_modules,
494 prompt_adapters=args.prompt_adapters,
@@ -494,7 +500,7 @@ def init_app_state(
500 state.openai_serving_completion = OpenAIServingCompletion(
501 engine_client,
502 model_config,
497 - served_model_names,
503 + base_model_paths,
504 lora_modules=args.lora_modules,
505 prompt_adapters=args.prompt_adapters,
506 request_logger=request_logger,
@@ -503,13 +509,13 @@ def init_app_state(
509 state.openai_serving_embedding = OpenAIServingEmbedding(
510 engine_client,
511 model_config,
506 - served_model_names,
512 + base_model_paths,
513 request_logger=request_logger,
514 )
515 state.openai_serving_tokenization = OpenAIServingTokenization(
516 engine_client,
517 model_config,
512 - served_model_names,
518 + base_model_paths,
519 lora_modules=args.lora_modules,
520 request_logger=request_logger,
521 chat_template=args.chat_template,
vllm/entrypoints/openai/cli_args.py
+23 -4
@@ -31,8 +31,23 @@ class LoRAParserAction(argparse.Action):
31
32 lora_list: List[LoRAModulePath] = []
33 for item in values:
34 - name, path = item.split('=')
35 - lora_list.append(LoRAModulePath(name, path))
34 + if item in [None, '']: # Skip if item is None or empty string
35 + continue
36 + if '=' in item and ',' not in item: # Old format: name=path
37 + name, path = item.split('=')
38 + lora_list.append(LoRAModulePath(name, path))
39 + else: # Assume JSON format
40 + try:
41 + lora_dict = json.loads(item)
42 + lora = LoRAModulePath(**lora_dict)
43 + lora_list.append(lora)
44 + except json.JSONDecodeError:
45 + parser.error(
46 + f"Invalid JSON format for --lora-modules: {item}")
47 + except TypeError as e:
48 + parser.error(
49 + f"Invalid fields for --lora-modules: {item} - {str(e)}"
50 + )
51 setattr(namespace, self.dest, lora_list)
52
53
@@ -95,8 +110,12 @@ def make_arg_parser(parser: FlexibleArgumentParser) -> FlexibleArgumentParser:
110 default=None,
111 nargs='+',
112 action=LoRAParserAction,
98 - help="LoRA module configurations in the format name=path. "
99 - "Multiple modules can be specified.")
113 + help="LoRA module configurations in either 'name=path' format"
114 + "or JSON format. "
115 + "Example (old format): 'name=path' "
116 + "Example (new format): "
117 + "'{\"name\": \"name\", \"local_path\": \"path\", "
118 + "\"base_model_name\": \"id\"}'")
119 parser.add_argument(
120 "--prompt-adapters",
121 type=nullable_str,
vllm/entrypoints/openai/run_batch.py
+7 -2
@@ -20,6 +20,7 @@ from vllm.entrypoints.openai.protocol import (BatchRequestInput,
20 # yapf: enable
21 from vllm.entrypoints.openai.serving_chat import OpenAIServingChat
22 from vllm.entrypoints.openai.serving_embedding import OpenAIServingEmbedding
23 +from vllm.entrypoints.openai.serving_engine import BaseModelPath
24 from vllm.usage.usage_lib import UsageContext
25 from vllm.utils import FlexibleArgumentParser, random_uuid
26 from vllm.version import __version__ as VLLM_VERSION
@@ -196,6 +197,10 @@ async def main(args):
197 engine_args, usage_context=UsageContext.OPENAI_BATCH_RUNNER)
198
199 model_config = await engine.get_model_config()
200 + base_model_paths = [
201 + BaseModelPath(name=name, model_path=args.model)
202 + for name in served_model_names
203 + ]
204
205 if args.disable_log_requests:
206 request_logger = None
@@ -206,7 +211,7 @@ async def main(args):
211 openai_serving_chat = OpenAIServingChat(
212 engine,
213 model_config,
209 - served_model_names,
214 + base_model_paths,
215 args.response_role,
216 lora_modules=None,
217 prompt_adapters=None,
@@ -216,7 +221,7 @@ async def main(args):
221 openai_serving_embedding = OpenAIServingEmbedding(
222 engine,
223 model_config,
219 - served_model_names,
224 + base_model_paths,
225 request_logger=request_logger,
226 )
227
vllm/entrypoints/openai/serving_chat.py
+6 -5
@@ -23,7 +23,8 @@ from vllm.entrypoints.openai.protocol import (
23 ChatCompletionResponseChoice, ChatCompletionResponseStreamChoice,
24 ChatCompletionStreamResponse, ChatMessage, DeltaFunctionCall, DeltaMessage,
25 DeltaToolCall, ErrorResponse, FunctionCall, ToolCall, UsageInfo)
26 -from vllm.entrypoints.openai.serving_engine import (LoRAModulePath,
26 +from vllm.entrypoints.openai.serving_engine import (BaseModelPath,
27 + LoRAModulePath,
28 OpenAIServing,
29 PromptAdapterPath,
30 TextTokensPrompt)
@@ -47,7 +48,7 @@ class OpenAIServingChat(OpenAIServing):
48 def __init__(self,
49 engine_client: EngineClient,
50 model_config: ModelConfig,
50 - served_model_names: List[str],
51 + base_model_paths: List[BaseModelPath],
52 response_role: str,
53 *,
54 lora_modules: Optional[List[LoRAModulePath]],
@@ -59,7 +60,7 @@ class OpenAIServingChat(OpenAIServing):
60 tool_parser: Optional[str] = None):
61 super().__init__(engine_client=engine_client,
62 model_config=model_config,
62 - served_model_names=served_model_names,
63 + base_model_paths=base_model_paths,
64 lora_modules=lora_modules,
65 prompt_adapters=prompt_adapters,
66 request_logger=request_logger,
@@ -262,7 +263,7 @@ class OpenAIServingChat(OpenAIServing):
263 conversation: List[ConversationMessage],
264 tokenizer: AnyTokenizer,
265 ) -> AsyncGenerator[str, None]:
265 - model_name = self.served_model_names[0]
266 + model_name = self.base_model_paths[0].name
267 created_time = int(time.time())
268 chunk_object_type: Final = "chat.completion.chunk"
269 first_iteration = True
@@ -596,7 +597,7 @@ class OpenAIServingChat(OpenAIServing):
597 tokenizer: AnyTokenizer,
598 ) -> Union[ErrorResponse, ChatCompletionResponse]:
599
599 - model_name = self.served_model_names[0]
600 + model_name = self.base_model_paths[0].name
601 created_time = int(time.time())
602 final_res: Optional[RequestOutput] = None
603
vllm/entrypoints/openai/serving_completion.py
+5 -4
@@ -20,7 +20,8 @@ from vllm.entrypoints.openai.protocol import (CompletionLogProbs,
20 CompletionStreamResponse,
21 ErrorResponse, UsageInfo)
22 # yapf: enable
23 -from vllm.entrypoints.openai.serving_engine import (LoRAModulePath,
23 +from vllm.entrypoints.openai.serving_engine import (BaseModelPath,
24 + LoRAModulePath,
25 OpenAIServing,
26 PromptAdapterPath)
27 from vllm.logger import init_logger
@@ -45,7 +46,7 @@ class OpenAIServingCompletion(OpenAIServing):
46 self,
47 engine_client: EngineClient,
48 model_config: ModelConfig,
48 - served_model_names: List[str],
49 + base_model_paths: List[BaseModelPath],
50 *,
51 lora_modules: Optional[List[LoRAModulePath]],
52 prompt_adapters: Optional[List[PromptAdapterPath]],
@@ -54,7 +55,7 @@ class OpenAIServingCompletion(OpenAIServing):
55 ):
56 super().__init__(engine_client=engine_client,
57 model_config=model_config,
57 - served_model_names=served_model_names,
58 + base_model_paths=base_model_paths,
59 lora_modules=lora_modules,
60 prompt_adapters=prompt_adapters,
61 request_logger=request_logger,
@@ -89,7 +90,7 @@ class OpenAIServingCompletion(OpenAIServing):
90 return self.create_error_response(
91 "suffix is not currently supported")
92
92 - model_name = self.served_model_names[0]
93 + model_name = self.base_model_paths[0].name
94 request_id = f"cmpl-{random_uuid()}"
95 created_time = int(time.time())
96
vllm/entrypoints/openai/serving_embedding.py
+3 -3
@@ -14,7 +14,7 @@ from vllm.entrypoints.openai.protocol import (EmbeddingRequest,
14 EmbeddingResponse,
15 EmbeddingResponseData,
16 ErrorResponse, UsageInfo)
17 -from vllm.entrypoints.openai.serving_engine import OpenAIServing
17 +from vllm.entrypoints.openai.serving_engine import BaseModelPath, OpenAIServing
18 from vllm.logger import init_logger
19 from vllm.outputs import EmbeddingOutput, EmbeddingRequestOutput
20 from vllm.utils import merge_async_iterators, random_uuid
@@ -73,13 +73,13 @@ class OpenAIServingEmbedding(OpenAIServing):
73 self,
74 engine_client: EngineClient,
75 model_config: ModelConfig,
76 - served_model_names: List[str],
76 + base_model_paths: List[BaseModelPath],
77 *,
78 request_logger: Optional[RequestLogger],
79 ):
80 super().__init__(engine_client=engine_client,
81 model_config=model_config,
82 - served_model_names=served_model_names,
82 + base_model_paths=base_model_paths,
83 lora_modules=None,
84 prompt_adapters=None,
85 request_logger=request_logger)
vllm/entrypoints/openai/serving_engine.py
+29 -14
@@ -39,6 +39,12 @@ from vllm.utils import AtomicCounter
39 logger = init_logger(__name__)
40
41
42 +@dataclass
43 +class BaseModelPath:
44 + name: str
45 + model_path: str
46 +
47 +
48 @dataclass
49 class PromptAdapterPath:
50 name: str
@@ -49,6 +55,7 @@ class PromptAdapterPath:
55 class LoRAModulePath:
56 name: str
57 path: str
58 + base_model_name: Optional[str] = None
59
60
61 AnyRequest = Union[ChatCompletionRequest, CompletionRequest, DetokenizeRequest,
@@ -66,7 +73,7 @@ class OpenAIServing:
73 self,
74 engine_client: EngineClient,
75 model_config: ModelConfig,
69 - served_model_names: List[str],
76 + base_model_paths: List[BaseModelPath],
77 *,
78 lora_modules: Optional[List[LoRAModulePath]],
79 prompt_adapters: Optional[List[PromptAdapterPath]],
@@ -79,17 +86,20 @@ class OpenAIServing:
86 self.model_config = model_config
87 self.max_model_len = model_config.max_model_len
88
82 - self.served_model_names = served_model_names
89 + self.base_model_paths = base_model_paths
90
91 self.lora_id_counter = AtomicCounter(0)
92 self.lora_requests = []
93 if lora_modules is not None:
94 self.lora_requests = [
88 - LoRARequest(
89 - lora_name=lora.name,
90 - lora_int_id=i,
91 - lora_path=lora.path,
92 - ) for i, lora in enumerate(lora_modules, start=1)
95 + LoRARequest(lora_name=lora.name,
96 + lora_int_id=i,
97 + lora_path=lora.path,
98 + base_model_name=lora.base_model_name
99 + if lora.base_model_name
100 + and self._is_model_supported(lora.base_model_name)
101 + else self.base_model_paths[0].name)
102 + for i, lora in enumerate(lora_modules, start=1)
103 ]
104
105 self.prompt_adapter_requests = []
@@ -112,21 +122,23 @@ class OpenAIServing:
122 async def show_available_models(self) -> ModelList:
123 """Show available models. Right now we only have one model."""
124 model_cards = [
115 - ModelCard(id=served_model_name,
125 + ModelCard(id=base_model.name,
126 max_model_len=self.max_model_len,
117 - root=self.served_model_names[0],
127 + root=base_model.model_path,
128 permission=[ModelPermission()])
119 - for served_model_name in self.served_model_names
129 + for base_model in self.base_model_paths
130 ]
131 lora_cards = [
132 ModelCard(id=lora.lora_name,
123 - root=self.served_model_names[0],
133 + root=lora.local_path,
134 + parent=lora.base_model_name if lora.base_model_name else
135 + self.base_model_paths[0].name,
136 permission=[ModelPermission()])
137 for lora in self.lora_requests
138 ]
139 prompt_adapter_cards = [
140 ModelCard(id=prompt_adapter.prompt_adapter_name,
129 - root=self.served_model_names[0],
141 + root=self.base_model_paths[0].name,
142 permission=[ModelPermission()])
143 for prompt_adapter in self.prompt_adapter_requests
144 ]
@@ -169,7 +181,7 @@ class OpenAIServing:
181 self,
182 request: AnyRequest,
183 ) -> Optional[ErrorResponse]:
172 - if request.model in self.served_model_names:
184 + if self._is_model_supported(request.model):
185 return None
186 if request.model in [lora.lora_name for lora in self.lora_requests]:
187 return None
@@ -187,7 +199,7 @@ class OpenAIServing:
199 self, request: AnyRequest
200 ) -> Union[Tuple[None, None], Tuple[LoRARequest, None], Tuple[
201 None, PromptAdapterRequest]]:
190 - if request.model in self.served_model_names:
202 + if self._is_model_supported(request.model):
203 return None, None
204 for lora in self.lora_requests:
205 if request.model == lora.lora_name:
@@ -480,3 +492,6 @@ class OpenAIServing:
492 if lora_request.lora_name != lora_name
493 ]
494 return f"Success: LoRA adapter '{lora_name}' removed successfully."
495 +
496 + def _is_model_supported(self, model_name):
497 + return any(model.name == model_name for model in self.base_model_paths)
vllm/entrypoints/openai/serving_tokenization.py
+4 -3
@@ -16,7 +16,8 @@ from vllm.entrypoints.openai.protocol import (DetokenizeRequest,
16 TokenizeRequest,
17 TokenizeResponse)
18 # yapf: enable
19 -from vllm.entrypoints.openai.serving_engine import (LoRAModulePath,
19 +from vllm.entrypoints.openai.serving_engine import (BaseModelPath,
20 + LoRAModulePath,
21 OpenAIServing)
22 from vllm.logger import init_logger
23 from vllm.transformers_utils.tokenizer import MistralTokenizer
@@ -31,7 +32,7 @@ class OpenAIServingTokenization(OpenAIServing):
32 self,
33 engine_client: EngineClient,
34 model_config: ModelConfig,
34 - served_model_names: List[str],
35 + base_model_paths: List[BaseModelPath],
36 *,
37 lora_modules: Optional[List[LoRAModulePath]],
38 request_logger: Optional[RequestLogger],
@@ -39,7 +40,7 @@ class OpenAIServingTokenization(OpenAIServing):
40 ):
41 super().__init__(engine_client=engine_client,
42 model_config=model_config,
42 - served_model_names=served_model_names,
43 + base_model_paths=base_model_paths,
44 lora_modules=lora_modules,
45 prompt_adapters=None,
46 request_logger=request_logger)
vllm/lora/request.py
+1
@@ -28,6 +28,7 @@ class LoRARequest(
28 lora_path: str = ""
29 lora_local_path: Optional[str] = msgspec.field(default=None)
30 long_lora_max_len: Optional[int] = None
31 + base_model_name: Optional[str] = msgspec.field(default=None)
32
33 def __post_init__(self):
34 if 'lora_local_path' in self.__struct_fields__: