@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 4db72e57f6
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 03:15 UTC

[Bugfix][Refactor] Unify model management in frontend (#11660)

Signed-off-by: Joe Runde <Joseph.Runde@ibm.com>

Joe Runde committed Dec 31, 2024 at 18:21 UTC 4db72e57f6e8da5e78285e9868e9327167bea973
15 files changed +365 -307
tests/entrypoints/openai/test_cli_args.py
+1 -1
@@ -4,7 +4,7 @@ import pytest
4
5 from vllm.entrypoints.openai.cli_args import (make_arg_parser,
6 validate_parsed_serve_args)
7 -from vllm.entrypoints.openai.serving_engine import LoRAModulePath
7 +from vllm.entrypoints.openai.serving_models import LoRAModulePath
8 from vllm.utils import FlexibleArgumentParser
9
10 from ...utils import VLLM_PATH
tests/entrypoints/openai/test_lora_lineage.py
+29 -3
@@ -55,7 +55,10 @@ def server_with_lora_modules_json(zephyr_lora_files):
55 "64",
56 ]
57
58 - with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
58 + # Enable the /v1/load_lora_adapter endpoint
59 + envs = {"VLLM_ALLOW_RUNTIME_LORA_UPDATING": "True"}
60 +
61 + with RemoteOpenAIServer(MODEL_NAME, args, env_dict=envs) as remote_server:
62 yield remote_server
63
64
@@ -67,8 +70,8 @@ async def client_for_lora_lineage(server_with_lora_modules_json):
70
71
72 @pytest.mark.asyncio
70 -async def test_check_lora_lineage(client_for_lora_lineage: openai.AsyncOpenAI,
71 - zephyr_lora_files):
73 +async def test_static_lora_lineage(client_for_lora_lineage: openai.AsyncOpenAI,
74 + zephyr_lora_files):
75 models = await client_for_lora_lineage.models.list()
76 models = models.data
77 served_model = models[0]
@@ -81,3 +84,26 @@ async def test_check_lora_lineage(client_for_lora_lineage: openai.AsyncOpenAI,
84 assert all(lora_model.parent == MODEL_NAME for lora_model in lora_models)
85 assert lora_models[0].id == "zephyr-lora"
86 assert lora_models[1].id == "zephyr-lora2"
87 +
88 +
89 +@pytest.mark.asyncio
90 +async def test_dynamic_lora_lineage(
91 + client_for_lora_lineage: openai.AsyncOpenAI, zephyr_lora_files):
92 +
93 + response = await client_for_lora_lineage.post("load_lora_adapter",
94 + cast_to=str,
95 + body={
96 + "lora_name":
97 + "zephyr-lora-3",
98 + "lora_path":
99 + zephyr_lora_files
100 + })
101 + # Ensure adapter loads before querying /models
102 + assert "success" in response
103 +
104 + models = await client_for_lora_lineage.models.list()
105 + models = models.data
106 + dynamic_lora_model = models[-1]
107 + assert dynamic_lora_model.root == zephyr_lora_files
108 + assert dynamic_lora_model.parent == MODEL_NAME
109 + assert dynamic_lora_model.id == "zephyr-lora-3"
tests/entrypoints/openai/test_serving_chat.py
+10 -10
@@ -8,7 +8,8 @@ from vllm.config import MultiModalConfig
8 from vllm.engine.multiprocessing.client import MQLLMEngineClient
9 from vllm.entrypoints.openai.protocol import ChatCompletionRequest
10 from vllm.entrypoints.openai.serving_chat import OpenAIServingChat
11 -from vllm.entrypoints.openai.serving_engine import BaseModelPath
11 +from vllm.entrypoints.openai.serving_models import (BaseModelPath,
12 + OpenAIServingModels)
13 from vllm.transformers_utils.tokenizer import get_tokenizer
14
15 MODEL_NAME = "openai-community/gpt2"
@@ -50,14 +51,13 @@ async def _async_serving_chat_init():
51 engine = MockEngine()
52 model_config = await engine.get_model_config()
53
54 + models = OpenAIServingModels(model_config, BASE_MODEL_PATHS)
55 serving_completion = OpenAIServingChat(engine,
56 model_config,
55 - BASE_MODEL_PATHS,
57 + models,
58 response_role="assistant",
59 chat_template=CHAT_TEMPLATE,
60 chat_template_content_format="auto",
59 - lora_modules=None,
60 - prompt_adapters=None,
61 request_logger=None)
62 return serving_completion
63
@@ -72,14 +72,14 @@ def test_serving_chat_should_set_correct_max_tokens():
72 mock_engine.get_tokenizer.return_value = get_tokenizer(MODEL_NAME)
73 mock_engine.errored = False
74
75 + models = OpenAIServingModels(base_model_paths=BASE_MODEL_PATHS,
76 + model_config=MockModelConfig())
77 serving_chat = OpenAIServingChat(mock_engine,
78 MockModelConfig(),
77 - BASE_MODEL_PATHS,
79 + models,
80 response_role="assistant",
81 chat_template=CHAT_TEMPLATE,
82 chat_template_content_format="auto",
81 - lora_modules=None,
82 - prompt_adapters=None,
83 request_logger=None)
84 req = ChatCompletionRequest(
85 model=MODEL_NAME,
@@ -115,14 +115,14 @@ def test_serving_chat_could_load_correct_generation_config():
115 mock_engine.errored = False
116
117 # Initialize the serving chat
118 + models = OpenAIServingModels(base_model_paths=BASE_MODEL_PATHS,
119 + model_config=mock_model_config)
120 serving_chat = OpenAIServingChat(mock_engine,
121 mock_model_config,
120 - BASE_MODEL_PATHS,
122 + models,
123 response_role="assistant",
124 chat_template=CHAT_TEMPLATE,
125 chat_template_content_format="auto",
124 - lora_modules=None,
125 - prompt_adapters=None,
126 request_logger=None)
127 req = ChatCompletionRequest(
128 model=MODEL_NAME,
tests/entrypoints/openai/test_serving_models.py renamed
+32 -34
@@ -4,11 +4,11 @@ from unittest.mock import MagicMock
4 import pytest
5
6 from vllm.config import ModelConfig
7 -from vllm.engine.protocol import EngineClient
7 from vllm.entrypoints.openai.protocol import (ErrorResponse,
8 LoadLoraAdapterRequest,
9 UnloadLoraAdapterRequest)
11 -from vllm.entrypoints.openai.serving_engine import BaseModelPath, OpenAIServing
10 +from vllm.entrypoints.openai.serving_models import (BaseModelPath,
11 + OpenAIServingModels)
12 from vllm.lora.request import LoRARequest
13
14 MODEL_NAME = "meta-llama/Llama-2-7b"
@@ -19,47 +19,45 @@ LORA_UNLOADING_SUCCESS_MESSAGE = (
19 "Success: LoRA adapter '{lora_name}' removed successfully.")
20
21
22 -async def _async_serving_engine_init():
23 - mock_engine_client = MagicMock(spec=EngineClient)
22 +async def _async_serving_models_init() -> OpenAIServingModels:
23 mock_model_config = MagicMock(spec=ModelConfig)
24 # Set the max_model_len attribute to avoid missing attribute
25 mock_model_config.max_model_len = 2048
26
28 - serving_engine = OpenAIServing(mock_engine_client,
29 - mock_model_config,
30 - BASE_MODEL_PATHS,
31 - lora_modules=None,
32 - prompt_adapters=None,
33 - request_logger=None)
34 - return serving_engine
27 + serving_models = OpenAIServingModels(base_model_paths=BASE_MODEL_PATHS,
28 + model_config=mock_model_config,
29 + lora_modules=None,
30 + prompt_adapters=None)
31 +
32 + return serving_models
33
34
35 @pytest.mark.asyncio
36 async def test_serving_model_name():
39 - serving_engine = await _async_serving_engine_init()
40 - assert serving_engine._get_model_name(None) == MODEL_NAME
37 + serving_models = await _async_serving_models_init()
38 + assert serving_models.model_name(None) == MODEL_NAME
39 request = LoRARequest(lora_name="adapter",
40 lora_path="/path/to/adapter2",
41 lora_int_id=1)
44 - assert serving_engine._get_model_name(request) == request.lora_name
42 + assert serving_models.model_name(request) == request.lora_name
43
44
45 @pytest.mark.asyncio
46 async def test_load_lora_adapter_success():
49 - serving_engine = await _async_serving_engine_init()
47 + serving_models = await _async_serving_models_init()
48 request = LoadLoraAdapterRequest(lora_name="adapter",
49 lora_path="/path/to/adapter2")
52 - response = await serving_engine.load_lora_adapter(request)
50 + response = await serving_models.load_lora_adapter(request)
51 assert response == LORA_LOADING_SUCCESS_MESSAGE.format(lora_name='adapter')
54 - assert len(serving_engine.lora_requests) == 1
55 - assert serving_engine.lora_requests[0].lora_name == "adapter"
52 + assert len(serving_models.lora_requests) == 1
53 + assert serving_models.lora_requests[0].lora_name == "adapter"
54
55
56 @pytest.mark.asyncio
57 async def test_load_lora_adapter_missing_fields():
60 - serving_engine = await _async_serving_engine_init()
58 + serving_models = await _async_serving_models_init()
59 request = LoadLoraAdapterRequest(lora_name="", lora_path="")
62 - response = await serving_engine.load_lora_adapter(request)
60 + response = await serving_models.load_lora_adapter(request)
61 assert isinstance(response, ErrorResponse)
62 assert response.type == "InvalidUserInput"
63 assert response.code == HTTPStatus.BAD_REQUEST
@@ -67,43 +65,43 @@ async def test_load_lora_adapter_missing_fields():
65
66 @pytest.mark.asyncio
67 async def test_load_lora_adapter_duplicate():
70 - serving_engine = await _async_serving_engine_init()
68 + serving_models = await _async_serving_models_init()
69 request = LoadLoraAdapterRequest(lora_name="adapter1",
70 lora_path="/path/to/adapter1")
73 - response = await serving_engine.load_lora_adapter(request)
71 + response = await serving_models.load_lora_adapter(request)
72 assert response == LORA_LOADING_SUCCESS_MESSAGE.format(
73 lora_name='adapter1')
76 - assert len(serving_engine.lora_requests) == 1
74 + assert len(serving_models.lora_requests) == 1
75
76 request = LoadLoraAdapterRequest(lora_name="adapter1",
77 lora_path="/path/to/adapter1")
80 - response = await serving_engine.load_lora_adapter(request)
78 + response = await serving_models.load_lora_adapter(request)
79 assert isinstance(response, ErrorResponse)
80 assert response.type == "InvalidUserInput"
81 assert response.code == HTTPStatus.BAD_REQUEST
84 - assert len(serving_engine.lora_requests) == 1
82 + assert len(serving_models.lora_requests) == 1
83
84
85 @pytest.mark.asyncio
86 async def test_unload_lora_adapter_success():
89 - serving_engine = await _async_serving_engine_init()
87 + serving_models = await _async_serving_models_init()
88 request = LoadLoraAdapterRequest(lora_name="adapter1",
89 lora_path="/path/to/adapter1")
92 - response = await serving_engine.load_lora_adapter(request)
93 - assert len(serving_engine.lora_requests) == 1
90 + response = await serving_models.load_lora_adapter(request)
91 + assert len(serving_models.lora_requests) == 1
92
93 request = UnloadLoraAdapterRequest(lora_name="adapter1")
96 - response = await serving_engine.unload_lora_adapter(request)
94 + response = await serving_models.unload_lora_adapter(request)
95 assert response == LORA_UNLOADING_SUCCESS_MESSAGE.format(
96 lora_name='adapter1')
99 - assert len(serving_engine.lora_requests) == 0
97 + assert len(serving_models.lora_requests) == 0
98
99
100 @pytest.mark.asyncio
101 async def test_unload_lora_adapter_missing_fields():
104 - serving_engine = await _async_serving_engine_init()
102 + serving_models = await _async_serving_models_init()
103 request = UnloadLoraAdapterRequest(lora_name="", lora_int_id=None)
106 - response = await serving_engine.unload_lora_adapter(request)
104 + response = await serving_models.unload_lora_adapter(request)
105 assert isinstance(response, ErrorResponse)
106 assert response.type == "InvalidUserInput"
107 assert response.code == HTTPStatus.BAD_REQUEST
@@ -111,9 +109,9 @@ async def test_unload_lora_adapter_missing_fields():
109
110 @pytest.mark.asyncio
111 async def test_unload_lora_adapter_not_found():
114 - serving_engine = await _async_serving_engine_init()
112 + serving_models = await _async_serving_models_init()
113 request = UnloadLoraAdapterRequest(lora_name="nonexistent_adapter")
116 - response = await serving_engine.unload_lora_adapter(request)
114 + response = await serving_models.unload_lora_adapter(request)
115 assert isinstance(response, ErrorResponse)
116 assert response.type == "InvalidUserInput"
117 assert response.code == HTTPStatus.BAD_REQUEST
vllm/entrypoints/openai/api_server.py
+33 -29
@@ -58,7 +58,9 @@ from vllm.entrypoints.openai.protocol import (ChatCompletionRequest,
58 from vllm.entrypoints.openai.serving_chat import OpenAIServingChat
59 from vllm.entrypoints.openai.serving_completion import OpenAIServingCompletion
60 from vllm.entrypoints.openai.serving_embedding import OpenAIServingEmbedding
61 -from vllm.entrypoints.openai.serving_engine import BaseModelPath, OpenAIServing
61 +from vllm.entrypoints.openai.serving_engine import OpenAIServing
62 +from vllm.entrypoints.openai.serving_models import (BaseModelPath,
63 + OpenAIServingModels)
64 from vllm.entrypoints.openai.serving_pooling import OpenAIServingPooling
65 from vllm.entrypoints.openai.serving_score import OpenAIServingScores
66 from vllm.entrypoints.openai.serving_tokenization import (
@@ -269,6 +271,10 @@ def base(request: Request) -> OpenAIServing:
271 return tokenization(request)
272
273
274 +def models(request: Request) -> OpenAIServingModels:
275 + return request.app.state.openai_serving_models
276 +
277 +
278 def chat(request: Request) -> Optional[OpenAIServingChat]:
279 return request.app.state.openai_serving_chat
280
@@ -336,10 +342,10 @@ async def detokenize(request: DetokenizeRequest, raw_request: Request):
342
343 @router.get("/v1/models")
344 async def show_available_models(raw_request: Request):
339 - handler = base(raw_request)
345 + handler = models(raw_request)
346
341 - models = await handler.show_available_models()
342 - return JSONResponse(content=models.model_dump())
347 + models_ = await handler.show_available_models()
348 + return JSONResponse(content=models_.model_dump())
349
350
351 @router.get("/version")
@@ -505,26 +511,22 @@ if envs.VLLM_ALLOW_RUNTIME_LORA_UPDATING:
511 @router.post("/v1/load_lora_adapter")
512 async def load_lora_adapter(request: LoadLoraAdapterRequest,
513 raw_request: Request):
508 - for route in [chat, completion, embedding]:
509 - handler = route(raw_request)
510 - if handler is not None:
511 - response = await handler.load_lora_adapter(request)
512 - if isinstance(response, ErrorResponse):
513 - return JSONResponse(content=response.model_dump(),
514 - status_code=response.code)
514 + handler = models(raw_request)
515 + response = await handler.load_lora_adapter(request)
516 + if isinstance(response, ErrorResponse):
517 + return JSONResponse(content=response.model_dump(),
518 + status_code=response.code)
519
520 return Response(status_code=200, content=response)
521
522 @router.post("/v1/unload_lora_adapter")
523 async def unload_lora_adapter(request: UnloadLoraAdapterRequest,
524 raw_request: Request):
521 - for route in [chat, completion, embedding]:
522 - handler = route(raw_request)
523 - if handler is not None:
524 - response = await handler.unload_lora_adapter(request)
525 - if isinstance(response, ErrorResponse):
526 - return JSONResponse(content=response.model_dump(),
527 - status_code=response.code)
525 + handler = models(raw_request)
526 + response = await handler.unload_lora_adapter(request)
527 + if isinstance(response, ErrorResponse):
528 + return JSONResponse(content=response.model_dump(),
529 + status_code=response.code)
530
531 return Response(status_code=200, content=response)
532
@@ -628,13 +630,18 @@ def init_app_state(
630 resolved_chat_template = load_chat_template(args.chat_template)
631 logger.info("Using supplied chat template:\n%s", resolved_chat_template)
632
633 + state.openai_serving_models = OpenAIServingModels(
634 + model_config=model_config,
635 + base_model_paths=base_model_paths,
636 + lora_modules=args.lora_modules,
637 + prompt_adapters=args.prompt_adapters,
638 + )
639 + # TODO: The chat template is now broken for lora adapters :(
640 state.openai_serving_chat = OpenAIServingChat(
641 engine_client,
642 model_config,
634 - base_model_paths,
643 + state.openai_serving_models,
644 args.response_role,
636 - lora_modules=args.lora_modules,
637 - prompt_adapters=args.prompt_adapters,
645 request_logger=request_logger,
646 chat_template=resolved_chat_template,
647 chat_template_content_format=args.chat_template_content_format,
@@ -646,16 +653,14 @@ def init_app_state(
653 state.openai_serving_completion = OpenAIServingCompletion(
654 engine_client,
655 model_config,
649 - base_model_paths,
650 - lora_modules=args.lora_modules,
651 - prompt_adapters=args.prompt_adapters,
656 + state.openai_serving_models,
657 request_logger=request_logger,
658 return_tokens_as_token_ids=args.return_tokens_as_token_ids,
659 ) if model_config.runner_type == "generate" else None
660 state.openai_serving_pooling = OpenAIServingPooling(
661 engine_client,
662 model_config,
658 - base_model_paths,
663 + state.openai_serving_models,
664 request_logger=request_logger,
665 chat_template=resolved_chat_template,
666 chat_template_content_format=args.chat_template_content_format,
@@ -663,7 +668,7 @@ def init_app_state(
668 state.openai_serving_embedding = OpenAIServingEmbedding(
669 engine_client,
670 model_config,
666 - base_model_paths,
671 + state.openai_serving_models,
672 request_logger=request_logger,
673 chat_template=resolved_chat_template,
674 chat_template_content_format=args.chat_template_content_format,
@@ -671,14 +676,13 @@ def init_app_state(
676 state.openai_serving_scores = OpenAIServingScores(
677 engine_client,
678 model_config,
674 - base_model_paths,
679 + state.openai_serving_models,
680 request_logger=request_logger
681 ) if model_config.task == "score" else None
682 state.openai_serving_tokenization = OpenAIServingTokenization(
683 engine_client,
684 model_config,
680 - base_model_paths,
681 - lora_modules=args.lora_modules,
685 + state.openai_serving_models,
686 request_logger=request_logger,
687 chat_template=resolved_chat_template,
688 chat_template_content_format=args.chat_template_content_format,
vllm/entrypoints/openai/cli_args.py
+1 -1
@@ -12,7 +12,7 @@ from typing import List, Optional, Sequence, Union, get_args
12 from vllm.engine.arg_utils import AsyncEngineArgs, nullable_str
13 from vllm.entrypoints.chat_utils import (ChatTemplateContentFormatOption,
14 validate_chat_template)
15 -from vllm.entrypoints.openai.serving_engine import (LoRAModulePath,
15 +from vllm.entrypoints.openai.serving_models import (LoRAModulePath,
16 PromptAdapterPath)
17 from vllm.entrypoints.openai.tool_parsers import ToolParserManager
18 from vllm.utils import FlexibleArgumentParser
vllm/entrypoints/openai/run_batch.py
+10 -5
@@ -20,7 +20,8 @@ from vllm.entrypoints.openai.protocol import (BatchRequestInput,
20 # yapf: enable
21 from vllm.entrypoints.openai.serving_chat import OpenAIServingChat
22 from vllm.entrypoints.openai.serving_embedding import OpenAIServingEmbedding
23 -from vllm.entrypoints.openai.serving_engine import BaseModelPath
23 +from vllm.entrypoints.openai.serving_models import (BaseModelPath,
24 + OpenAIServingModels)
25 from vllm.usage.usage_lib import UsageContext
26 from vllm.utils import FlexibleArgumentParser, random_uuid
27 from vllm.version import __version__ as VLLM_VERSION
@@ -213,13 +214,17 @@ async def main(args):
214 request_logger = RequestLogger(max_log_len=args.max_log_len)
215
216 # Create the openai serving objects.
217 + openai_serving_models = OpenAIServingModels(
218 + model_config=model_config,
219 + base_model_paths=base_model_paths,
220 + lora_modules=None,
221 + prompt_adapters=None,
222 + )
223 openai_serving_chat = OpenAIServingChat(
224 engine,
225 model_config,
219 - base_model_paths,
226 + openai_serving_models,
227 args.response_role,
221 - lora_modules=None,
222 - prompt_adapters=None,
228 request_logger=request_logger,
229 chat_template=None,
230 chat_template_content_format="auto",
@@ -228,7 +233,7 @@ async def main(args):
233 openai_serving_embedding = OpenAIServingEmbedding(
234 engine,
235 model_config,
231 - base_model_paths,
236 + openai_serving_models,
237 request_logger=request_logger,
238 chat_template=None,
239 chat_template_content_format="auto",
vllm/entrypoints/openai/serving_chat.py
+5 -11
@@ -21,10 +21,8 @@ from vllm.entrypoints.openai.protocol import (
21 ChatCompletionStreamResponse, ChatMessage, DeltaFunctionCall, DeltaMessage,
22 DeltaToolCall, ErrorResponse, FunctionCall, PromptTokenUsageInfo,
23 RequestResponseMetadata, ToolCall, UsageInfo)
24 -from vllm.entrypoints.openai.serving_engine import (BaseModelPath,
25 - LoRAModulePath,
26 - OpenAIServing,
27 - PromptAdapterPath)
24 +from vllm.entrypoints.openai.serving_engine import OpenAIServing
25 +from vllm.entrypoints.openai.serving_models import OpenAIServingModels
26 from vllm.entrypoints.openai.tool_parsers import ToolParser, ToolParserManager
27 from vllm.logger import init_logger
28 from vllm.outputs import CompletionOutput, RequestOutput
@@ -42,11 +40,9 @@ class OpenAIServingChat(OpenAIServing):
40 self,
41 engine_client: EngineClient,
42 model_config: ModelConfig,
45 - base_model_paths: List[BaseModelPath],
43 + models: OpenAIServingModels,
44 response_role: str,
45 *,
48 - lora_modules: Optional[List[LoRAModulePath]],
49 - prompt_adapters: Optional[List[PromptAdapterPath]],
46 request_logger: Optional[RequestLogger],
47 chat_template: Optional[str],
48 chat_template_content_format: ChatTemplateContentFormatOption,
@@ -57,9 +53,7 @@ class OpenAIServingChat(OpenAIServing):
53 ) -> None:
54 super().__init__(engine_client=engine_client,
55 model_config=model_config,
60 - base_model_paths=base_model_paths,
61 - lora_modules=lora_modules,
62 - prompt_adapters=prompt_adapters,
56 + models=models,
57 request_logger=request_logger,
58 return_tokens_as_token_ids=return_tokens_as_token_ids)
59
@@ -126,7 +120,7 @@ class OpenAIServingChat(OpenAIServing):
120 prompt_adapter_request,
121 ) = self._maybe_get_adapters(request)
122
129 - model_name = self._get_model_name(lora_request)
123 + model_name = self.models.model_name(lora_request)
124
125 tokenizer = await self.engine_client.get_tokenizer(lora_request)
126
vllm/entrypoints/openai/serving_completion.py
+5 -11
@@ -21,10 +21,8 @@ from vllm.entrypoints.openai.protocol import (CompletionLogProbs,
21 RequestResponseMetadata,
22 UsageInfo)
23 # yapf: enable
24 -from vllm.entrypoints.openai.serving_engine import (BaseModelPath,
25 - LoRAModulePath,
26 - OpenAIServing,
27 - PromptAdapterPath)
24 +from vllm.entrypoints.openai.serving_engine import OpenAIServing
25 +from vllm.entrypoints.openai.serving_models import OpenAIServingModels
26 from vllm.logger import init_logger
27 from vllm.outputs import RequestOutput
28 from vllm.sampling_params import BeamSearchParams, SamplingParams
@@ -41,18 +39,14 @@ class OpenAIServingCompletion(OpenAIServing):
39 self,
40 engine_client: EngineClient,
41 model_config: ModelConfig,
44 - base_model_paths: List[BaseModelPath],
42 + models: OpenAIServingModels,
43 *,
46 - lora_modules: Optional[List[LoRAModulePath]],
47 - prompt_adapters: Optional[List[PromptAdapterPath]],
44 request_logger: Optional[RequestLogger],
45 return_tokens_as_token_ids: bool = False,
46 ):
47 super().__init__(engine_client=engine_client,
48 model_config=model_config,
53 - base_model_paths=base_model_paths,
54 - lora_modules=lora_modules,
55 - prompt_adapters=prompt_adapters,
49 + models=models,
50 request_logger=request_logger,
51 return_tokens_as_token_ids=return_tokens_as_token_ids)
52 diff_sampling_param = self.model_config.get_diff_sampling_param()
@@ -170,7 +164,7 @@ class OpenAIServingCompletion(OpenAIServing):
164
165 result_generator = merge_async_iterators(*generators)
166
173 - model_name = self._get_model_name(lora_request)
167 + model_name = self.models.model_name(lora_request)
168 num_prompts = len(engine_prompts)
169
170 # Similar to the OpenAI API, when n != best_of, we do not stream the
vllm/entrypoints/openai/serving_embedding.py
+4 -5
@@ -16,7 +16,8 @@ from vllm.entrypoints.openai.protocol import (EmbeddingChatRequest,
16 EmbeddingResponse,
17 EmbeddingResponseData,
18 ErrorResponse, UsageInfo)
19 -from vllm.entrypoints.openai.serving_engine import BaseModelPath, OpenAIServing
19 +from vllm.entrypoints.openai.serving_engine import OpenAIServing
20 +from vllm.entrypoints.openai.serving_models import OpenAIServingModels
21 from vllm.logger import init_logger
22 from vllm.outputs import (EmbeddingOutput, EmbeddingRequestOutput,
23 PoolingRequestOutput)
@@ -46,7 +47,7 @@ class OpenAIServingEmbedding(OpenAIServing):
47 self,
48 engine_client: EngineClient,
49 model_config: ModelConfig,
49 - base_model_paths: List[BaseModelPath],
50 + models: OpenAIServingModels,
51 *,
52 request_logger: Optional[RequestLogger],
53 chat_template: Optional[str],
@@ -54,9 +55,7 @@ class OpenAIServingEmbedding(OpenAIServing):
55 ) -> None:
56 super().__init__(engine_client=engine_client,
57 model_config=model_config,
57 - base_model_paths=base_model_paths,
58 - lora_modules=None,
59 - prompt_adapters=None,
58 + models=models,
59 request_logger=request_logger)
60
61 self.chat_template = chat_template
vllm/entrypoints/openai/serving_engine.py
+13 -179
@@ -1,7 +1,5 @@
1 import json
2 -import pathlib
2 from concurrent.futures.thread import ThreadPoolExecutor
4 -from dataclasses import dataclass
3 from http import HTTPStatus
4 from typing import (Any, Callable, Dict, Iterable, Iterator, List, Mapping,
5 Optional, Sequence, Tuple, TypedDict, Union)
@@ -28,13 +26,10 @@ from vllm.entrypoints.openai.protocol import (ChatCompletionRequest,
26 DetokenizeRequest,
27 EmbeddingChatRequest,
28 EmbeddingCompletionRequest,
31 - ErrorResponse,
32 - LoadLoraAdapterRequest,
33 - ModelCard, ModelList,
34 - ModelPermission, ScoreRequest,
29 + ErrorResponse, ScoreRequest,
30 TokenizeChatRequest,
36 - TokenizeCompletionRequest,
37 - UnloadLoraAdapterRequest)
31 + TokenizeCompletionRequest)
32 +from vllm.entrypoints.openai.serving_models import OpenAIServingModels
33 from vllm.entrypoints.openai.tool_parsers import ToolParser
34 # yapf: enable
35 from vllm.inputs import TokensPrompt
@@ -48,30 +43,10 @@ from vllm.sequence import Logprob
43 from vllm.tracing import (contains_trace_headers, extract_trace_headers,
44 log_tracing_disabled_warning)
45 from vllm.transformers_utils.tokenizer import AnyTokenizer, MistralTokenizer
51 -from vllm.utils import AtomicCounter, is_list_of, make_async, random_uuid
46 +from vllm.utils import is_list_of, make_async, random_uuid
47
48 logger = init_logger(__name__)
49
55 -
56 -@dataclass
57 -class BaseModelPath:
58 - name: str
59 - model_path: str
60 -
61 -
62 -@dataclass
63 -class PromptAdapterPath:
64 - name: str
65 - local_path: str
66 -
67 -
68 -@dataclass
69 -class LoRAModulePath:
70 - name: str
71 - path: str
72 - base_model_name: Optional[str] = None
73 -
74 -
50 CompletionLikeRequest = Union[CompletionRequest, DetokenizeRequest,
51 EmbeddingCompletionRequest, ScoreRequest,
52 TokenizeCompletionRequest]
@@ -96,10 +71,8 @@ class OpenAIServing:
71 self,
72 engine_client: EngineClient,
73 model_config: ModelConfig,
99 - base_model_paths: List[BaseModelPath],
74 + models: OpenAIServingModels,
75 *,
101 - lora_modules: Optional[List[LoRAModulePath]],
102 - prompt_adapters: Optional[List[PromptAdapterPath]],
76 request_logger: Optional[RequestLogger],
77 return_tokens_as_token_ids: bool = False,
78 ):
@@ -109,35 +82,7 @@ class OpenAIServing:
82 self.model_config = model_config
83 self.max_model_len = model_config.max_model_len
84
112 - self.base_model_paths = base_model_paths
113 -
114 - self.lora_id_counter = AtomicCounter(0)
115 - self.lora_requests = []
116 - if lora_modules is not None:
117 - self.lora_requests = [
118 - LoRARequest(lora_name=lora.name,
119 - lora_int_id=i,
120 - lora_path=lora.path,
121 - base_model_name=lora.base_model_name
122 - if lora.base_model_name
123 - and self._is_model_supported(lora.base_model_name)
124 - else self.base_model_paths[0].name)
125 - for i, lora in enumerate(lora_modules, start=1)
126 - ]
127 -
128 - self.prompt_adapter_requests = []
129 - if prompt_adapters is not None:
130 - for i, prompt_adapter in enumerate(prompt_adapters, start=1):
131 - with pathlib.Path(prompt_adapter.local_path,
132 - "adapter_config.json").open() as f:
133 - adapter_config = json.load(f)
134 - num_virtual_tokens = adapter_config["num_virtual_tokens"]
135 - self.prompt_adapter_requests.append(
136 - PromptAdapterRequest(
137 - prompt_adapter_name=prompt_adapter.name,
138 - prompt_adapter_id=i,
139 - prompt_adapter_local_path=prompt_adapter.local_path,
140 - prompt_adapter_num_virtual_tokens=num_virtual_tokens))
85 + self.models = models
86
87 self.request_logger = request_logger
88 self.return_tokens_as_token_ids = return_tokens_as_token_ids
@@ -150,33 +95,6 @@ class OpenAIServing:
95 self._tokenize_prompt_input_or_inputs,
96 executor=self._tokenizer_executor)
97
153 - async def show_available_models(self) -> ModelList:
154 - """Show available models. Right now we only have one model."""
155 - model_cards = [
156 - ModelCard(id=base_model.name,
157 - max_model_len=self.max_model_len,
158 - root=base_model.model_path,
159 - permission=[ModelPermission()])
160 - for base_model in self.base_model_paths
161 - ]
162 - lora_cards = [
163 - ModelCard(id=lora.lora_name,
164 - root=lora.local_path,
165 - parent=lora.base_model_name if lora.base_model_name else
166 - self.base_model_paths[0].name,
167 - permission=[ModelPermission()])
168 - for lora in self.lora_requests
169 - ]
170 - prompt_adapter_cards = [
171 - ModelCard(id=prompt_adapter.prompt_adapter_name,
172 - root=self.base_model_paths[0].name,
173 - permission=[ModelPermission()])
174 - for prompt_adapter in self.prompt_adapter_requests
175 - ]
176 - model_cards.extend(lora_cards)
177 - model_cards.extend(prompt_adapter_cards)
178 - return ModelList(data=model_cards)
179 -
98 def create_error_response(
99 self,
100 message: str,
@@ -205,11 +123,13 @@ class OpenAIServing:
123 ) -> Optional[ErrorResponse]:
124 if self._is_model_supported(request.model):
125 return None
208 - if request.model in [lora.lora_name for lora in self.lora_requests]:
126 + if request.model in [
127 + lora.lora_name for lora in self.models.lora_requests
128 + ]:
129 return None
130 if request.model in [
131 prompt_adapter.prompt_adapter_name
212 - for prompt_adapter in self.prompt_adapter_requests
132 + for prompt_adapter in self.models.prompt_adapter_requests
133 ]:
134 return None
135 return self.create_error_response(
@@ -223,10 +143,10 @@ class OpenAIServing:
143 None, PromptAdapterRequest]]:
144 if self._is_model_supported(request.model):
145 return None, None
226 - for lora in self.lora_requests:
146 + for lora in self.models.lora_requests:
147 if request.model == lora.lora_name:
148 return lora, None
229 - for prompt_adapter in self.prompt_adapter_requests:
149 + for prompt_adapter in self.models.prompt_adapter_requests:
150 if request.model == prompt_adapter.prompt_adapter_name:
151 return None, prompt_adapter
152 # if _check_model has been called earlier, this will be unreachable
@@ -588,91 +508,5 @@ class OpenAIServing:
508 return logprob.decoded_token
509 return tokenizer.decode(token_id)
510
591 - async def _check_load_lora_adapter_request(
592 - self, request: LoadLoraAdapterRequest) -> Optional[ErrorResponse]:
593 - # Check if both 'lora_name' and 'lora_path' are provided
594 - if not request.lora_name or not request.lora_path:
595 - return self.create_error_response(
596 - message="Both 'lora_name' and 'lora_path' must be provided.",
597 - err_type="InvalidUserInput",
598 - status_code=HTTPStatus.BAD_REQUEST)
599 -
600 - # Check if the lora adapter with the given name already exists
601 - if any(lora_request.lora_name == request.lora_name
602 - for lora_request in self.lora_requests):
603 - return self.create_error_response(
604 - message=
605 - f"The lora adapter '{request.lora_name}' has already been"
606 - "loaded.",
607 - err_type="InvalidUserInput",
608 - status_code=HTTPStatus.BAD_REQUEST)
609 -
610 - return None
611 -
612 - async def _check_unload_lora_adapter_request(
613 - self,
614 - request: UnloadLoraAdapterRequest) -> Optional[ErrorResponse]:
615 - # Check if either 'lora_name' or 'lora_int_id' is provided
616 - if not request.lora_name and not request.lora_int_id:
617 - return self.create_error_response(
618 - message=
619 - "either 'lora_name' and 'lora_int_id' needs to be provided.",
620 - err_type="InvalidUserInput",
621 - status_code=HTTPStatus.BAD_REQUEST)
622 -
623 - # Check if the lora adapter with the given name exists
624 - if not any(lora_request.lora_name == request.lora_name
625 - for lora_request in self.lora_requests):
626 - return self.create_error_response(
627 - message=
628 - f"The lora adapter '{request.lora_name}' cannot be found.",
629 - err_type="InvalidUserInput",
630 - status_code=HTTPStatus.BAD_REQUEST)
631 -
632 - return None
633 -
634 - async def load_lora_adapter(
635 - self,
636 - request: LoadLoraAdapterRequest) -> Union[ErrorResponse, str]:
637 - error_check_ret = await self._check_load_lora_adapter_request(request)
638 - if error_check_ret is not None:
639 - return error_check_ret
640 -
641 - lora_name, lora_path = request.lora_name, request.lora_path
642 - unique_id = self.lora_id_counter.inc(1)
643 - self.lora_requests.append(
644 - LoRARequest(lora_name=lora_name,
645 - lora_int_id=unique_id,
646 - lora_path=lora_path))
647 - return f"Success: LoRA adapter '{lora_name}' added successfully."
648 -
649 - async def unload_lora_adapter(
650 - self,
651 - request: UnloadLoraAdapterRequest) -> Union[ErrorResponse, str]:
652 - error_check_ret = await self._check_unload_lora_adapter_request(request
653 - )
654 - if error_check_ret is not None:
655 - return error_check_ret
656 -
657 - lora_name = request.lora_name
658 - self.lora_requests = [
659 - lora_request for lora_request in self.lora_requests
660 - if lora_request.lora_name != lora_name
661 - ]
662 - return f"Success: LoRA adapter '{lora_name}' removed successfully."
663 -
511 def _is_model_supported(self, model_name):
665 - return any(model.name == model_name for model in self.base_model_paths)
666 -
667 - def _get_model_name(self, lora: Optional[LoRARequest]):
668 - """
669 - Returns the appropriate model name depending on the availability
670 - and support of the LoRA or base model.
671 - Parameters:
672 - - lora: LoRARequest that contain a base_model_name.
673 - Returns:
674 - - str: The name of the base model or the first available model path.
675 - """
676 - if lora is not None:
677 - return lora.lora_name
678 - return self.base_model_paths[0].name
512 + return self.models.is_base_model(model_name)
vllm/entrypoints/openai/serving_models.py new
+210
@@ -0,0 +1,210 @@
1 +import json
2 +import pathlib
3 +from dataclasses import dataclass
4 +from http import HTTPStatus
5 +from typing import List, Optional, Union
6 +
7 +from vllm.config import ModelConfig
8 +from vllm.entrypoints.openai.protocol import (ErrorResponse,
9 + LoadLoraAdapterRequest,
10 + ModelCard, ModelList,
11 + ModelPermission,
12 + UnloadLoraAdapterRequest)
13 +from vllm.lora.request import LoRARequest
14 +from vllm.prompt_adapter.request import PromptAdapterRequest
15 +from vllm.utils import AtomicCounter
16 +
17 +
18 +@dataclass
19 +class BaseModelPath:
20 + name: str
21 + model_path: str
22 +
23 +
24 +@dataclass
25 +class PromptAdapterPath:
26 + name: str
27 + local_path: str
28 +
29 +
30 +@dataclass
31 +class LoRAModulePath:
32 + name: str
33 + path: str
34 + base_model_name: Optional[str] = None
35 +
36 +
37 +class OpenAIServingModels:
38 + """Shared instance to hold data about the loaded base model(s) and adapters.
39 +
40 + Handles the routes:
41 + - /v1/models
42 + - /v1/load_lora_adapter
43 + - /v1/unload_lora_adapter
44 + """
45 +
46 + def __init__(
47 + self,
48 + model_config: ModelConfig,
49 + base_model_paths: List[BaseModelPath],
50 + *,
51 + lora_modules: Optional[List[LoRAModulePath]] = None,
52 + prompt_adapters: Optional[List[PromptAdapterPath]] = None,
53 + ):
54 + super().__init__()
55 +
56 + self.base_model_paths = base_model_paths
57 + self.max_model_len = model_config.max_model_len
58 +
59 + self.lora_id_counter = AtomicCounter(0)
60 + self.lora_requests = []
61 + if lora_modules is not None:
62 + self.lora_requests = [
63 + LoRARequest(lora_name=lora.name,
64 + lora_int_id=i,
65 + lora_path=lora.path,
66 + base_model_name=lora.base_model_name
67 + if lora.base_model_name
68 + and self.is_base_model(lora.base_model_name) else
69 + self.base_model_paths[0].name)
70 + for i, lora in enumerate(lora_modules, start=1)
71 + ]
72 +
73 + self.prompt_adapter_requests = []
74 + if prompt_adapters is not None:
75 + for i, prompt_adapter in enumerate(prompt_adapters, start=1):
76 + with pathlib.Path(prompt_adapter.local_path,
77 + "adapter_config.json").open() as f:
78 + adapter_config = json.load(f)
79 + num_virtual_tokens = adapter_config["num_virtual_tokens"]
80 + self.prompt_adapter_requests.append(
81 + PromptAdapterRequest(
82 + prompt_adapter_name=prompt_adapter.name,
83 + prompt_adapter_id=i,
84 + prompt_adapter_local_path=prompt_adapter.local_path,
85 + prompt_adapter_num_virtual_tokens=num_virtual_tokens))
86 +
87 + def is_base_model(self, model_name):
88 + return any(model.name == model_name for model in self.base_model_paths)
89 +
90 + def model_name(self, lora_request: Optional[LoRARequest] = None) -> str:
91 + """Returns the appropriate model name depending on the availability
92 + and support of the LoRA or base model.
93 + Parameters:
94 + - lora: LoRARequest that contain a base_model_name.
95 + Returns:
96 + - str: The name of the base model or the first available model path.
97 + """
98 + if lora_request is not None:
99 + return lora_request.lora_name
100 + return self.base_model_paths[0].name
101 +
102 + async def show_available_models(self) -> ModelList:
103 + """Show available models. This includes the base model and all
104 + adapters"""
105 + model_cards = [
106 + ModelCard(id=base_model.name,
107 + max_model_len=self.max_model_len,
108 + root=base_model.model_path,
109 + permission=[ModelPermission()])
110 + for base_model in self.base_model_paths
111 + ]
112 + lora_cards = [
113 + ModelCard(id=lora.lora_name,
114 + root=lora.local_path,
115 + parent=lora.base_model_name if lora.base_model_name else
116 + self.base_model_paths[0].name,
117 + permission=[ModelPermission()])
118 + for lora in self.lora_requests
119 + ]
120 + prompt_adapter_cards = [
121 + ModelCard(id=prompt_adapter.prompt_adapter_name,
122 + root=self.base_model_paths[0].name,
123 + permission=[ModelPermission()])
124 + for prompt_adapter in self.prompt_adapter_requests
125 + ]
126 + model_cards.extend(lora_cards)
127 + model_cards.extend(prompt_adapter_cards)
128 + return ModelList(data=model_cards)
129 +
130 + async def load_lora_adapter(
131 + self,
132 + request: LoadLoraAdapterRequest) -> Union[ErrorResponse, str]:
133 + error_check_ret = await self._check_load_lora_adapter_request(request)
134 + if error_check_ret is not None:
135 + return error_check_ret
136 +
137 + lora_name, lora_path = request.lora_name, request.lora_path
138 + unique_id = self.lora_id_counter.inc(1)
139 + self.lora_requests.append(
140 + LoRARequest(lora_name=lora_name,
141 + lora_int_id=unique_id,
142 + lora_path=lora_path))
143 + return f"Success: LoRA adapter '{lora_name}' added successfully."
144 +
145 + async def unload_lora_adapter(
146 + self,
147 + request: UnloadLoraAdapterRequest) -> Union[ErrorResponse, str]:
148 + error_check_ret = await self._check_unload_lora_adapter_request(request
149 + )
150 + if error_check_ret is not None:
151 + return error_check_ret
152 +
153 + lora_name = request.lora_name
154 + self.lora_requests = [
155 + lora_request for lora_request in self.lora_requests
156 + if lora_request.lora_name != lora_name
157 + ]
158 + return f"Success: LoRA adapter '{lora_name}' removed successfully."
159 +
160 + async def _check_load_lora_adapter_request(
161 + self, request: LoadLoraAdapterRequest) -> Optional[ErrorResponse]:
162 + # Check if both 'lora_name' and 'lora_path' are provided
163 + if not request.lora_name or not request.lora_path:
164 + return create_error_response(
165 + message="Both 'lora_name' and 'lora_path' must be provided.",
166 + err_type="InvalidUserInput",
167 + status_code=HTTPStatus.BAD_REQUEST)
168 +
169 + # Check if the lora adapter with the given name already exists
170 + if any(lora_request.lora_name == request.lora_name
171 + for lora_request in self.lora_requests):
172 + return create_error_response(
173 + message=
174 + f"The lora adapter '{request.lora_name}' has already been"
175 + "loaded.",
176 + err_type="InvalidUserInput",
177 + status_code=HTTPStatus.BAD_REQUEST)
178 +
179 + return None
180 +
181 + async def _check_unload_lora_adapter_request(
182 + self,
183 + request: UnloadLoraAdapterRequest) -> Optional[ErrorResponse]:
184 + # Check if either 'lora_name' or 'lora_int_id' is provided
185 + if not request.lora_name and not request.lora_int_id:
186 + return create_error_response(
187 + message=
188 + "either 'lora_name' and 'lora_int_id' needs to be provided.",
189 + err_type="InvalidUserInput",
190 + status_code=HTTPStatus.BAD_REQUEST)
191 +
192 + # Check if the lora adapter with the given name exists
193 + if not any(lora_request.lora_name == request.lora_name
194 + for lora_request in self.lora_requests):
195 + return create_error_response(
196 + message=
197 + f"The lora adapter '{request.lora_name}' cannot be found.",
198 + err_type="InvalidUserInput",
199 + status_code=HTTPStatus.BAD_REQUEST)
200 +
201 + return None
202 +
203 +
204 +def create_error_response(
205 + message: str,
206 + err_type: str = "BadRequestError",
207 + status_code: HTTPStatus = HTTPStatus.BAD_REQUEST) -> ErrorResponse:
208 + return ErrorResponse(message=message,
209 + type=err_type,
210 + code=status_code.value)
vllm/entrypoints/openai/serving_pooling.py
+4 -5
@@ -15,7 +15,8 @@ from vllm.entrypoints.openai.protocol import (ErrorResponse,
15 PoolingChatRequest,
16 PoolingRequest, PoolingResponse,
17 PoolingResponseData, UsageInfo)
18 -from vllm.entrypoints.openai.serving_engine import BaseModelPath, OpenAIServing
18 +from vllm.entrypoints.openai.serving_engine import OpenAIServing
19 +from vllm.entrypoints.openai.serving_models import OpenAIServingModels
20 from vllm.logger import init_logger
21 from vllm.outputs import PoolingOutput, PoolingRequestOutput
22 from vllm.utils import merge_async_iterators
@@ -44,7 +45,7 @@ class OpenAIServingPooling(OpenAIServing):
45 self,
46 engine_client: EngineClient,
47 model_config: ModelConfig,
47 - base_model_paths: List[BaseModelPath],
48 + models: OpenAIServingModels,
49 *,
50 request_logger: Optional[RequestLogger],
51 chat_template: Optional[str],
@@ -52,9 +53,7 @@ class OpenAIServingPooling(OpenAIServing):
53 ) -> None:
54 super().__init__(engine_client=engine_client,
55 model_config=model_config,
55 - base_model_paths=base_model_paths,
56 - lora_modules=None,
57 - prompt_adapters=None,
56 + models=models,
57 request_logger=request_logger)
58
59 self.chat_template = chat_template
vllm/entrypoints/openai/serving_score.py
+4 -5
@@ -10,7 +10,8 @@ from vllm.entrypoints.logger import RequestLogger
10 from vllm.entrypoints.openai.protocol import (ErrorResponse, ScoreRequest,
11 ScoreResponse, ScoreResponseData,
12 UsageInfo)
13 -from vllm.entrypoints.openai.serving_engine import BaseModelPath, OpenAIServing
13 +from vllm.entrypoints.openai.serving_engine import OpenAIServing
14 +from vllm.entrypoints.openai.serving_models import OpenAIServingModels
15 from vllm.inputs.data import TokensPrompt
16 from vllm.logger import init_logger
17 from vllm.outputs import PoolingRequestOutput, ScoringRequestOutput
@@ -50,15 +51,13 @@ class OpenAIServingScores(OpenAIServing):
51 self,
52 engine_client: EngineClient,
53 model_config: ModelConfig,
53 - base_model_paths: List[BaseModelPath],
54 + models: OpenAIServingModels,
55 *,
56 request_logger: Optional[RequestLogger],
57 ) -> None:
58 super().__init__(engine_client=engine_client,
59 model_config=model_config,
59 - base_model_paths=base_model_paths,
60 - lora_modules=None,
61 - prompt_adapters=None,
60 + models=models,
61 request_logger=request_logger)
62
63 async def create_score(
vllm/entrypoints/openai/serving_tokenization.py
+4 -8
@@ -15,9 +15,8 @@ from vllm.entrypoints.openai.protocol import (DetokenizeRequest,
15 TokenizeRequest,
16 TokenizeResponse)
17 # yapf: enable
18 -from vllm.entrypoints.openai.serving_engine import (BaseModelPath,
19 - LoRAModulePath,
20 - OpenAIServing)
18 +from vllm.entrypoints.openai.serving_engine import OpenAIServing
19 +from vllm.entrypoints.openai.serving_models import OpenAIServingModels
20 from vllm.logger import init_logger
21
22 logger = init_logger(__name__)
@@ -29,18 +28,15 @@ class OpenAIServingTokenization(OpenAIServing):
28 self,
29 engine_client: EngineClient,
30 model_config: ModelConfig,
32 - base_model_paths: List[BaseModelPath],
31 + models: OpenAIServingModels,
32 *,
34 - lora_modules: Optional[List[LoRAModulePath]],
33 request_logger: Optional[RequestLogger],
34 chat_template: Optional[str],
35 chat_template_content_format: ChatTemplateContentFormatOption,
36 ) -> None:
37 super().__init__(engine_client=engine_client,
38 model_config=model_config,
41 - base_model_paths=base_model_paths,
42 - lora_modules=lora_modules,
43 - prompt_adapters=None,
39 + models=models,
40 request_logger=request_logger)
41
42 self.chat_template = chat_template