A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 03:15 UTC
[Bugfix][Refactor] Unify model management in frontend (#11660)
Signed-off-by: Joe Runde <Joseph.Runde@ibm.com>
Joe Runde committed
Dec 31, 2024 at 18:21 UTC
4db72e57f6e8da5e78285e9868e9327167bea973
15 files changed
+365
-307
tests/entrypoints/openai/test_cli_args.py
+1
-1
@@ -4,7 +4,7 @@ import pytest
4
5
from vllm.entrypoints.openai.cli_args import (make_arg_parser,
6
validate_parsed_serve_args)
7
-from vllm.entrypoints.openai.serving_engine import LoRAModulePath
7
+from vllm.entrypoints.openai.serving_models import LoRAModulePath
8
from vllm.utils import FlexibleArgumentParser
9
10
from ...utils import VLLM_PATH
tests/entrypoints/openai/test_lora_lineage.py
+29
-3
@@ -55,7 +55,10 @@ def server_with_lora_modules_json(zephyr_lora_files):
55
"64",
56
]
57
58
- with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
58
+ # Enable the /v1/load_lora_adapter endpoint
59
+ envs = {"VLLM_ALLOW_RUNTIME_LORA_UPDATING": "True"}
60
+
61
+ with RemoteOpenAIServer(MODEL_NAME, args, env_dict=envs) as remote_server:
62
yield remote_server
63
64
@@ -67,8 +70,8 @@ async def client_for_lora_lineage(server_with_lora_modules_json):
70
71
72
@pytest.mark.asyncio
70
-async def test_check_lora_lineage(client_for_lora_lineage: openai.AsyncOpenAI,
71
- zephyr_lora_files):
73
+async def test_static_lora_lineage(client_for_lora_lineage: openai.AsyncOpenAI,
74
+ zephyr_lora_files):
75
models = await client_for_lora_lineage.models.list()
76
models = models.data
77
served_model = models[0]
@@ -81,3 +84,26 @@ async def test_check_lora_lineage(client_for_lora_lineage: openai.AsyncOpenAI,
84
assert all(lora_model.parent == MODEL_NAME for lora_model in lora_models)
85
assert lora_models[0].id == "zephyr-lora"
86
assert lora_models[1].id == "zephyr-lora2"
87
+
88
+
89
+@pytest.mark.asyncio
90
+async def test_dynamic_lora_lineage(
91
+ client_for_lora_lineage: openai.AsyncOpenAI, zephyr_lora_files):
92
+
93
+ response = await client_for_lora_lineage.post("load_lora_adapter",
94
+ cast_to=str,
95
+ body={
96
+ "lora_name":
97
+ "zephyr-lora-3",
98
+ "lora_path":
99
+ zephyr_lora_files
100
+ })
101
+ # Ensure adapter loads before querying /models
102
+ assert "success" in response
103
+
104
+ models = await client_for_lora_lineage.models.list()
105
+ models = models.data
106
+ dynamic_lora_model = models[-1]
107
+ assert dynamic_lora_model.root == zephyr_lora_files
108
+ assert dynamic_lora_model.parent == MODEL_NAME
109
+ assert dynamic_lora_model.id == "zephyr-lora-3"
tests/entrypoints/openai/test_serving_chat.py
+10
-10
@@ -8,7 +8,8 @@ from vllm.config import MultiModalConfig
8
from vllm.engine.multiprocessing.client import MQLLMEngineClient
9
from vllm.entrypoints.openai.protocol import ChatCompletionRequest
10
from vllm.entrypoints.openai.serving_chat import OpenAIServingChat
11
-from vllm.entrypoints.openai.serving_engine import BaseModelPath
11
+from vllm.entrypoints.openai.serving_models import (BaseModelPath,
12
+ OpenAIServingModels)
13
from vllm.transformers_utils.tokenizer import get_tokenizer
14
15
MODEL_NAME = "openai-community/gpt2"
@@ -50,14 +51,13 @@ async def _async_serving_chat_init():
51
engine = MockEngine()
52
model_config = await engine.get_model_config()
53
54
+ models = OpenAIServingModels(model_config, BASE_MODEL_PATHS)
55
serving_completion = OpenAIServingChat(engine,
56
model_config,
55
- BASE_MODEL_PATHS,
57
+ models,
58
response_role="assistant",
59
chat_template=CHAT_TEMPLATE,
60
chat_template_content_format="auto",
59
- lora_modules=None,
60
- prompt_adapters=None,
61
request_logger=None)
62
return serving_completion
63
@@ -72,14 +72,14 @@ def test_serving_chat_should_set_correct_max_tokens():
72
mock_engine.get_tokenizer.return_value = get_tokenizer(MODEL_NAME)
73
mock_engine.errored = False
74
75
+ models = OpenAIServingModels(base_model_paths=BASE_MODEL_PATHS,
76
+ model_config=MockModelConfig())
77
serving_chat = OpenAIServingChat(mock_engine,
78
MockModelConfig(),
77
- BASE_MODEL_PATHS,
79
+ models,
80
response_role="assistant",
81
chat_template=CHAT_TEMPLATE,
82
chat_template_content_format="auto",
81
- lora_modules=None,
82
- prompt_adapters=None,
83
request_logger=None)
84
req = ChatCompletionRequest(
85
model=MODEL_NAME,
@@ -115,14 +115,14 @@ def test_serving_chat_could_load_correct_generation_config():
115
mock_engine.errored = False
116
117
# Initialize the serving chat
118
+ models = OpenAIServingModels(base_model_paths=BASE_MODEL_PATHS,
119
+ model_config=mock_model_config)
120
serving_chat = OpenAIServingChat(mock_engine,
121
mock_model_config,
120
- BASE_MODEL_PATHS,
122
+ models,
123
response_role="assistant",
124
chat_template=CHAT_TEMPLATE,
125
chat_template_content_format="auto",
124
- lora_modules=None,
125
- prompt_adapters=None,
126
request_logger=None)
127
req = ChatCompletionRequest(
128
model=MODEL_NAME,
tests/entrypoints/openai/test_serving_models.py
renamed
+32
-34
@@ -4,11 +4,11 @@ from unittest.mock import MagicMock
4
import pytest
5
6
from vllm.config import ModelConfig
7
-from vllm.engine.protocol import EngineClient
7
from vllm.entrypoints.openai.protocol import (ErrorResponse,
8
LoadLoraAdapterRequest,
9
UnloadLoraAdapterRequest)
11
-from vllm.entrypoints.openai.serving_engine import BaseModelPath, OpenAIServing
10
+from vllm.entrypoints.openai.serving_models import (BaseModelPath,
11
+ OpenAIServingModels)
12
from vllm.lora.request import LoRARequest
13
14
MODEL_NAME = "meta-llama/Llama-2-7b"
@@ -19,47 +19,45 @@ LORA_UNLOADING_SUCCESS_MESSAGE = (
19
"Success: LoRA adapter '{lora_name}' removed successfully.")
20
21
22
-async def _async_serving_engine_init():
23
- mock_engine_client = MagicMock(spec=EngineClient)
22
+async def _async_serving_models_init() -> OpenAIServingModels:
23
mock_model_config = MagicMock(spec=ModelConfig)
24
# Set the max_model_len attribute to avoid missing attribute
25
mock_model_config.max_model_len = 2048
26
28
- serving_engine = OpenAIServing(mock_engine_client,
29
- mock_model_config,
30
- BASE_MODEL_PATHS,
31
- lora_modules=None,
32
- prompt_adapters=None,
33
- request_logger=None)
34
- return serving_engine
27
+ serving_models = OpenAIServingModels(base_model_paths=BASE_MODEL_PATHS,
28
+ model_config=mock_model_config,
29
+ lora_modules=None,
30
+ prompt_adapters=None)
31
+
32
+ return serving_models
33
34
35
@pytest.mark.asyncio
36
async def test_serving_model_name():
39
- serving_engine = await _async_serving_engine_init()
40
- assert serving_engine._get_model_name(None) == MODEL_NAME
37
+ serving_models = await _async_serving_models_init()
38
+ assert serving_models.model_name(None) == MODEL_NAME
39
request = LoRARequest(lora_name="adapter",
40
lora_path="/path/to/adapter2",
41
lora_int_id=1)
44
- assert serving_engine._get_model_name(request) == request.lora_name
42
+ assert serving_models.model_name(request) == request.lora_name
43
44
45
@pytest.mark.asyncio
46
async def test_load_lora_adapter_success():
49
- serving_engine = await _async_serving_engine_init()
47
+ serving_models = await _async_serving_models_init()
48
request = LoadLoraAdapterRequest(lora_name="adapter",
49
lora_path="/path/to/adapter2")
52
- response = await serving_engine.load_lora_adapter(request)
50
+ response = await serving_models.load_lora_adapter(request)
51
assert response == LORA_LOADING_SUCCESS_MESSAGE.format(lora_name='adapter')
54
- assert len(serving_engine.lora_requests) == 1
55
- assert serving_engine.lora_requests[0].lora_name == "adapter"
52
+ assert len(serving_models.lora_requests) == 1
53
+ assert serving_models.lora_requests[0].lora_name == "adapter"
54
55
56
@pytest.mark.asyncio
57
async def test_load_lora_adapter_missing_fields():
60
- serving_engine = await _async_serving_engine_init()
58
+ serving_models = await _async_serving_models_init()
59
request = LoadLoraAdapterRequest(lora_name="", lora_path="")
62
- response = await serving_engine.load_lora_adapter(request)
60
+ response = await serving_models.load_lora_adapter(request)
61
assert isinstance(response, ErrorResponse)
62
assert response.type == "InvalidUserInput"
63
assert response.code == HTTPStatus.BAD_REQUEST
@@ -67,43 +65,43 @@ async def test_load_lora_adapter_missing_fields():
65
66
@pytest.mark.asyncio
67
async def test_load_lora_adapter_duplicate():
70
- serving_engine = await _async_serving_engine_init()
68
+ serving_models = await _async_serving_models_init()
69
request = LoadLoraAdapterRequest(lora_name="adapter1",
70
lora_path="/path/to/adapter1")
73
- response = await serving_engine.load_lora_adapter(request)
71
+ response = await serving_models.load_lora_adapter(request)
72
assert response == LORA_LOADING_SUCCESS_MESSAGE.format(
73
lora_name='adapter1')
76
- assert len(serving_engine.lora_requests) == 1
74
+ assert len(serving_models.lora_requests) == 1
75
76
request = LoadLoraAdapterRequest(lora_name="adapter1",
77
lora_path="/path/to/adapter1")
80
- response = await serving_engine.load_lora_adapter(request)
78
+ response = await serving_models.load_lora_adapter(request)
79
assert isinstance(response, ErrorResponse)
80
assert response.type == "InvalidUserInput"
81
assert response.code == HTTPStatus.BAD_REQUEST
84
- assert len(serving_engine.lora_requests) == 1
82
+ assert len(serving_models.lora_requests) == 1
83
84
85
@pytest.mark.asyncio
86
async def test_unload_lora_adapter_success():
89
- serving_engine = await _async_serving_engine_init()
87
+ serving_models = await _async_serving_models_init()
88
request = LoadLoraAdapterRequest(lora_name="adapter1",
89
lora_path="/path/to/adapter1")
92
- response = await serving_engine.load_lora_adapter(request)
93
- assert len(serving_engine.lora_requests) == 1
90
+ response = await serving_models.load_lora_adapter(request)
91
+ assert len(serving_models.lora_requests) == 1
92
93
request = UnloadLoraAdapterRequest(lora_name="adapter1")
96
- response = await serving_engine.unload_lora_adapter(request)
94
+ response = await serving_models.unload_lora_adapter(request)
95
assert response == LORA_UNLOADING_SUCCESS_MESSAGE.format(
96
lora_name='adapter1')
99
- assert len(serving_engine.lora_requests) == 0
97
+ assert len(serving_models.lora_requests) == 0
98
99
100
@pytest.mark.asyncio
101
async def test_unload_lora_adapter_missing_fields():
104
- serving_engine = await _async_serving_engine_init()
102
+ serving_models = await _async_serving_models_init()
103
request = UnloadLoraAdapterRequest(lora_name="", lora_int_id=None)
106
- response = await serving_engine.unload_lora_adapter(request)
104
+ response = await serving_models.unload_lora_adapter(request)
105
assert isinstance(response, ErrorResponse)
106
assert response.type == "InvalidUserInput"
107
assert response.code == HTTPStatus.BAD_REQUEST
@@ -111,9 +109,9 @@ async def test_unload_lora_adapter_missing_fields():
109
110
@pytest.mark.asyncio
111
async def test_unload_lora_adapter_not_found():
114
- serving_engine = await _async_serving_engine_init()
112
+ serving_models = await _async_serving_models_init()
113
request = UnloadLoraAdapterRequest(lora_name="nonexistent_adapter")
116
- response = await serving_engine.unload_lora_adapter(request)
114
+ response = await serving_models.unload_lora_adapter(request)
115
assert isinstance(response, ErrorResponse)
116
assert response.type == "InvalidUserInput"
117
assert response.code == HTTPStatus.BAD_REQUEST
vllm/entrypoints/openai/api_server.py
+33
-29
@@ -58,7 +58,9 @@ from vllm.entrypoints.openai.protocol import (ChatCompletionRequest,
58
from vllm.entrypoints.openai.serving_chat import OpenAIServingChat
59
from vllm.entrypoints.openai.serving_completion import OpenAIServingCompletion
60
from vllm.entrypoints.openai.serving_embedding import OpenAIServingEmbedding
61
-from vllm.entrypoints.openai.serving_engine import BaseModelPath, OpenAIServing
61
+from vllm.entrypoints.openai.serving_engine import OpenAIServing
62
+from vllm.entrypoints.openai.serving_models import (BaseModelPath,
63
+ OpenAIServingModels)
64
from vllm.entrypoints.openai.serving_pooling import OpenAIServingPooling
65
from vllm.entrypoints.openai.serving_score import OpenAIServingScores
66
from vllm.entrypoints.openai.serving_tokenization import (
@@ -269,6 +271,10 @@ def base(request: Request) -> OpenAIServing:
271
return tokenization(request)
272
273
274
+def models(request: Request) -> OpenAIServingModels:
275
+ return request.app.state.openai_serving_models
276
+
277
+
278
def chat(request: Request) -> Optional[OpenAIServingChat]:
279
return request.app.state.openai_serving_chat
280
@@ -336,10 +342,10 @@ async def detokenize(request: DetokenizeRequest, raw_request: Request):
342
343
@router.get("/v1/models")
344
async def show_available_models(raw_request: Request):
339
- handler = base(raw_request)
345
+ handler = models(raw_request)
346
341
- models = await handler.show_available_models()
342
- return JSONResponse(content=models.model_dump())
347
+ models_ = await handler.show_available_models()
348
+ return JSONResponse(content=models_.model_dump())
349
350
351
@router.get("/version")
@@ -505,26 +511,22 @@ if envs.VLLM_ALLOW_RUNTIME_LORA_UPDATING:
511
@router.post("/v1/load_lora_adapter")
512
async def load_lora_adapter(request: LoadLoraAdapterRequest,
513
raw_request: Request):
508
- for route in [chat, completion, embedding]:
509
- handler = route(raw_request)
510
- if handler is not None:
511
- response = await handler.load_lora_adapter(request)
512
- if isinstance(response, ErrorResponse):
513
- return JSONResponse(content=response.model_dump(),
514
- status_code=response.code)
514
+ handler = models(raw_request)
515
+ response = await handler.load_lora_adapter(request)
516
+ if isinstance(response, ErrorResponse):
517
+ return JSONResponse(content=response.model_dump(),
518
+ status_code=response.code)
519
520
return Response(status_code=200, content=response)
521
522
@router.post("/v1/unload_lora_adapter")
523
async def unload_lora_adapter(request: UnloadLoraAdapterRequest,
524
raw_request: Request):
521
- for route in [chat, completion, embedding]:
522
- handler = route(raw_request)
523
- if handler is not None:
524
- response = await handler.unload_lora_adapter(request)
525
- if isinstance(response, ErrorResponse):
526
- return JSONResponse(content=response.model_dump(),
527
- status_code=response.code)
525
+ handler = models(raw_request)
526
+ response = await handler.unload_lora_adapter(request)
527
+ if isinstance(response, ErrorResponse):
528
+ return JSONResponse(content=response.model_dump(),
529
+ status_code=response.code)
530
531
return Response(status_code=200, content=response)
532
@@ -628,13 +630,18 @@ def init_app_state(
630
resolved_chat_template = load_chat_template(args.chat_template)
631
logger.info("Using supplied chat template:\n%s", resolved_chat_template)
632
633
+ state.openai_serving_models = OpenAIServingModels(
634
+ model_config=model_config,
635
+ base_model_paths=base_model_paths,
636
+ lora_modules=args.lora_modules,
637
+ prompt_adapters=args.prompt_adapters,
638
+ )
639
+ # TODO: The chat template is now broken for lora adapters :(
640
state.openai_serving_chat = OpenAIServingChat(
641
engine_client,
642
model_config,
634
- base_model_paths,
643
+ state.openai_serving_models,
644
args.response_role,
636
- lora_modules=args.lora_modules,
637
- prompt_adapters=args.prompt_adapters,
645
request_logger=request_logger,
646
chat_template=resolved_chat_template,
647
chat_template_content_format=args.chat_template_content_format,
@@ -646,16 +653,14 @@ def init_app_state(
653
state.openai_serving_completion = OpenAIServingCompletion(
654
engine_client,
655
model_config,
649
- base_model_paths,
650
- lora_modules=args.lora_modules,
651
- prompt_adapters=args.prompt_adapters,
656
+ state.openai_serving_models,
657
request_logger=request_logger,
658
return_tokens_as_token_ids=args.return_tokens_as_token_ids,
659
) if model_config.runner_type == "generate" else None
660
state.openai_serving_pooling = OpenAIServingPooling(
661
engine_client,
662
model_config,
658
- base_model_paths,
663
+ state.openai_serving_models,
664
request_logger=request_logger,
665
chat_template=resolved_chat_template,
666
chat_template_content_format=args.chat_template_content_format,
@@ -663,7 +668,7 @@ def init_app_state(
668
state.openai_serving_embedding = OpenAIServingEmbedding(
669
engine_client,
670
model_config,
666
- base_model_paths,
671
+ state.openai_serving_models,
672
request_logger=request_logger,
673
chat_template=resolved_chat_template,
674
chat_template_content_format=args.chat_template_content_format,
@@ -671,14 +676,13 @@ def init_app_state(
676
state.openai_serving_scores = OpenAIServingScores(
677
engine_client,
678
model_config,
674
- base_model_paths,
679
+ state.openai_serving_models,
680
request_logger=request_logger
681
) if model_config.task == "score" else None
682
state.openai_serving_tokenization = OpenAIServingTokenization(
683
engine_client,
684
model_config,
680
- base_model_paths,
681
- lora_modules=args.lora_modules,
685
+ state.openai_serving_models,
686
request_logger=request_logger,
687
chat_template=resolved_chat_template,
688
chat_template_content_format=args.chat_template_content_format,
vllm/entrypoints/openai/cli_args.py
+1
-1
@@ -12,7 +12,7 @@ from typing import List, Optional, Sequence, Union, get_args
12
from vllm.engine.arg_utils import AsyncEngineArgs, nullable_str
13
from vllm.entrypoints.chat_utils import (ChatTemplateContentFormatOption,
14
validate_chat_template)
15
-from vllm.entrypoints.openai.serving_engine import (LoRAModulePath,
15
+from vllm.entrypoints.openai.serving_models import (LoRAModulePath,
16
PromptAdapterPath)
17
from vllm.entrypoints.openai.tool_parsers import ToolParserManager
18
from vllm.utils import FlexibleArgumentParser
vllm/entrypoints/openai/run_batch.py
+10
-5
@@ -20,7 +20,8 @@ from vllm.entrypoints.openai.protocol import (BatchRequestInput,
20
# yapf: enable
21
from vllm.entrypoints.openai.serving_chat import OpenAIServingChat
22
from vllm.entrypoints.openai.serving_embedding import OpenAIServingEmbedding
23
-from vllm.entrypoints.openai.serving_engine import BaseModelPath
23
+from vllm.entrypoints.openai.serving_models import (BaseModelPath,
24
+ OpenAIServingModels)
25
from vllm.usage.usage_lib import UsageContext
26
from vllm.utils import FlexibleArgumentParser, random_uuid
27
from vllm.version import __version__ as VLLM_VERSION
@@ -213,13 +214,17 @@ async def main(args):
214
request_logger = RequestLogger(max_log_len=args.max_log_len)
215
216
# Create the openai serving objects.
217
+ openai_serving_models = OpenAIServingModels(
218
+ model_config=model_config,
219
+ base_model_paths=base_model_paths,
220
+ lora_modules=None,
221
+ prompt_adapters=None,
222
+ )
223
openai_serving_chat = OpenAIServingChat(
224
engine,
225
model_config,
219
- base_model_paths,
226
+ openai_serving_models,
227
args.response_role,
221
- lora_modules=None,
222
- prompt_adapters=None,
228
request_logger=request_logger,
229
chat_template=None,
230
chat_template_content_format="auto",
@@ -228,7 +233,7 @@ async def main(args):
233
openai_serving_embedding = OpenAIServingEmbedding(
234
engine,
235
model_config,
231
- base_model_paths,
236
+ openai_serving_models,
237
request_logger=request_logger,
238
chat_template=None,
239
chat_template_content_format="auto",
vllm/entrypoints/openai/serving_chat.py
+5
-11
@@ -21,10 +21,8 @@ from vllm.entrypoints.openai.protocol import (
21
ChatCompletionStreamResponse, ChatMessage, DeltaFunctionCall, DeltaMessage,
22
DeltaToolCall, ErrorResponse, FunctionCall, PromptTokenUsageInfo,
23
RequestResponseMetadata, ToolCall, UsageInfo)
24
-from vllm.entrypoints.openai.serving_engine import (BaseModelPath,
25
- LoRAModulePath,
26
- OpenAIServing,
27
- PromptAdapterPath)
24
+from vllm.entrypoints.openai.serving_engine import OpenAIServing
25
+from vllm.entrypoints.openai.serving_models import OpenAIServingModels
26
from vllm.entrypoints.openai.tool_parsers import ToolParser, ToolParserManager
27
from vllm.logger import init_logger
28
from vllm.outputs import CompletionOutput, RequestOutput
@@ -42,11 +40,9 @@ class OpenAIServingChat(OpenAIServing):
40
self,
41
engine_client: EngineClient,
42
model_config: ModelConfig,
45
- base_model_paths: List[BaseModelPath],
43
+ models: OpenAIServingModels,
44
response_role: str,
45
*,
48
- lora_modules: Optional[List[LoRAModulePath]],
49
- prompt_adapters: Optional[List[PromptAdapterPath]],
46
request_logger: Optional[RequestLogger],
47
chat_template: Optional[str],
48
chat_template_content_format: ChatTemplateContentFormatOption,
@@ -57,9 +53,7 @@ class OpenAIServingChat(OpenAIServing):
53
) -> None:
54
super().__init__(engine_client=engine_client,
55
model_config=model_config,
60
- base_model_paths=base_model_paths,
61
- lora_modules=lora_modules,
62
- prompt_adapters=prompt_adapters,
56
+ models=models,
57
request_logger=request_logger,
58
return_tokens_as_token_ids=return_tokens_as_token_ids)
59
@@ -126,7 +120,7 @@ class OpenAIServingChat(OpenAIServing):
120
prompt_adapter_request,
121
) = self._maybe_get_adapters(request)
122
129
- model_name = self._get_model_name(lora_request)
123
+ model_name = self.models.model_name(lora_request)
124
125
tokenizer = await self.engine_client.get_tokenizer(lora_request)
126
vllm/entrypoints/openai/serving_completion.py
+5
-11
@@ -21,10 +21,8 @@ from vllm.entrypoints.openai.protocol import (CompletionLogProbs,
21
RequestResponseMetadata,
22
UsageInfo)
23
# yapf: enable
24
-from vllm.entrypoints.openai.serving_engine import (BaseModelPath,
25
- LoRAModulePath,
26
- OpenAIServing,
27
- PromptAdapterPath)
24
+from vllm.entrypoints.openai.serving_engine import OpenAIServing
25
+from vllm.entrypoints.openai.serving_models import OpenAIServingModels
26
from vllm.logger import init_logger
27
from vllm.outputs import RequestOutput
28
from vllm.sampling_params import BeamSearchParams, SamplingParams
@@ -41,18 +39,14 @@ class OpenAIServingCompletion(OpenAIServing):
39
self,
40
engine_client: EngineClient,
41
model_config: ModelConfig,
44
- base_model_paths: List[BaseModelPath],
42
+ models: OpenAIServingModels,
43
*,
46
- lora_modules: Optional[List[LoRAModulePath]],
47
- prompt_adapters: Optional[List[PromptAdapterPath]],
44
request_logger: Optional[RequestLogger],
45
return_tokens_as_token_ids: bool = False,
46
):
47
super().__init__(engine_client=engine_client,
48
model_config=model_config,
53
- base_model_paths=base_model_paths,
54
- lora_modules=lora_modules,
55
- prompt_adapters=prompt_adapters,
49
+ models=models,
50
request_logger=request_logger,
51
return_tokens_as_token_ids=return_tokens_as_token_ids)
52
diff_sampling_param = self.model_config.get_diff_sampling_param()
@@ -170,7 +164,7 @@ class OpenAIServingCompletion(OpenAIServing):
164
165
result_generator = merge_async_iterators(*generators)
166
173
- model_name = self._get_model_name(lora_request)
167
+ model_name = self.models.model_name(lora_request)
168
num_prompts = len(engine_prompts)
169
170
# Similar to the OpenAI API, when n != best_of, we do not stream the
vllm/entrypoints/openai/serving_embedding.py
+4
-5
@@ -16,7 +16,8 @@ from vllm.entrypoints.openai.protocol import (EmbeddingChatRequest,
16
EmbeddingResponse,
17
EmbeddingResponseData,
18
ErrorResponse, UsageInfo)
19
-from vllm.entrypoints.openai.serving_engine import BaseModelPath, OpenAIServing
19
+from vllm.entrypoints.openai.serving_engine import OpenAIServing
20
+from vllm.entrypoints.openai.serving_models import OpenAIServingModels
21
from vllm.logger import init_logger
22
from vllm.outputs import (EmbeddingOutput, EmbeddingRequestOutput,
23
PoolingRequestOutput)
@@ -46,7 +47,7 @@ class OpenAIServingEmbedding(OpenAIServing):
47
self,
48
engine_client: EngineClient,
49
model_config: ModelConfig,
49
- base_model_paths: List[BaseModelPath],
50
+ models: OpenAIServingModels,
51
*,
52
request_logger: Optional[RequestLogger],
53
chat_template: Optional[str],
@@ -54,9 +55,7 @@ class OpenAIServingEmbedding(OpenAIServing):
55
) -> None:
56
super().__init__(engine_client=engine_client,
57
model_config=model_config,
57
- base_model_paths=base_model_paths,
58
- lora_modules=None,
59
- prompt_adapters=None,
58
+ models=models,
59
request_logger=request_logger)
60
61
self.chat_template = chat_template
vllm/entrypoints/openai/serving_engine.py
+13
-179
@@ -1,7 +1,5 @@
1
import json
2
-import pathlib
2
from concurrent.futures.thread import ThreadPoolExecutor
4
-from dataclasses import dataclass
3
from http import HTTPStatus
4
from typing import (Any, Callable, Dict, Iterable, Iterator, List, Mapping,
5
Optional, Sequence, Tuple, TypedDict, Union)
@@ -28,13 +26,10 @@ from vllm.entrypoints.openai.protocol import (ChatCompletionRequest,
26
DetokenizeRequest,
27
EmbeddingChatRequest,
28
EmbeddingCompletionRequest,
31
- ErrorResponse,
32
- LoadLoraAdapterRequest,
33
- ModelCard, ModelList,
34
- ModelPermission, ScoreRequest,
29
+ ErrorResponse, ScoreRequest,
30
TokenizeChatRequest,
36
- TokenizeCompletionRequest,
37
- UnloadLoraAdapterRequest)
31
+ TokenizeCompletionRequest)
32
+from vllm.entrypoints.openai.serving_models import OpenAIServingModels
33
from vllm.entrypoints.openai.tool_parsers import ToolParser
34
# yapf: enable
35
from vllm.inputs import TokensPrompt
@@ -48,30 +43,10 @@ from vllm.sequence import Logprob
43
from vllm.tracing import (contains_trace_headers, extract_trace_headers,
44
log_tracing_disabled_warning)
45
from vllm.transformers_utils.tokenizer import AnyTokenizer, MistralTokenizer
51
-from vllm.utils import AtomicCounter, is_list_of, make_async, random_uuid
46
+from vllm.utils import is_list_of, make_async, random_uuid
47
48
logger = init_logger(__name__)
49
55
-
56
-@dataclass
57
-class BaseModelPath:
58
- name: str
59
- model_path: str
60
-
61
-
62
-@dataclass
63
-class PromptAdapterPath:
64
- name: str
65
- local_path: str
66
-
67
-
68
-@dataclass
69
-class LoRAModulePath:
70
- name: str
71
- path: str
72
- base_model_name: Optional[str] = None
73
-
74
-
50
CompletionLikeRequest = Union[CompletionRequest, DetokenizeRequest,
51
EmbeddingCompletionRequest, ScoreRequest,
52
TokenizeCompletionRequest]
@@ -96,10 +71,8 @@ class OpenAIServing:
71
self,
72
engine_client: EngineClient,
73
model_config: ModelConfig,
99
- base_model_paths: List[BaseModelPath],
74
+ models: OpenAIServingModels,
75
*,
101
- lora_modules: Optional[List[LoRAModulePath]],
102
- prompt_adapters: Optional[List[PromptAdapterPath]],
76
request_logger: Optional[RequestLogger],
77
return_tokens_as_token_ids: bool = False,
78
):
@@ -109,35 +82,7 @@ class OpenAIServing:
82
self.model_config = model_config
83
self.max_model_len = model_config.max_model_len
84
112
- self.base_model_paths = base_model_paths
113
-
114
- self.lora_id_counter = AtomicCounter(0)
115
- self.lora_requests = []
116
- if lora_modules is not None:
117
- self.lora_requests = [
118
- LoRARequest(lora_name=lora.name,
119
- lora_int_id=i,
120
- lora_path=lora.path,
121
- base_model_name=lora.base_model_name
122
- if lora.base_model_name
123
- and self._is_model_supported(lora.base_model_name)
124
- else self.base_model_paths[0].name)
125
- for i, lora in enumerate(lora_modules, start=1)
126
- ]
127
-
128
- self.prompt_adapter_requests = []
129
- if prompt_adapters is not None:
130
- for i, prompt_adapter in enumerate(prompt_adapters, start=1):
131
- with pathlib.Path(prompt_adapter.local_path,
132
- "adapter_config.json").open() as f:
133
- adapter_config = json.load(f)
134
- num_virtual_tokens = adapter_config["num_virtual_tokens"]
135
- self.prompt_adapter_requests.append(
136
- PromptAdapterRequest(
137
- prompt_adapter_name=prompt_adapter.name,
138
- prompt_adapter_id=i,
139
- prompt_adapter_local_path=prompt_adapter.local_path,
140
- prompt_adapter_num_virtual_tokens=num_virtual_tokens))
85
+ self.models = models
86
87
self.request_logger = request_logger
88
self.return_tokens_as_token_ids = return_tokens_as_token_ids
@@ -150,33 +95,6 @@ class OpenAIServing:
95
self._tokenize_prompt_input_or_inputs,
96
executor=self._tokenizer_executor)
97
153
- async def show_available_models(self) -> ModelList:
154
- """Show available models. Right now we only have one model."""
155
- model_cards = [
156
- ModelCard(id=base_model.name,
157
- max_model_len=self.max_model_len,
158
- root=base_model.model_path,
159
- permission=[ModelPermission()])
160
- for base_model in self.base_model_paths
161
- ]
162
- lora_cards = [
163
- ModelCard(id=lora.lora_name,
164
- root=lora.local_path,
165
- parent=lora.base_model_name if lora.base_model_name else
166
- self.base_model_paths[0].name,
167
- permission=[ModelPermission()])
168
- for lora in self.lora_requests
169
- ]
170
- prompt_adapter_cards = [
171
- ModelCard(id=prompt_adapter.prompt_adapter_name,
172
- root=self.base_model_paths[0].name,
173
- permission=[ModelPermission()])
174
- for prompt_adapter in self.prompt_adapter_requests
175
- ]
176
- model_cards.extend(lora_cards)
177
- model_cards.extend(prompt_adapter_cards)
178
- return ModelList(data=model_cards)
179
-
98
def create_error_response(
99
self,
100
message: str,
@@ -205,11 +123,13 @@ class OpenAIServing:
123
) -> Optional[ErrorResponse]:
124
if self._is_model_supported(request.model):
125
return None
208
- if request.model in [lora.lora_name for lora in self.lora_requests]:
126
+ if request.model in [
127
+ lora.lora_name for lora in self.models.lora_requests
128
+ ]:
129
return None
130
if request.model in [
131
prompt_adapter.prompt_adapter_name
212
- for prompt_adapter in self.prompt_adapter_requests
132
+ for prompt_adapter in self.models.prompt_adapter_requests
133
]:
134
return None
135
return self.create_error_response(
@@ -223,10 +143,10 @@ class OpenAIServing:
143
None, PromptAdapterRequest]]:
144
if self._is_model_supported(request.model):
145
return None, None
226
- for lora in self.lora_requests:
146
+ for lora in self.models.lora_requests:
147
if request.model == lora.lora_name:
148
return lora, None
229
- for prompt_adapter in self.prompt_adapter_requests:
149
+ for prompt_adapter in self.models.prompt_adapter_requests:
150
if request.model == prompt_adapter.prompt_adapter_name:
151
return None, prompt_adapter
152
# if _check_model has been called earlier, this will be unreachable
@@ -588,91 +508,5 @@ class OpenAIServing:
508
return logprob.decoded_token
509
return tokenizer.decode(token_id)
510
591
- async def _check_load_lora_adapter_request(
592
- self, request: LoadLoraAdapterRequest) -> Optional[ErrorResponse]:
593
- # Check if both 'lora_name' and 'lora_path' are provided
594
- if not request.lora_name or not request.lora_path:
595
- return self.create_error_response(
596
- message="Both 'lora_name' and 'lora_path' must be provided.",
597
- err_type="InvalidUserInput",
598
- status_code=HTTPStatus.BAD_REQUEST)
599
-
600
- # Check if the lora adapter with the given name already exists
601
- if any(lora_request.lora_name == request.lora_name
602
- for lora_request in self.lora_requests):
603
- return self.create_error_response(
604
- message=
605
- f"The lora adapter '{request.lora_name}' has already been"
606
- "loaded.",
607
- err_type="InvalidUserInput",
608
- status_code=HTTPStatus.BAD_REQUEST)
609
-
610
- return None
611
-
612
- async def _check_unload_lora_adapter_request(
613
- self,
614
- request: UnloadLoraAdapterRequest) -> Optional[ErrorResponse]:
615
- # Check if either 'lora_name' or 'lora_int_id' is provided
616
- if not request.lora_name and not request.lora_int_id:
617
- return self.create_error_response(
618
- message=
619
- "either 'lora_name' and 'lora_int_id' needs to be provided.",
620
- err_type="InvalidUserInput",
621
- status_code=HTTPStatus.BAD_REQUEST)
622
-
623
- # Check if the lora adapter with the given name exists
624
- if not any(lora_request.lora_name == request.lora_name
625
- for lora_request in self.lora_requests):
626
- return self.create_error_response(
627
- message=
628
- f"The lora adapter '{request.lora_name}' cannot be found.",
629
- err_type="InvalidUserInput",
630
- status_code=HTTPStatus.BAD_REQUEST)
631
-
632
- return None
633
-
634
- async def load_lora_adapter(
635
- self,
636
- request: LoadLoraAdapterRequest) -> Union[ErrorResponse, str]:
637
- error_check_ret = await self._check_load_lora_adapter_request(request)
638
- if error_check_ret is not None:
639
- return error_check_ret
640
-
641
- lora_name, lora_path = request.lora_name, request.lora_path
642
- unique_id = self.lora_id_counter.inc(1)
643
- self.lora_requests.append(
644
- LoRARequest(lora_name=lora_name,
645
- lora_int_id=unique_id,
646
- lora_path=lora_path))
647
- return f"Success: LoRA adapter '{lora_name}' added successfully."
648
-
649
- async def unload_lora_adapter(
650
- self,
651
- request: UnloadLoraAdapterRequest) -> Union[ErrorResponse, str]:
652
- error_check_ret = await self._check_unload_lora_adapter_request(request
653
- )
654
- if error_check_ret is not None:
655
- return error_check_ret
656
-
657
- lora_name = request.lora_name
658
- self.lora_requests = [
659
- lora_request for lora_request in self.lora_requests
660
- if lora_request.lora_name != lora_name
661
- ]
662
- return f"Success: LoRA adapter '{lora_name}' removed successfully."
663
-
511
def _is_model_supported(self, model_name):
665
- return any(model.name == model_name for model in self.base_model_paths)
666
-
667
- def _get_model_name(self, lora: Optional[LoRARequest]):
668
- """
669
- Returns the appropriate model name depending on the availability
670
- and support of the LoRA or base model.
671
- Parameters:
672
- - lora: LoRARequest that contain a base_model_name.
673
- Returns:
674
- - str: The name of the base model or the first available model path.
675
- """
676
- if lora is not None:
677
- return lora.lora_name
678
- return self.base_model_paths[0].name
512
+ return self.models.is_base_model(model_name)
vllm/entrypoints/openai/serving_models.py
new
+210
@@ -0,0 +1,210 @@
1
+import json
2
+import pathlib
3
+from dataclasses import dataclass
4
+from http import HTTPStatus
5
+from typing import List, Optional, Union
6
+
7
+from vllm.config import ModelConfig
8
+from vllm.entrypoints.openai.protocol import (ErrorResponse,
9
+ LoadLoraAdapterRequest,
10
+ ModelCard, ModelList,
11
+ ModelPermission,
12
+ UnloadLoraAdapterRequest)
13
+from vllm.lora.request import LoRARequest
14
+from vllm.prompt_adapter.request import PromptAdapterRequest
15
+from vllm.utils import AtomicCounter
16
+
17
+
18
+@dataclass
19
+class BaseModelPath:
20
+ name: str
21
+ model_path: str
22
+
23
+
24
+@dataclass
25
+class PromptAdapterPath:
26
+ name: str
27
+ local_path: str
28
+
29
+
30
+@dataclass
31
+class LoRAModulePath:
32
+ name: str
33
+ path: str
34
+ base_model_name: Optional[str] = None
35
+
36
+
37
+class OpenAIServingModels:
38
+ """Shared instance to hold data about the loaded base model(s) and adapters.
39
+
40
+ Handles the routes:
41
+ - /v1/models
42
+ - /v1/load_lora_adapter
43
+ - /v1/unload_lora_adapter
44
+ """
45
+
46
+ def __init__(
47
+ self,
48
+ model_config: ModelConfig,
49
+ base_model_paths: List[BaseModelPath],
50
+ *,
51
+ lora_modules: Optional[List[LoRAModulePath]] = None,
52
+ prompt_adapters: Optional[List[PromptAdapterPath]] = None,
53
+ ):
54
+ super().__init__()
55
+
56
+ self.base_model_paths = base_model_paths
57
+ self.max_model_len = model_config.max_model_len
58
+
59
+ self.lora_id_counter = AtomicCounter(0)
60
+ self.lora_requests = []
61
+ if lora_modules is not None:
62
+ self.lora_requests = [
63
+ LoRARequest(lora_name=lora.name,
64
+ lora_int_id=i,
65
+ lora_path=lora.path,
66
+ base_model_name=lora.base_model_name
67
+ if lora.base_model_name
68
+ and self.is_base_model(lora.base_model_name) else
69
+ self.base_model_paths[0].name)
70
+ for i, lora in enumerate(lora_modules, start=1)
71
+ ]
72
+
73
+ self.prompt_adapter_requests = []
74
+ if prompt_adapters is not None:
75
+ for i, prompt_adapter in enumerate(prompt_adapters, start=1):
76
+ with pathlib.Path(prompt_adapter.local_path,
77
+ "adapter_config.json").open() as f:
78
+ adapter_config = json.load(f)
79
+ num_virtual_tokens = adapter_config["num_virtual_tokens"]
80
+ self.prompt_adapter_requests.append(
81
+ PromptAdapterRequest(
82
+ prompt_adapter_name=prompt_adapter.name,
83
+ prompt_adapter_id=i,
84
+ prompt_adapter_local_path=prompt_adapter.local_path,
85
+ prompt_adapter_num_virtual_tokens=num_virtual_tokens))
86
+
87
+ def is_base_model(self, model_name):
88
+ return any(model.name == model_name for model in self.base_model_paths)
89
+
90
+ def model_name(self, lora_request: Optional[LoRARequest] = None) -> str:
91
+ """Returns the appropriate model name depending on the availability
92
+ and support of the LoRA or base model.
93
+ Parameters:
94
+ - lora: LoRARequest that contain a base_model_name.
95
+ Returns:
96
+ - str: The name of the base model or the first available model path.
97
+ """
98
+ if lora_request is not None:
99
+ return lora_request.lora_name
100
+ return self.base_model_paths[0].name
101
+
102
+ async def show_available_models(self) -> ModelList:
103
+ """Show available models. This includes the base model and all
104
+ adapters"""
105
+ model_cards = [
106
+ ModelCard(id=base_model.name,
107
+ max_model_len=self.max_model_len,
108
+ root=base_model.model_path,
109
+ permission=[ModelPermission()])
110
+ for base_model in self.base_model_paths
111
+ ]
112
+ lora_cards = [
113
+ ModelCard(id=lora.lora_name,
114
+ root=lora.local_path,
115
+ parent=lora.base_model_name if lora.base_model_name else
116
+ self.base_model_paths[0].name,
117
+ permission=[ModelPermission()])
118
+ for lora in self.lora_requests
119
+ ]
120
+ prompt_adapter_cards = [
121
+ ModelCard(id=prompt_adapter.prompt_adapter_name,
122
+ root=self.base_model_paths[0].name,
123
+ permission=[ModelPermission()])
124
+ for prompt_adapter in self.prompt_adapter_requests
125
+ ]
126
+ model_cards.extend(lora_cards)
127
+ model_cards.extend(prompt_adapter_cards)
128
+ return ModelList(data=model_cards)
129
+
130
+ async def load_lora_adapter(
131
+ self,
132
+ request: LoadLoraAdapterRequest) -> Union[ErrorResponse, str]:
133
+ error_check_ret = await self._check_load_lora_adapter_request(request)
134
+ if error_check_ret is not None:
135
+ return error_check_ret
136
+
137
+ lora_name, lora_path = request.lora_name, request.lora_path
138
+ unique_id = self.lora_id_counter.inc(1)
139
+ self.lora_requests.append(
140
+ LoRARequest(lora_name=lora_name,
141
+ lora_int_id=unique_id,
142
+ lora_path=lora_path))
143
+ return f"Success: LoRA adapter '{lora_name}' added successfully."
144
+
145
+ async def unload_lora_adapter(
146
+ self,
147
+ request: UnloadLoraAdapterRequest) -> Union[ErrorResponse, str]:
148
+ error_check_ret = await self._check_unload_lora_adapter_request(request
149
+ )
150
+ if error_check_ret is not None:
151
+ return error_check_ret
152
+
153
+ lora_name = request.lora_name
154
+ self.lora_requests = [
155
+ lora_request for lora_request in self.lora_requests
156
+ if lora_request.lora_name != lora_name
157
+ ]
158
+ return f"Success: LoRA adapter '{lora_name}' removed successfully."
159
+
160
+ async def _check_load_lora_adapter_request(
161
+ self, request: LoadLoraAdapterRequest) -> Optional[ErrorResponse]:
162
+ # Check if both 'lora_name' and 'lora_path' are provided
163
+ if not request.lora_name or not request.lora_path:
164
+ return create_error_response(
165
+ message="Both 'lora_name' and 'lora_path' must be provided.",
166
+ err_type="InvalidUserInput",
167
+ status_code=HTTPStatus.BAD_REQUEST)
168
+
169
+ # Check if the lora adapter with the given name already exists
170
+ if any(lora_request.lora_name == request.lora_name
171
+ for lora_request in self.lora_requests):
172
+ return create_error_response(
173
+ message=
174
+ f"The lora adapter '{request.lora_name}' has already been"
175
+ "loaded.",
176
+ err_type="InvalidUserInput",
177
+ status_code=HTTPStatus.BAD_REQUEST)
178
+
179
+ return None
180
+
181
+ async def _check_unload_lora_adapter_request(
182
+ self,
183
+ request: UnloadLoraAdapterRequest) -> Optional[ErrorResponse]:
184
+ # Check if either 'lora_name' or 'lora_int_id' is provided
185
+ if not request.lora_name and not request.lora_int_id:
186
+ return create_error_response(
187
+ message=
188
+ "either 'lora_name' and 'lora_int_id' needs to be provided.",
189
+ err_type="InvalidUserInput",
190
+ status_code=HTTPStatus.BAD_REQUEST)
191
+
192
+ # Check if the lora adapter with the given name exists
193
+ if not any(lora_request.lora_name == request.lora_name
194
+ for lora_request in self.lora_requests):
195
+ return create_error_response(
196
+ message=
197
+ f"The lora adapter '{request.lora_name}' cannot be found.",
198
+ err_type="InvalidUserInput",
199
+ status_code=HTTPStatus.BAD_REQUEST)
200
+
201
+ return None
202
+
203
+
204
+def create_error_response(
205
+ message: str,
206
+ err_type: str = "BadRequestError",
207
+ status_code: HTTPStatus = HTTPStatus.BAD_REQUEST) -> ErrorResponse:
208
+ return ErrorResponse(message=message,
209
+ type=err_type,
210
+ code=status_code.value)
vllm/entrypoints/openai/serving_pooling.py
+4
-5
@@ -15,7 +15,8 @@ from vllm.entrypoints.openai.protocol import (ErrorResponse,
15
PoolingChatRequest,
16
PoolingRequest, PoolingResponse,
17
PoolingResponseData, UsageInfo)
18
-from vllm.entrypoints.openai.serving_engine import BaseModelPath, OpenAIServing
18
+from vllm.entrypoints.openai.serving_engine import OpenAIServing
19
+from vllm.entrypoints.openai.serving_models import OpenAIServingModels
20
from vllm.logger import init_logger
21
from vllm.outputs import PoolingOutput, PoolingRequestOutput
22
from vllm.utils import merge_async_iterators
@@ -44,7 +45,7 @@ class OpenAIServingPooling(OpenAIServing):
45
self,
46
engine_client: EngineClient,
47
model_config: ModelConfig,
47
- base_model_paths: List[BaseModelPath],
48
+ models: OpenAIServingModels,
49
*,
50
request_logger: Optional[RequestLogger],
51
chat_template: Optional[str],
@@ -52,9 +53,7 @@ class OpenAIServingPooling(OpenAIServing):
53
) -> None:
54
super().__init__(engine_client=engine_client,
55
model_config=model_config,
55
- base_model_paths=base_model_paths,
56
- lora_modules=None,
57
- prompt_adapters=None,
56
+ models=models,
57
request_logger=request_logger)
58
59
self.chat_template = chat_template
vllm/entrypoints/openai/serving_score.py
+4
-5
@@ -10,7 +10,8 @@ from vllm.entrypoints.logger import RequestLogger
10
from vllm.entrypoints.openai.protocol import (ErrorResponse, ScoreRequest,
11
ScoreResponse, ScoreResponseData,
12
UsageInfo)
13
-from vllm.entrypoints.openai.serving_engine import BaseModelPath, OpenAIServing
13
+from vllm.entrypoints.openai.serving_engine import OpenAIServing
14
+from vllm.entrypoints.openai.serving_models import OpenAIServingModels
15
from vllm.inputs.data import TokensPrompt
16
from vllm.logger import init_logger
17
from vllm.outputs import PoolingRequestOutput, ScoringRequestOutput
@@ -50,15 +51,13 @@ class OpenAIServingScores(OpenAIServing):
51
self,
52
engine_client: EngineClient,
53
model_config: ModelConfig,
53
- base_model_paths: List[BaseModelPath],
54
+ models: OpenAIServingModels,
55
*,
56
request_logger: Optional[RequestLogger],
57
) -> None:
58
super().__init__(engine_client=engine_client,
59
model_config=model_config,
59
- base_model_paths=base_model_paths,
60
- lora_modules=None,
61
- prompt_adapters=None,
60
+ models=models,
61
request_logger=request_logger)
62
63
async def create_score(
vllm/entrypoints/openai/serving_tokenization.py
+4
-8
@@ -15,9 +15,8 @@ from vllm.entrypoints.openai.protocol import (DetokenizeRequest,
15
TokenizeRequest,
16
TokenizeResponse)
17
# yapf: enable
18
-from vllm.entrypoints.openai.serving_engine import (BaseModelPath,
19
- LoRAModulePath,
20
- OpenAIServing)
18
+from vllm.entrypoints.openai.serving_engine import OpenAIServing
19
+from vllm.entrypoints.openai.serving_models import OpenAIServingModels
20
from vllm.logger import init_logger
21
22
logger = init_logger(__name__)
@@ -29,18 +28,15 @@ class OpenAIServingTokenization(OpenAIServing):
28
self,
29
engine_client: EngineClient,
30
model_config: ModelConfig,
32
- base_model_paths: List[BaseModelPath],
31
+ models: OpenAIServingModels,
32
*,
34
- lora_modules: Optional[List[LoRAModulePath]],
33
request_logger: Optional[RequestLogger],
34
chat_template: Optional[str],
35
chat_template_content_format: ChatTemplateContentFormatOption,
36
) -> None:
37
super().__init__(engine_client=engine_client,
38
model_config=model_config,
41
- base_model_paths=base_model_paths,
42
- lora_modules=lora_modules,
43
- prompt_adapters=None,
39
+ models=models,
40
request_logger=request_logger)
41
42
self.chat_template = chat_template