A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 6, 20:30 UTC
[Frontend] Split `OpenAIServingModels` into `OpenAIModelRegistry` + `OpenAIServingModels` (#36536)
Signed-off-by: Sage Ahrac <sagiahrak@gmail.com>
Sage committed
Mar 12, 2026 at 12:29 UTC
06e0bc21d2f978ef86ea7f98868922aecc524d26
4 files changed
+73
-62
vllm/entrypoints/openai/api_server.py
+10
-3
@@ -414,11 +414,19 @@ async def init_render_app_state(
414
directly from the :class:`~vllm.config.VllmConfig`.
415
"""
416
from vllm.entrypoints.chat_utils import load_chat_template
417
+ from vllm.entrypoints.openai.models.serving import OpenAIModelRegistry
418
from vllm.entrypoints.serve.render.serving import OpenAIServingRender
419
from vllm.plugins.io_processors import get_io_processor
420
from vllm.renderers import renderer_from_config
421
422
served_model_names = args.served_model_name or [args.model]
423
+ model_registry = OpenAIModelRegistry(
424
+ model_config=vllm_config.model_config,
425
+ base_model_paths=[
426
+ BaseModelPath(name=name, model_path=args.model)
427
+ for name in served_model_names
428
+ ],
429
+ )
430
431
if args.enable_log_requests:
432
request_logger = RequestLogger(max_log_len=args.max_log_len)
@@ -435,7 +443,7 @@ async def init_render_app_state(
443
model_config=vllm_config.model_config,
444
renderer=renderer,
445
io_processor=io_processor,
438
- served_model_names=served_model_names,
446
+ model_registry=model_registry,
447
request_logger=request_logger,
448
chat_template=resolved_chat_template,
449
chat_template_content_format=args.chat_template_content_format,
@@ -447,8 +455,7 @@ async def init_render_app_state(
455
log_error_stack=args.log_error_stack,
456
)
457
450
- # Expose models endpoint via the render handler.
451
- state.openai_serving_models = state.openai_serving_render
458
+ state.openai_serving_models = model_registry
459
460
state.vllm_config = vllm_config
461
# Disable stats logging — there is no engine to poll.
vllm/entrypoints/openai/generate/api_router.py
+1
-3
@@ -169,9 +169,7 @@ async def init_generate_state(
169
model_config=engine_client.model_config,
170
renderer=engine_client.renderer,
171
io_processor=engine_client.io_processor,
172
- served_model_names=[
173
- mp.name for mp in state.openai_serving_models.base_model_paths
174
- ],
172
+ model_registry=state.openai_serving_models.registry,
173
request_logger=request_logger,
174
chat_template=resolved_chat_template,
175
chat_template_content_format=args.chat_template_content_format,
vllm/entrypoints/openai/models/serving.py
+58
-23
@@ -5,6 +5,7 @@ from asyncio import Lock
5
from collections import defaultdict
6
from http import HTTPStatus
7
8
+from vllm.config import ModelConfig
9
from vllm.engine.protocol import EngineClient
10
from vllm.entrypoints.openai.engine.protocol import (
11
ErrorResponse,
@@ -27,6 +28,51 @@ from vllm.utils.counter import AtomicCounter
28
logger = init_logger(__name__)
29
30
31
+class OpenAIModelRegistry:
32
+ """Read-only view of the loaded base models with no engine dependency.
33
+
34
+ Suitable for CPU-only / render-only contexts that have no engine client
35
+ and no LoRA support.
36
+ """
37
+
38
+ def __init__(
39
+ self,
40
+ model_config: ModelConfig,
41
+ base_model_paths: list[BaseModelPath],
42
+ ) -> None:
43
+ self.model_config = model_config
44
+ self.base_model_paths = base_model_paths
45
+
46
+ def is_base_model(self, model_name: str) -> bool:
47
+ return any(model.name == model_name for model in self.base_model_paths)
48
+
49
+ async def check_model(self, model_name: str | None) -> ErrorResponse | None:
50
+ """Return an ErrorResponse if model_name is not served, else None."""
51
+ if not model_name or self.is_base_model(model_name):
52
+ return None
53
+ return create_error_response(
54
+ message=f"The model `{model_name}` does not exist.",
55
+ err_type="NotFoundError",
56
+ status_code=HTTPStatus.NOT_FOUND,
57
+ param="model",
58
+ )
59
+
60
+ async def show_available_models(self) -> ModelList:
61
+ """Show available models (base models only)."""
62
+ max_model_len = self.model_config.max_model_len
63
+ return ModelList(
64
+ data=[
65
+ ModelCard(
66
+ id=base_model.name,
67
+ max_model_len=max_model_len,
68
+ root=base_model.model_path,
69
+ permission=[ModelPermission()],
70
+ )
71
+ for base_model in self.base_model_paths
72
+ ]
73
+ )
74
+
75
+
76
class OpenAIServingModels:
77
"""Shared instance to hold data about the loaded base model(s) and adapters.
78
@@ -45,6 +91,11 @@ class OpenAIServingModels:
91
):
92
super().__init__()
93
94
+ self.registry = OpenAIModelRegistry(
95
+ model_config=engine_client.model_config,
96
+ base_model_paths=base_model_paths,
97
+ )
98
+
99
self.engine_client = engine_client
100
self.base_model_paths = base_model_paths
101
@@ -79,34 +130,18 @@ class OpenAIServingModels:
130
if isinstance(load_result, ErrorResponse):
131
raise ValueError(load_result.error.message)
132
82
- def is_base_model(self, model_name) -> bool:
83
- return any(model.name == model_name for model in self.base_model_paths)
133
+ def is_base_model(self, model_name: str) -> bool:
134
+ return self.registry.is_base_model(model_name)
135
136
def model_name(self, lora_request: LoRARequest | None = None) -> str:
86
- """Returns the appropriate model name depending on the availability
87
- and support of the LoRA or base model.
88
- Parameters:
89
- - lora: LoRARequest that contain a base_model_name.
90
- Returns:
91
- - str: The name of the base model or the first available model path.
92
- """
137
if lora_request is not None:
138
return lora_request.lora_name
139
return self.base_model_paths[0].name
140
141
async def show_available_models(self) -> ModelList:
98
- """Show available models. This includes the base model and all adapters."""
99
- max_model_len = self.model_config.max_model_len
100
-
101
- model_cards = [
102
- ModelCard(
103
- id=base_model.name,
104
- max_model_len=max_model_len,
105
- root=base_model.model_path,
106
- permission=[ModelPermission()],
107
- )
108
- for base_model in self.base_model_paths
109
- ]
142
+ """Show available models. This includes the base model and all
143
+ adapters."""
144
+ model_list = await self.registry.show_available_models()
145
lora_cards = [
146
ModelCard(
147
id=lora.lora_name,
@@ -118,8 +153,8 @@ class OpenAIServingModels:
153
)
154
for lora in self.lora_requests.values()
155
]
121
- model_cards.extend(lora_cards)
122
- return ModelList(data=model_cards)
156
+ model_list.data.extend(lora_cards)
157
+ return model_list
158
159
async def load_lora_adapter(
160
self, request: LoadLoRAAdapterRequest, base_model_name: str | None = None
vllm/entrypoints/serve/render/serving.py
+4
-33
@@ -16,10 +16,8 @@ from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionReque
16
from vllm.entrypoints.openai.completion.protocol import CompletionRequest
17
from vllm.entrypoints.openai.engine.protocol import (
18
ErrorResponse,
19
- ModelCard,
20
- ModelList,
21
- ModelPermission,
19
)
20
+from vllm.entrypoints.openai.models.serving import OpenAIModelRegistry
21
from vllm.entrypoints.openai.parser.harmony_utils import (
22
get_developer_message,
23
get_system_message,
@@ -46,7 +44,7 @@ class OpenAIServingRender:
44
model_config: ModelConfig,
45
renderer: BaseRenderer,
46
io_processor: Any,
49
- served_model_names: list[str],
47
+ model_registry: OpenAIModelRegistry,
48
*,
49
request_logger: RequestLogger | None,
50
chat_template: str | None,
@@ -61,7 +59,7 @@ class OpenAIServingRender:
59
self.model_config = model_config
60
self.renderer = renderer
61
self.io_processor = io_processor
64
- self.served_model_names = served_model_names
62
+ self.model_registry = model_registry
63
self.request_logger = request_logger
64
self.chat_template = chat_template
65
self.chat_template_content_format: ChatTemplateContentFormatOption = (
@@ -252,21 +250,6 @@ class OpenAIServingRender:
250
251
return messages, [engine_prompt]
252
255
- async def show_available_models(self) -> ModelList:
256
- """Returns the models served by this render server."""
257
- max_model_len = self.model_config.max_model_len
258
- return ModelList(
259
- data=[
260
- ModelCard(
261
- id=name,
262
- max_model_len=max_model_len,
263
- root=self.model_config.model,
264
- permission=[ModelPermission()],
265
- )
266
- for name in self.served_model_names
267
- ]
268
- )
269
-
253
def create_error_response(
254
self,
255
message: str | Exception,
@@ -276,23 +259,11 @@ class OpenAIServingRender:
259
) -> ErrorResponse:
260
return create_error_response(message, err_type, status_code, param)
261
279
- def _is_model_supported(self, model_name: str) -> bool:
280
- """Simplified from OpenAIServing._is_model_supported (no LoRA support)."""
281
- return model_name in self.served_model_names
282
-
262
async def _check_model(
263
self,
264
request: Any,
265
) -> ErrorResponse | None:
287
- """Simplified from OpenAIServing._check_model (no LoRA support)."""
288
- if self._is_model_supported(request.model):
289
- return None
290
- return self.create_error_response(
291
- message=f"The model `{request.model}` does not exist.",
292
- err_type="NotFoundError",
293
- status_code=HTTPStatus.NOT_FOUND,
294
- param="model",
295
- )
266
+ return await self.model_registry.check_model(request.model)
267
268
def _validate_chat_template(
269
self,