[Bugfix][Frontend] Lazy-import model_hosting_container_standards to prevent log suppression (#51366)
Signed-off-by: wenjinhust <wenjin.sh@huawei.com>
wenjinhust committed
Sep 16, 2026 at 16:25 UTC
0d8173d1537f60331084e12af6b72a6565978c9d
2 files changed
+87
-4
vllm/entrypoints/serve/lora/api_router.py
+28
-2
@@ -1,8 +1,7 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+import logging
4
4
-
5
-import model_hosting_container_standards.sagemaker as sagemaker_standards
5
from fastapi import APIRouter, Depends, FastAPI, Request
6
from fastapi.responses import JSONResponse, Response
7
@@ -22,6 +21,12 @@ router = APIRouter()
21
22
23
def attach_router(app: FastAPI):
24
+ """Attach the LoRA adapter load/unload endpoints to the API server.
25
+
26
+ Does nothing when dynamic LoRA updating is disabled. Handler levels are
27
+ snapshotted and restored because importing
28
+ model_hosting_container_standards may reconfigure root logging.
29
+ """
30
if not envs.VLLM_ALLOW_RUNTIME_LORA_UPDATING:
31
"""If LoRA dynamic loading & unloading is not enabled, do nothing."""
32
return
@@ -30,6 +35,23 @@ def attach_router(app: FastAPI):
35
"This should ONLY be used for local development!"
36
)
37
38
+ snapshot = [
39
+ (h, h.level)
40
+ for lg in (logging.getLogger(), logging.getLogger("vllm"))
41
+ for h in lg.handlers
42
+ ]
43
+
44
+ try:
45
+ _attach_router(app)
46
+ finally:
47
+ for handler, level in snapshot:
48
+ handler.setLevel(level)
49
+
50
+
51
+def _attach_router(app: FastAPI):
52
+ """Register the LoRA adapter load/unload routes on the app."""
53
+ import model_hosting_container_standards.sagemaker as sagemaker_standards
54
+
55
@sagemaker_standards.register_load_adapter_handler(
56
request_shape={
57
"lora_name": "body.name",
@@ -40,6 +62,8 @@ def attach_router(app: FastAPI):
62
)
63
@router.post("/v1/load_lora_adapter", dependencies=[Depends(validate_json_request)])
64
async def load_lora_adapter(request: LoadLoRAAdapterRequest, raw_request: Request):
65
+ """Handle POST /v1/load_lora_adapter: load a LoRA adapter into the
66
+ serving engine."""
67
handler: OpenAIServingModels = models(raw_request)
68
response = await handler.load_lora_adapter(request)
69
if isinstance(response, ErrorResponse):
@@ -60,6 +84,8 @@ def attach_router(app: FastAPI):
84
async def unload_lora_adapter(
85
request: UnloadLoRAAdapterRequest, raw_request: Request
86
):
87
+ """Handle POST /v1/unload_lora_adapter: unload a LoRA adapter from
88
+ the serving engine."""
89
handler: OpenAIServingModels = models(raw_request)
90
response = await handler.unload_lora_adapter(request)
91
if isinstance(response, ErrorResponse):
vllm/entrypoints/serve/sagemaker/api_router.py
+59
-2
@@ -1,11 +1,11 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
import json
4
+import logging
5
from collections.abc import Awaitable, Callable
6
from http import HTTPStatus
7
from typing import Any
8
8
-import model_hosting_container_standards.sagemaker as sagemaker_standards
9
import pydantic
10
from fastapi import APIRouter, Depends, FastAPI, HTTPException, Request
11
from fastapi.responses import JSONResponse, Response
@@ -27,11 +27,55 @@ GetHandlerFn = Callable[[Request], BaseServing | None]
27
EndpointFn = Callable[[RequestType, Request], Awaitable[Any]]
28
29
30
+def _snapshot_handler_levels() -> list[tuple[logging.Handler, int]]:
31
+ """Snapshot handler levels for loggers that may be affected by third-party
32
+ logging configuration side effects (e.g. model_hosting_container_standards
33
+ calling configure_root_logger() at import time)."""
34
+ loggers = [logging.getLogger(), logging.getLogger("vllm")]
35
+ seen: set[int] = set()
36
+ snapshot: list[tuple[logging.Handler, int]] = []
37
+ for logger in loggers:
38
+ for handler in logger.handlers:
39
+ if id(handler) not in seen:
40
+ seen.add(id(handler))
41
+ snapshot.append((handler, handler.level))
42
+ return snapshot
43
+
44
+
45
+def _restore_handler_levels(
46
+ snapshot: list[tuple[logging.Handler, int]],
47
+) -> None:
48
+ """Restore handler levels from a snapshot."""
49
+ for handler, level in snapshot:
50
+ handler.setLevel(level)
51
+
52
+
53
def attach_router(
54
app: FastAPI,
55
supported_tasks: tuple["SupportedTask", ...],
56
model_config: ModelConfig | None = None,
57
):
58
+ """Attach the SageMaker hosting endpoints to the API server.
59
+
60
+ Handler levels are snapshotted and restored because importing
61
+ model_hosting_container_standards may reconfigure root logging.
62
+ """
63
+ snapshot = _snapshot_handler_levels()
64
+ try:
65
+ _attach_router(app, supported_tasks, model_config)
66
+ finally:
67
+ _restore_handler_levels(snapshot)
68
+
69
+
70
+def _attach_router(
71
+ app: FastAPI,
72
+ supported_tasks: tuple["SupportedTask", ...],
73
+ model_config: ModelConfig | None = None,
74
+):
75
+ """Register the SageMaker hosting routes (/ping, /invocations) on the
76
+ app."""
77
+ import model_hosting_container_standards.sagemaker as sagemaker_standards
78
+
79
router = APIRouter()
80
81
# NOTE: Construct the TypeAdapters only once
@@ -99,4 +143,17 @@ def attach_router(
143
144
145
def sagemaker_standards_bootstrap(app: FastAPI) -> FastAPI:
102
- return sagemaker_standards.bootstrap(app)
146
+ """Bootstrap the app with the SageMaker hosting standards.
147
+
148
+ Handler levels are restored right after the import because importing
149
+ model_hosting_container_standards may reconfigure root logging, and
150
+ bootstrap must run with the original levels.
151
+ """
152
+ snapshot = _snapshot_handler_levels()
153
+ try:
154
+ import model_hosting_container_standards.sagemaker as sagemaker_standards
155
+
156
+ app = sagemaker_standards.bootstrap(app)
157
+ finally:
158
+ _restore_handler_levels(snapshot)
159
+ return app