A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 9, 13:33 UTC
[Frontend] Cleanup api server (#33158)
Signed-off-by: wang.yuqi <yuqi.wang@daocloud.io> Signed-off-by: wang.yuqi <noooop@126.com>
wang.yuqi committed
Jan 27, 2026 at 23:18 UTC
7cbbca9aaa5fd5c42efd22994b14e7b3336fe3da
5 files changed
+471
-425
vllm/entrypoints/openai/api_server.py
+23
-424
@@ -1,34 +1,23 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
-import asyncio
4
-import hashlib
3
import importlib
4
import inspect
7
-import json
5
import multiprocessing
6
import multiprocessing.forkserver as forkserver
7
import os
11
-import secrets
8
import signal
9
import socket
10
import tempfile
15
-import uuid
11
from argparse import Namespace
17
-from collections.abc import AsyncIterator, Awaitable
12
+from collections.abc import AsyncIterator
13
from contextlib import asynccontextmanager
19
-from http import HTTPStatus
14
from typing import Any
15
22
-import model_hosting_container_standards.sagemaker as sagemaker_standards
23
-import pydantic
16
import uvloop
25
-from fastapi import APIRouter, FastAPI, HTTPException, Request
17
+from fastapi import FastAPI, HTTPException
18
from fastapi.exceptions import RequestValidationError
19
from fastapi.middleware.cors import CORSMiddleware
28
-from fastapi.responses import JSONResponse
29
-from starlette.concurrency import iterate_in_threadpool
30
-from starlette.datastructures import URL, Headers, MutableHeaders, State
31
-from starlette.types import ASGIApp, Message, Receive, Scope, Send
20
+from starlette.datastructures import State
21
22
import vllm.envs as envs
23
from vllm.engine.arg_utils import AsyncEngineArgs
@@ -37,15 +26,16 @@ from vllm.entrypoints.chat_utils import load_chat_template
26
from vllm.entrypoints.launcher import serve_http
27
from vllm.entrypoints.logger import RequestLogger
28
from vllm.entrypoints.openai.cli_args import make_arg_parser, validate_parsed_serve_args
40
-from vllm.entrypoints.openai.engine.protocol import (
41
- ErrorInfo,
42
- ErrorResponse,
43
-)
44
-from vllm.entrypoints.openai.engine.serving import OpenAIServing
29
from vllm.entrypoints.openai.models.protocol import BaseModelPath
46
-from vllm.entrypoints.openai.models.serving import (
47
- OpenAIServingModels,
30
+from vllm.entrypoints.openai.models.serving import OpenAIServingModels
31
+from vllm.entrypoints.openai.server_utils import (
32
+ get_uvicorn_log_config,
33
+ http_exception_handler,
34
+ lifespan,
35
+ log_response,
36
+ validation_exception_handler,
37
)
38
+from vllm.entrypoints.sagemaker.api_router import sagemaker_standards_bootstrap
39
from vllm.entrypoints.serve.elastic_ep.middleware import (
40
ScalingMiddleware,
41
)
@@ -55,16 +45,13 @@ from vllm.entrypoints.utils import (
45
log_non_default_args,
46
log_version_and_model,
47
process_lora_modules,
58
- sanitize_message,
48
)
60
-from vllm.exceptions import VLLMValidationError
49
from vllm.logger import init_logger
50
from vllm.reasoning import ReasoningParserManager
51
from vllm.tasks import POOLING_TASKS, SupportedTask
52
from vllm.tool_parsers import ToolParserManager
53
from vllm.usage.usage_lib import UsageContext
54
from vllm.utils.argparse_utils import FlexibleArgumentParser
67
-from vllm.utils.gc_utils import freeze_gc_heap
55
from vllm.utils.network_utils import is_valid_ipv6_address
56
from vllm.utils.system_utils import decorate_logs, set_ulimit
57
from vllm.version import __version__ as VLLM_VERSION
@@ -75,39 +62,6 @@ prometheus_multiproc_dir: tempfile.TemporaryDirectory
62
logger = init_logger("vllm.entrypoints.openai.api_server")
63
64
78
-_running_tasks: set[asyncio.Task] = set()
79
-
80
-
81
-@asynccontextmanager
82
-async def lifespan(app: FastAPI):
83
- try:
84
- if app.state.log_stats:
85
- engine_client: EngineClient = app.state.engine_client
86
-
87
- async def _force_log():
88
- while True:
89
- await asyncio.sleep(envs.VLLM_LOG_STATS_INTERVAL)
90
- await engine_client.do_log_stats()
91
-
92
- task = asyncio.create_task(_force_log())
93
- _running_tasks.add(task)
94
- task.add_done_callback(_running_tasks.remove)
95
- else:
96
- task = None
97
-
98
- # Mark the startup heap as static so that it's ignored by GC.
99
- # Reduces pause times of oldest generation collections.
100
- freeze_gc_heap()
101
- try:
102
- yield
103
- finally:
104
- if task is not None:
105
- task.cancel()
106
- finally:
107
- # Ensure app state including engine ref is gc'd
108
- del app.state
109
-
110
-
65
@asynccontextmanager
66
async def build_async_engine_client(
67
args: Namespace,
@@ -197,313 +151,6 @@ async def build_async_engine_client_from_engine_args(
151
async_llm.shutdown()
152
153
200
-router = APIRouter()
201
-
202
-
203
-def base(request: Request) -> OpenAIServing:
204
- # Reuse the existing instance
205
- return tokenization(request)
206
-
207
-
208
-def tokenization(request: Request) -> OpenAIServingTokenization:
209
- return request.app.state.openai_serving_tokenization
210
-
211
-
212
-def engine_client(request: Request) -> EngineClient:
213
- return request.app.state.engine_client
214
-
215
-
216
-@router.get("/load")
217
-async def get_server_load_metrics(request: Request):
218
- # This endpoint returns the current server load metrics.
219
- # It tracks requests utilizing the GPU from the following routes:
220
- # - /v1/responses
221
- # - /v1/responses/{response_id}
222
- # - /v1/responses/{response_id}/cancel
223
- # - /v1/messages
224
- # - /v1/chat/completions
225
- # - /v1/completions
226
- # - /v1/audio/transcriptions
227
- # - /v1/audio/translations
228
- # - /v1/embeddings
229
- # - /pooling
230
- # - /classify
231
- # - /score
232
- # - /v1/score
233
- # - /rerank
234
- # - /v1/rerank
235
- # - /v2/rerank
236
- return JSONResponse(content={"server_load": request.app.state.server_load_metrics})
237
-
238
-
239
-@router.get("/version")
240
-async def show_version():
241
- ver = {"version": VLLM_VERSION}
242
- return JSONResponse(content=ver)
243
-
244
-
245
-def load_log_config(log_config_file: str | None) -> dict | None:
246
- if not log_config_file:
247
- return None
248
- try:
249
- with open(log_config_file) as f:
250
- return json.load(f)
251
- except Exception as e:
252
- logger.warning(
253
- "Failed to load log config from file %s: error %s", log_config_file, e
254
- )
255
- return None
256
-
257
-
258
-def get_uvicorn_log_config(args: Namespace) -> dict | None:
259
- """
260
- Get the uvicorn log config based on the provided arguments.
261
-
262
- Priority:
263
- 1. If log_config_file is specified, use it
264
- 2. If disable_access_log_for_endpoints is specified, create a config with
265
- the access log filter
266
- 3. Otherwise, return None (use uvicorn defaults)
267
- """
268
- # First, try to load from file if specified
269
- log_config = load_log_config(args.log_config_file)
270
- if log_config is not None:
271
- return log_config
272
-
273
- # If endpoints to filter are specified, create a config with the filter
274
- if args.disable_access_log_for_endpoints:
275
- from vllm.logging_utils import create_uvicorn_log_config
276
-
277
- # Parse comma-separated string into list
278
- excluded_paths = [
279
- p.strip()
280
- for p in args.disable_access_log_for_endpoints.split(",")
281
- if p.strip()
282
- ]
283
- return create_uvicorn_log_config(
284
- excluded_paths=excluded_paths,
285
- log_level=args.uvicorn_log_level,
286
- )
287
-
288
- return None
289
-
290
-
291
-class AuthenticationMiddleware:
292
- """
293
- Pure ASGI middleware that authenticates each request by checking
294
- if the Authorization Bearer token exists and equals anyof "{api_key}".
295
-
296
- Notes
297
- -----
298
- There are two cases in which authentication is skipped:
299
- 1. The HTTP method is OPTIONS.
300
- 2. The request path doesn't start with /v1 (e.g. /health).
301
- """
302
-
303
- def __init__(self, app: ASGIApp, tokens: list[str]) -> None:
304
- self.app = app
305
- self.api_tokens = [hashlib.sha256(t.encode("utf-8")).digest() for t in tokens]
306
-
307
- def verify_token(self, headers: Headers) -> bool:
308
- authorization_header_value = headers.get("Authorization")
309
- if not authorization_header_value:
310
- return False
311
-
312
- scheme, _, param = authorization_header_value.partition(" ")
313
- if scheme.lower() != "bearer":
314
- return False
315
-
316
- param_hash = hashlib.sha256(param.encode("utf-8")).digest()
317
-
318
- token_match = False
319
- for token_hash in self.api_tokens:
320
- token_match |= secrets.compare_digest(param_hash, token_hash)
321
-
322
- return token_match
323
-
324
- def __call__(self, scope: Scope, receive: Receive, send: Send) -> Awaitable[None]:
325
- if scope["type"] not in ("http", "websocket") or scope["method"] == "OPTIONS":
326
- # scope["type"] can be "lifespan" or "startup" for example,
327
- # in which case we don't need to do anything
328
- return self.app(scope, receive, send)
329
- root_path = scope.get("root_path", "")
330
- url_path = URL(scope=scope).path.removeprefix(root_path)
331
- headers = Headers(scope=scope)
332
- # Type narrow to satisfy mypy.
333
- if url_path.startswith("/v1") and not self.verify_token(headers):
334
- response = JSONResponse(content={"error": "Unauthorized"}, status_code=401)
335
- return response(scope, receive, send)
336
- return self.app(scope, receive, send)
337
-
338
-
339
-class XRequestIdMiddleware:
340
- """
341
- Middleware the set's the X-Request-Id header for each response
342
- to a random uuid4 (hex) value if the header isn't already
343
- present in the request, otherwise use the provided request id.
344
- """
345
-
346
- def __init__(self, app: ASGIApp) -> None:
347
- self.app = app
348
-
349
- def __call__(self, scope: Scope, receive: Receive, send: Send) -> Awaitable[None]:
350
- if scope["type"] not in ("http", "websocket"):
351
- return self.app(scope, receive, send)
352
-
353
- # Extract the request headers.
354
- request_headers = Headers(scope=scope)
355
-
356
- async def send_with_request_id(message: Message) -> None:
357
- """
358
- Custom send function to mutate the response headers
359
- and append X-Request-Id to it.
360
- """
361
- if message["type"] == "http.response.start":
362
- response_headers = MutableHeaders(raw=message["headers"])
363
- request_id = request_headers.get("X-Request-Id", uuid.uuid4().hex)
364
- response_headers.append("X-Request-Id", request_id)
365
- await send(message)
366
-
367
- return self.app(scope, receive, send_with_request_id)
368
-
369
-
370
-def _extract_content_from_chunk(chunk_data: dict) -> str:
371
- """Extract content from a streaming response chunk."""
372
- try:
373
- from vllm.entrypoints.openai.chat_completion.protocol import (
374
- ChatCompletionStreamResponse,
375
- )
376
- from vllm.entrypoints.openai.completion.protocol import (
377
- CompletionStreamResponse,
378
- )
379
-
380
- # Try using Completion types for type-safe parsing
381
- if chunk_data.get("object") == "chat.completion.chunk":
382
- chat_response = ChatCompletionStreamResponse.model_validate(chunk_data)
383
- if chat_response.choices and chat_response.choices[0].delta.content:
384
- return chat_response.choices[0].delta.content
385
- elif chunk_data.get("object") == "text_completion":
386
- completion_response = CompletionStreamResponse.model_validate(chunk_data)
387
- if completion_response.choices and completion_response.choices[0].text:
388
- return completion_response.choices[0].text
389
- except pydantic.ValidationError:
390
- # Fallback to manual parsing
391
- if "choices" in chunk_data and chunk_data["choices"]:
392
- choice = chunk_data["choices"][0]
393
- if "delta" in choice and choice["delta"].get("content"):
394
- return choice["delta"]["content"]
395
- elif choice.get("text"):
396
- return choice["text"]
397
- return ""
398
-
399
-
400
-class SSEDecoder:
401
- """Robust Server-Sent Events decoder for streaming responses."""
402
-
403
- def __init__(self):
404
- self.buffer = ""
405
- self.content_buffer = []
406
-
407
- def decode_chunk(self, chunk: bytes) -> list[dict]:
408
- """Decode a chunk of SSE data and return parsed events."""
409
- import json
410
-
411
- try:
412
- chunk_str = chunk.decode("utf-8")
413
- except UnicodeDecodeError:
414
- # Skip malformed chunks
415
- return []
416
-
417
- self.buffer += chunk_str
418
- events = []
419
-
420
- # Process complete lines
421
- while "\n" in self.buffer:
422
- line, self.buffer = self.buffer.split("\n", 1)
423
- line = line.rstrip("\r") # Handle CRLF
424
-
425
- if line.startswith("data: "):
426
- data_str = line[6:].strip()
427
- if data_str == "[DONE]":
428
- events.append({"type": "done"})
429
- elif data_str:
430
- try:
431
- event_data = json.loads(data_str)
432
- events.append({"type": "data", "data": event_data})
433
- except json.JSONDecodeError:
434
- # Skip malformed JSON
435
- continue
436
-
437
- return events
438
-
439
- def extract_content(self, event_data: dict) -> str:
440
- """Extract content from event data."""
441
- return _extract_content_from_chunk(event_data)
442
-
443
- def add_content(self, content: str) -> None:
444
- """Add content to the buffer."""
445
- if content:
446
- self.content_buffer.append(content)
447
-
448
- def get_complete_content(self) -> str:
449
- """Get the complete buffered content."""
450
- return "".join(self.content_buffer)
451
-
452
-
453
-def _log_streaming_response(response, response_body: list) -> None:
454
- """Log streaming response with robust SSE parsing."""
455
- from starlette.concurrency import iterate_in_threadpool
456
-
457
- sse_decoder = SSEDecoder()
458
- chunk_count = 0
459
-
460
- def buffered_iterator():
461
- nonlocal chunk_count
462
-
463
- for chunk in response_body:
464
- chunk_count += 1
465
- yield chunk
466
-
467
- # Parse SSE events from chunk
468
- events = sse_decoder.decode_chunk(chunk)
469
-
470
- for event in events:
471
- if event["type"] == "data":
472
- content = sse_decoder.extract_content(event["data"])
473
- sse_decoder.add_content(content)
474
- elif event["type"] == "done":
475
- # Log complete content when done
476
- full_content = sse_decoder.get_complete_content()
477
- if full_content:
478
- # Truncate if too long
479
- if len(full_content) > 2048:
480
- full_content = full_content[:2048] + ""
481
- "...[truncated]"
482
- logger.info(
483
- "response_body={streaming_complete: content=%r, chunks=%d}",
484
- full_content,
485
- chunk_count,
486
- )
487
- else:
488
- logger.info(
489
- "response_body={streaming_complete: no_content, chunks=%d}",
490
- chunk_count,
491
- )
492
- return
493
-
494
- response.body_iterator = iterate_in_threadpool(buffered_iterator())
495
- logger.info("response_body={streaming_started: chunks=%d}", len(response_body))
496
-
497
-
498
-def _log_non_streaming_response(response_body: list) -> None:
499
- """Log non-streaming response."""
500
- try:
501
- decoded_body = response_body[0].decode()
502
- logger.info("response_body={%s}", decoded_body)
503
- except UnicodeDecodeError:
504
- logger.info("response_body={<binary_data>}")
505
-
506
-
154
def build_app(args: Namespace, supported_tasks: tuple["SupportedTask", ...]) -> FastAPI:
155
if args.disable_fastapi_docs:
156
app = FastAPI(
@@ -514,7 +161,10 @@ def build_app(args: Namespace, supported_tasks: tuple["SupportedTask", ...]) ->
161
else:
162
app = FastAPI(lifespan=lifespan)
163
app.state.args = args
517
- app.include_router(router)
164
+
165
+ from vllm.entrypoints.openai.basic.api_router import register_basic_api_routers
166
+
167
+ register_basic_api_routers(app)
168
169
from vllm.entrypoints.serve import register_vllm_serve_api_routers
170
@@ -560,51 +210,18 @@ def build_app(args: Namespace, supported_tasks: tuple["SupportedTask", ...]) ->
210
allow_headers=args.allowed_headers,
211
)
212
563
- @app.exception_handler(HTTPException)
564
- async def http_exception_handler(_: Request, exc: HTTPException):
565
- err = ErrorResponse(
566
- error=ErrorInfo(
567
- message=sanitize_message(exc.detail),
568
- type=HTTPStatus(exc.status_code).phrase,
569
- code=exc.status_code,
570
- )
571
- )
572
- return JSONResponse(err.model_dump(), status_code=exc.status_code)
573
-
574
- @app.exception_handler(RequestValidationError)
575
- async def validation_exception_handler(_: Request, exc: RequestValidationError):
576
- param = None
577
- errors = exc.errors()
578
- for error in errors:
579
- if "ctx" in error and "error" in error["ctx"]:
580
- ctx_error = error["ctx"]["error"]
581
- if isinstance(ctx_error, VLLMValidationError):
582
- param = ctx_error.parameter
583
- break
584
-
585
- exc_str = str(exc)
586
- errors_str = str(errors)
587
-
588
- if errors and errors_str and errors_str != exc_str:
589
- message = f"{exc_str} {errors_str}"
590
- else:
591
- message = exc_str
592
-
593
- err = ErrorResponse(
594
- error=ErrorInfo(
595
- message=sanitize_message(message),
596
- type=HTTPStatus.BAD_REQUEST.phrase,
597
- code=HTTPStatus.BAD_REQUEST,
598
- param=param,
599
- )
600
- )
601
- return JSONResponse(err.model_dump(), status_code=HTTPStatus.BAD_REQUEST)
213
+ app.exception_handler(HTTPException)(http_exception_handler)
214
+ app.exception_handler(RequestValidationError)(validation_exception_handler)
215
216
# Ensure --api-key option from CLI takes precedence over VLLM_API_KEY
217
if tokens := [key for key in (args.api_key or [envs.VLLM_API_KEY]) if key]:
218
+ from vllm.entrypoints.openai.server_utils import AuthenticationMiddleware
219
+
220
app.add_middleware(AuthenticationMiddleware, tokens=tokens)
221
222
if args.enable_request_id_headers:
223
+ from vllm.entrypoints.openai.server_utils import XRequestIdMiddleware
224
+
225
app.add_middleware(XRequestIdMiddleware)
226
227
# Add scaling middleware to check for scaling state
@@ -616,24 +233,7 @@ def build_app(args: Namespace, supported_tasks: tuple["SupportedTask", ...]) ->
233
"This can include sensitive information and should be "
234
"avoided in production."
235
)
619
-
620
- @app.middleware("http")
621
- async def log_response(request: Request, call_next):
622
- response = await call_next(request)
623
- response_body = [section async for section in response.body_iterator]
624
- response.body_iterator = iterate_in_threadpool(iter(response_body))
625
- # Check if this is a streaming response by looking at content-type
626
- content_type = response.headers.get("content-type", "")
627
- is_streaming = content_type == "text/event-stream; charset=utf-8"
628
-
629
- # Log response body based on type
630
- if not response_body:
631
- logger.info("response_body={<empty>}")
632
- elif is_streaming:
633
- _log_streaming_response(response, response_body)
634
- else:
635
- _log_non_streaming_response(response_body)
636
- return response
236
+ app.middleware("http")(log_response)
237
238
for middleware in args.middleware:
239
module_path, object_name = middleware.rsplit(".", 1)
@@ -647,8 +247,7 @@ def build_app(args: Namespace, supported_tasks: tuple["SupportedTask", ...]) ->
247
f"Invalid middleware {middleware}. Must be a function or a class."
248
)
249
650
- app = sagemaker_standards.bootstrap(app)
651
-
250
+ app = sagemaker_standards_bootstrap(app)
251
return app
252
253
vllm/entrypoints/openai/basic/__init__.py
vllm/entrypoints/openai/basic/api_router.py
new
+61
@@ -0,0 +1,61 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+from fastapi import APIRouter, FastAPI, Request
5
+from fastapi.responses import JSONResponse
6
+
7
+from vllm.engine.protocol import EngineClient
8
+from vllm.entrypoints.openai.engine.serving import OpenAIServing
9
+from vllm.entrypoints.serve.tokenize.serving import OpenAIServingTokenization
10
+from vllm.logger import init_logger
11
+from vllm.version import __version__ as VLLM_VERSION
12
+
13
+router = APIRouter()
14
+
15
+logger = init_logger(__name__)
16
+
17
+
18
+def base(request: Request) -> OpenAIServing:
19
+ # Reuse the existing instance
20
+ return tokenization(request)
21
+
22
+
23
+def tokenization(request: Request) -> OpenAIServingTokenization:
24
+ return request.app.state.openai_serving_tokenization
25
+
26
+
27
+def engine_client(request: Request) -> EngineClient:
28
+ return request.app.state.engine_client
29
+
30
+
31
+@router.get("/load")
32
+async def get_server_load_metrics(request: Request):
33
+ # This endpoint returns the current server load metrics.
34
+ # It tracks requests utilizing the GPU from the following routes:
35
+ # - /v1/responses
36
+ # - /v1/responses/{response_id}
37
+ # - /v1/responses/{response_id}/cancel
38
+ # - /v1/messages
39
+ # - /v1/chat/completions
40
+ # - /v1/completions
41
+ # - /v1/audio/transcriptions
42
+ # - /v1/audio/translations
43
+ # - /v1/embeddings
44
+ # - /pooling
45
+ # - /classify
46
+ # - /score
47
+ # - /v1/score
48
+ # - /rerank
49
+ # - /v1/rerank
50
+ # - /v2/rerank
51
+ return JSONResponse(content={"server_load": request.app.state.server_load_metrics})
52
+
53
+
54
+@router.get("/version")
55
+async def show_version():
56
+ ver = {"version": VLLM_VERSION}
57
+ return JSONResponse(content=ver)
58
+
59
+
60
+def register_basic_api_routers(app: FastAPI):
61
+ app.include_router(router)
vllm/entrypoints/openai/server_utils.py
new
+382
@@ -0,0 +1,382 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+import asyncio
4
+import hashlib
5
+import json
6
+import secrets
7
+import uuid
8
+from argparse import Namespace
9
+from collections.abc import Awaitable
10
+from contextlib import asynccontextmanager
11
+from http import HTTPStatus
12
+
13
+import pydantic
14
+from fastapi import FastAPI, HTTPException, Request
15
+from fastapi.exceptions import RequestValidationError
16
+from fastapi.responses import JSONResponse
17
+from starlette.concurrency import iterate_in_threadpool
18
+from starlette.datastructures import URL, Headers, MutableHeaders
19
+from starlette.types import ASGIApp, Message, Receive, Scope, Send
20
+
21
+from vllm import envs
22
+from vllm.engine.protocol import EngineClient
23
+from vllm.entrypoints.openai.engine.protocol import ErrorInfo, ErrorResponse
24
+from vllm.entrypoints.utils import sanitize_message
25
+from vllm.exceptions import VLLMValidationError
26
+from vllm.logger import init_logger
27
+from vllm.utils.gc_utils import freeze_gc_heap
28
+
29
+logger = init_logger("vllm.entrypoints.openai.server_utils")
30
+
31
+
32
+class AuthenticationMiddleware:
33
+ """
34
+ Pure ASGI middleware that authenticates each request by checking
35
+ if the Authorization Bearer token exists and equals anyof "{api_key}".
36
+
37
+ Notes
38
+ -----
39
+ There are two cases in which authentication is skipped:
40
+ 1. The HTTP method is OPTIONS.
41
+ 2. The request path doesn't start with /v1 (e.g. /health).
42
+ """
43
+
44
+ def __init__(self, app: ASGIApp, tokens: list[str]) -> None:
45
+ self.app = app
46
+ self.api_tokens = [hashlib.sha256(t.encode("utf-8")).digest() for t in tokens]
47
+
48
+ def verify_token(self, headers: Headers) -> bool:
49
+ authorization_header_value = headers.get("Authorization")
50
+ if not authorization_header_value:
51
+ return False
52
+
53
+ scheme, _, param = authorization_header_value.partition(" ")
54
+ if scheme.lower() != "bearer":
55
+ return False
56
+
57
+ param_hash = hashlib.sha256(param.encode("utf-8")).digest()
58
+
59
+ token_match = False
60
+ for token_hash in self.api_tokens:
61
+ token_match |= secrets.compare_digest(param_hash, token_hash)
62
+
63
+ return token_match
64
+
65
+ def __call__(self, scope: Scope, receive: Receive, send: Send) -> Awaitable[None]:
66
+ if scope["type"] not in ("http", "websocket") or scope["method"] == "OPTIONS":
67
+ # scope["type"] can be "lifespan" or "startup" for example,
68
+ # in which case we don't need to do anything
69
+ return self.app(scope, receive, send)
70
+ root_path = scope.get("root_path", "")
71
+ url_path = URL(scope=scope).path.removeprefix(root_path)
72
+ headers = Headers(scope=scope)
73
+ # Type narrow to satisfy mypy.
74
+ if url_path.startswith("/v1") and not self.verify_token(headers):
75
+ response = JSONResponse(content={"error": "Unauthorized"}, status_code=401)
76
+ return response(scope, receive, send)
77
+ return self.app(scope, receive, send)
78
+
79
+
80
+class XRequestIdMiddleware:
81
+ """
82
+ Middleware the set's the X-Request-Id header for each response
83
+ to a random uuid4 (hex) value if the header isn't already
84
+ present in the request, otherwise use the provided request id.
85
+ """
86
+
87
+ def __init__(self, app: ASGIApp) -> None:
88
+ self.app = app
89
+
90
+ def __call__(self, scope: Scope, receive: Receive, send: Send) -> Awaitable[None]:
91
+ if scope["type"] not in ("http", "websocket"):
92
+ return self.app(scope, receive, send)
93
+
94
+ # Extract the request headers.
95
+ request_headers = Headers(scope=scope)
96
+
97
+ async def send_with_request_id(message: Message) -> None:
98
+ """
99
+ Custom send function to mutate the response headers
100
+ and append X-Request-Id to it.
101
+ """
102
+ if message["type"] == "http.response.start":
103
+ response_headers = MutableHeaders(raw=message["headers"])
104
+ request_id = request_headers.get("X-Request-Id", uuid.uuid4().hex)
105
+ response_headers.append("X-Request-Id", request_id)
106
+ await send(message)
107
+
108
+ return self.app(scope, receive, send_with_request_id)
109
+
110
+
111
+def load_log_config(log_config_file: str | None) -> dict | None:
112
+ if not log_config_file:
113
+ return None
114
+ try:
115
+ with open(log_config_file) as f:
116
+ return json.load(f)
117
+ except Exception as e:
118
+ logger.warning(
119
+ "Failed to load log config from file %s: error %s", log_config_file, e
120
+ )
121
+ return None
122
+
123
+
124
+def get_uvicorn_log_config(args: Namespace) -> dict | None:
125
+ """
126
+ Get the uvicorn log config based on the provided arguments.
127
+
128
+ Priority:
129
+ 1. If log_config_file is specified, use it
130
+ 2. If disable_access_log_for_endpoints is specified, create a config with
131
+ the access log filter
132
+ 3. Otherwise, return None (use uvicorn defaults)
133
+ """
134
+ # First, try to load from file if specified
135
+ log_config = load_log_config(args.log_config_file)
136
+ if log_config is not None:
137
+ return log_config
138
+
139
+ # If endpoints to filter are specified, create a config with the filter
140
+ if args.disable_access_log_for_endpoints:
141
+ from vllm.logging_utils import create_uvicorn_log_config
142
+
143
+ # Parse comma-separated string into list
144
+ excluded_paths = [
145
+ p.strip()
146
+ for p in args.disable_access_log_for_endpoints.split(",")
147
+ if p.strip()
148
+ ]
149
+ return create_uvicorn_log_config(
150
+ excluded_paths=excluded_paths,
151
+ log_level=args.uvicorn_log_level,
152
+ )
153
+
154
+ return None
155
+
156
+
157
+def _extract_content_from_chunk(chunk_data: dict) -> str:
158
+ """Extract content from a streaming response chunk."""
159
+ try:
160
+ from vllm.entrypoints.openai.chat_completion.protocol import (
161
+ ChatCompletionStreamResponse,
162
+ )
163
+ from vllm.entrypoints.openai.completion.protocol import (
164
+ CompletionStreamResponse,
165
+ )
166
+
167
+ # Try using Completion types for type-safe parsing
168
+ if chunk_data.get("object") == "chat.completion.chunk":
169
+ chat_response = ChatCompletionStreamResponse.model_validate(chunk_data)
170
+ if chat_response.choices and chat_response.choices[0].delta.content:
171
+ return chat_response.choices[0].delta.content
172
+ elif chunk_data.get("object") == "text_completion":
173
+ completion_response = CompletionStreamResponse.model_validate(chunk_data)
174
+ if completion_response.choices and completion_response.choices[0].text:
175
+ return completion_response.choices[0].text
176
+ except pydantic.ValidationError:
177
+ # Fallback to manual parsing
178
+ if "choices" in chunk_data and chunk_data["choices"]:
179
+ choice = chunk_data["choices"][0]
180
+ if "delta" in choice and choice["delta"].get("content"):
181
+ return choice["delta"]["content"]
182
+ elif choice.get("text"):
183
+ return choice["text"]
184
+ return ""
185
+
186
+
187
+class SSEDecoder:
188
+ """Robust Server-Sent Events decoder for streaming responses."""
189
+
190
+ def __init__(self):
191
+ self.buffer = ""
192
+ self.content_buffer = []
193
+
194
+ def decode_chunk(self, chunk: bytes) -> list[dict]:
195
+ """Decode a chunk of SSE data and return parsed events."""
196
+ import json
197
+
198
+ try:
199
+ chunk_str = chunk.decode("utf-8")
200
+ except UnicodeDecodeError:
201
+ # Skip malformed chunks
202
+ return []
203
+
204
+ self.buffer += chunk_str
205
+ events = []
206
+
207
+ # Process complete lines
208
+ while "\n" in self.buffer:
209
+ line, self.buffer = self.buffer.split("\n", 1)
210
+ line = line.rstrip("\r") # Handle CRLF
211
+
212
+ if line.startswith("data: "):
213
+ data_str = line[6:].strip()
214
+ if data_str == "[DONE]":
215
+ events.append({"type": "done"})
216
+ elif data_str:
217
+ try:
218
+ event_data = json.loads(data_str)
219
+ events.append({"type": "data", "data": event_data})
220
+ except json.JSONDecodeError:
221
+ # Skip malformed JSON
222
+ continue
223
+
224
+ return events
225
+
226
+ def extract_content(self, event_data: dict) -> str:
227
+ """Extract content from event data."""
228
+ return _extract_content_from_chunk(event_data)
229
+
230
+ def add_content(self, content: str) -> None:
231
+ """Add content to the buffer."""
232
+ if content:
233
+ self.content_buffer.append(content)
234
+
235
+ def get_complete_content(self) -> str:
236
+ """Get the complete buffered content."""
237
+ return "".join(self.content_buffer)
238
+
239
+
240
+def _log_streaming_response(response, response_body: list) -> None:
241
+ """Log streaming response with robust SSE parsing."""
242
+ from starlette.concurrency import iterate_in_threadpool
243
+
244
+ sse_decoder = SSEDecoder()
245
+ chunk_count = 0
246
+
247
+ def buffered_iterator():
248
+ nonlocal chunk_count
249
+
250
+ for chunk in response_body:
251
+ chunk_count += 1
252
+ yield chunk
253
+
254
+ # Parse SSE events from chunk
255
+ events = sse_decoder.decode_chunk(chunk)
256
+
257
+ for event in events:
258
+ if event["type"] == "data":
259
+ content = sse_decoder.extract_content(event["data"])
260
+ sse_decoder.add_content(content)
261
+ elif event["type"] == "done":
262
+ # Log complete content when done
263
+ full_content = sse_decoder.get_complete_content()
264
+ if full_content:
265
+ # Truncate if too long
266
+ if len(full_content) > 2048:
267
+ full_content = full_content[:2048] + ""
268
+ "...[truncated]"
269
+ logger.info(
270
+ "response_body={streaming_complete: content=%r, chunks=%d}",
271
+ full_content,
272
+ chunk_count,
273
+ )
274
+ else:
275
+ logger.info(
276
+ "response_body={streaming_complete: no_content, chunks=%d}",
277
+ chunk_count,
278
+ )
279
+ return
280
+
281
+ response.body_iterator = iterate_in_threadpool(buffered_iterator())
282
+ logger.info("response_body={streaming_started: chunks=%d}", len(response_body))
283
+
284
+
285
+def _log_non_streaming_response(response_body: list) -> None:
286
+ """Log non-streaming response."""
287
+ try:
288
+ decoded_body = response_body[0].decode()
289
+ logger.info("response_body={%s}", decoded_body)
290
+ except UnicodeDecodeError:
291
+ logger.info("response_body={<binary_data>}")
292
+
293
+
294
+async def log_response(request: Request, call_next):
295
+ response = await call_next(request)
296
+ response_body = [section async for section in response.body_iterator]
297
+ response.body_iterator = iterate_in_threadpool(iter(response_body))
298
+ # Check if this is a streaming response by looking at content-type
299
+ content_type = response.headers.get("content-type", "")
300
+ is_streaming = content_type == "text/event-stream; charset=utf-8"
301
+
302
+ # Log response body based on type
303
+ if not response_body:
304
+ logger.info("response_body={<empty>}")
305
+ elif is_streaming:
306
+ _log_streaming_response(response, response_body)
307
+ else:
308
+ _log_non_streaming_response(response_body)
309
+ return response
310
+
311
+
312
+async def http_exception_handler(_: Request, exc: HTTPException):
313
+ err = ErrorResponse(
314
+ error=ErrorInfo(
315
+ message=sanitize_message(exc.detail),
316
+ type=HTTPStatus(exc.status_code).phrase,
317
+ code=exc.status_code,
318
+ )
319
+ )
320
+ return JSONResponse(err.model_dump(), status_code=exc.status_code)
321
+
322
+
323
+async def validation_exception_handler(_: Request, exc: RequestValidationError):
324
+ param = None
325
+ errors = exc.errors()
326
+ for error in errors:
327
+ if "ctx" in error and "error" in error["ctx"]:
328
+ ctx_error = error["ctx"]["error"]
329
+ if isinstance(ctx_error, VLLMValidationError):
330
+ param = ctx_error.parameter
331
+ break
332
+
333
+ exc_str = str(exc)
334
+ errors_str = str(errors)
335
+
336
+ if errors and errors_str and errors_str != exc_str:
337
+ message = f"{exc_str} {errors_str}"
338
+ else:
339
+ message = exc_str
340
+
341
+ err = ErrorResponse(
342
+ error=ErrorInfo(
343
+ message=sanitize_message(message),
344
+ type=HTTPStatus.BAD_REQUEST.phrase,
345
+ code=HTTPStatus.BAD_REQUEST,
346
+ param=param,
347
+ )
348
+ )
349
+ return JSONResponse(err.model_dump(), status_code=HTTPStatus.BAD_REQUEST)
350
+
351
+
352
+_running_tasks: set[asyncio.Task] = set()
353
+
354
+
355
+@asynccontextmanager
356
+async def lifespan(app: FastAPI):
357
+ try:
358
+ if app.state.log_stats:
359
+ engine_client: EngineClient = app.state.engine_client
360
+
361
+ async def _force_log():
362
+ while True:
363
+ await asyncio.sleep(envs.VLLM_LOG_STATS_INTERVAL)
364
+ await engine_client.do_log_stats()
365
+
366
+ task = asyncio.create_task(_force_log())
367
+ _running_tasks.add(task)
368
+ task.add_done_callback(_running_tasks.remove)
369
+ else:
370
+ task = None
371
+
372
+ # Mark the startup heap as static so that it's ignored by GC.
373
+ # Reduces pause times of oldest generation collections.
374
+ freeze_gc_heap()
375
+ try:
376
+ yield
377
+ finally:
378
+ if task is not None:
379
+ task.cancel()
380
+ finally:
381
+ # Ensure app state including engine ref is gc'd
382
+ del app.state
vllm/entrypoints/sagemaker/api_router.py
+5
-1
@@ -10,7 +10,7 @@ import pydantic
10
from fastapi import APIRouter, Depends, FastAPI, HTTPException, Request
11
from fastapi.responses import JSONResponse, Response
12
13
-from vllm.entrypoints.openai.api_server import base
13
+from vllm.entrypoints.openai.basic.api_router import base
14
from vllm.entrypoints.openai.engine.protocol import ErrorResponse
15
from vllm.entrypoints.openai.engine.serving import OpenAIServing
16
from vllm.entrypoints.openai.utils import validate_json_request
@@ -158,3 +158,7 @@ def attach_router(app: FastAPI, supported_tasks: tuple["SupportedTask", ...]):
158
return JSONResponse(content=res.model_dump(), status_code=res.error.code)
159
160
app.include_router(router)
161
+
162
+
163
+def sagemaker_standards_bootstrap(app: FastAPI) -> FastAPI:
164
+ return sagemaker_standards.bootstrap(app)