A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 12, 03:30 UTC
[Refactor] [1/N] to simplify the vLLM serving architecture (#28040)
Signed-off-by: chaunceyjiang <chaunceyjiang@gmail.com>
Chauncey committed
Dec 3, 2025 at 17:26 UTC
3f42b05fbc53e50813a1619f5fc770f17ac2a1b6
27 files changed
+850
-455
tests/entrypoints/openai/test_basic.py
+1
-1
@@ -232,7 +232,7 @@ async def test_server_load(server: RemoteOpenAIServer):
232
@pytest.mark.asyncio
233
async def test_health_check_engine_dead_error():
234
# Import the health function directly to test it in isolation
235
- from vllm.entrypoints.openai.api_server import health
235
+ from vllm.entrypoints.serve.instrumentator.health import health
236
237
# Create a mock request that simulates what FastAPI would provide
238
mock_request = Mock(spec=Request)
vllm/entrypoints/api_server.py
+1
@@ -118,6 +118,7 @@ async def init_app(
118
)
119
)
120
app.state.engine_client = engine
121
+ app.state.args = args
122
return app
123
124
vllm/entrypoints/openai/api_server.py
+10
-445
@@ -20,21 +20,15 @@ from http import HTTPStatus
20
from typing import Annotated, Any, Literal
21
22
import model_hosting_container_standards.sagemaker as sagemaker_standards
23
-import prometheus_client
23
import pydantic
25
-import regex as re
24
import uvloop
25
from fastapi import APIRouter, Depends, FastAPI, Form, HTTPException, Query, Request
26
from fastapi.exceptions import RequestValidationError
27
from fastapi.middleware.cors import CORSMiddleware
28
from fastapi.responses import JSONResponse, Response, StreamingResponse
31
-from prometheus_client import make_asgi_app
32
-from prometheus_fastapi_instrumentator import Instrumentator
29
from starlette.concurrency import iterate_in_threadpool
30
from starlette.datastructures import URL, Headers, MutableHeaders, State
35
-from starlette.routing import Mount
31
from starlette.types import ASGIApp, Message, Receive, Scope, Send
37
-from typing_extensions import assert_never
32
33
import vllm.envs as envs
34
from vllm.config import VllmConfig
@@ -56,17 +50,11 @@ from vllm.entrypoints.openai.protocol import (
50
ChatCompletionResponse,
51
CompletionRequest,
52
CompletionResponse,
59
- DetokenizeRequest,
60
- DetokenizeResponse,
53
ErrorInfo,
54
ErrorResponse,
63
- GenerateRequest,
64
- GenerateResponse,
55
ResponsesRequest,
56
ResponsesResponse,
57
StreamingResponsesResponse,
68
- TokenizeRequest,
69
- TokenizeResponse,
58
TranscriptionRequest,
59
TranscriptionResponseVariant,
60
TranslationRequest,
@@ -80,8 +68,6 @@ from vllm.entrypoints.openai.serving_models import (
68
OpenAIServingModels,
69
)
70
from vllm.entrypoints.openai.serving_responses import OpenAIServingResponses
83
-from vllm.entrypoints.openai.serving_tokenization import OpenAIServingTokenization
84
-from vllm.entrypoints.openai.serving_tokens import ServingTokens
71
from vllm.entrypoints.openai.serving_transcription import (
72
OpenAIServingTranscription,
73
OpenAIServingTranslation,
@@ -92,6 +78,11 @@ from vllm.entrypoints.pooling.classify.serving import ServingClassification
78
from vllm.entrypoints.pooling.embed.serving import OpenAIServingEmbedding
79
from vllm.entrypoints.pooling.pooling.serving import OpenAIServingPooling
80
from vllm.entrypoints.pooling.score.serving import ServingScores
81
+from vllm.entrypoints.serve.disagg.serving import ServingTokens
82
+from vllm.entrypoints.serve.elastic_ep.middleware import (
83
+ ScalingMiddleware,
84
+)
85
+from vllm.entrypoints.serve.tokenize.serving import OpenAIServingTokenization
86
from vllm.entrypoints.tool_server import DemoToolServer, MCPToolServer, ToolServer
87
from vllm.entrypoints.utils import (
88
cli_env_setup,
@@ -109,8 +100,6 @@ from vllm.utils.argparse_utils import FlexibleArgumentParser
100
from vllm.utils.gc_utils import freeze_gc_heap
101
from vllm.utils.network_utils import is_valid_ipv6_address
102
from vllm.utils.system_utils import decorate_logs, set_ulimit
112
-from vllm.v1.engine.exceptions import EngineDeadError
113
-from vllm.v1.metrics.prometheus import get_prometheus_registry
103
from vllm.version import __version__ as VLLM_VERSION
104
105
prometheus_multiproc_dir: tempfile.TemporaryDirectory
@@ -245,39 +234,6 @@ async def build_async_engine_client_from_engine_args(
234
router = APIRouter()
235
236
248
-class PrometheusResponse(Response):
249
- media_type = prometheus_client.CONTENT_TYPE_LATEST
250
-
251
-
252
-def mount_metrics(app: FastAPI):
253
- """Mount prometheus metrics to a FastAPI app."""
254
-
255
- registry = get_prometheus_registry()
256
-
257
- # `response_class=PrometheusResponse` is needed to return an HTTP response
258
- # with header "Content-Type: text/plain; version=0.0.4; charset=utf-8"
259
- # instead of the default "application/json" which is incorrect.
260
- # See https://github.com/trallnag/prometheus-fastapi-instrumentator/issues/163#issue-1296092364
261
- Instrumentator(
262
- excluded_handlers=[
263
- "/metrics",
264
- "/health",
265
- "/load",
266
- "/ping",
267
- "/version",
268
- "/server_info",
269
- ],
270
- registry=registry,
271
- ).add().instrument(app).expose(app, response_class=PrometheusResponse)
272
-
273
- # Add prometheus asgi middleware to route /metrics requests
274
- metrics_route = Mount("/metrics", make_asgi_app(registry=registry))
275
-
276
- # Workaround for 307 Redirect for /metrics
277
- metrics_route.path_regex = re.compile("^/metrics(?P<path>.*)$")
278
- app.routes.append(metrics_route)
279
-
280
-
237
def base(request: Request) -> OpenAIServing:
238
# Reuse the existing instance
239
return tokenization(request)
@@ -323,16 +279,6 @@ def generate_tokens(request: Request) -> ServingTokens | None:
279
return request.app.state.serving_tokens
280
281
326
-@router.get("/health", response_class=Response)
327
-async def health(raw_request: Request) -> Response:
328
- """Health check."""
329
- try:
330
- await engine_client(raw_request).check_health()
331
- return Response(status_code=200)
332
- except EngineDeadError:
333
- return Response(status_code=503)
334
-
335
-
282
@router.get("/load")
283
async def get_server_load_metrics(request: Request):
284
# This endpoint returns the current server load metrics.
@@ -352,167 +298,6 @@ async def get_server_load_metrics(request: Request):
298
return JSONResponse(content={"server_load": request.app.state.server_load_metrics})
299
300
355
-@router.post("/pause")
356
-async def pause_generation(
357
- raw_request: Request,
358
- wait_for_inflight_requests: bool = Query(False),
359
- clear_cache: bool = Query(True),
360
-) -> JSONResponse:
361
- """Pause generation requests to allow weight updates.
362
-
363
- Args:
364
- wait_for_inflight_requests: When ``True`` waits for in-flight
365
- requests to finish before pausing. When ``False`` (default),
366
- aborts any in-flight requests immediately.
367
- clear_cache: Whether to clear KV/prefix caches after draining.
368
- """
369
-
370
- engine = engine_client(raw_request)
371
-
372
- try:
373
- await engine.pause_generation(
374
- wait_for_inflight_requests=wait_for_inflight_requests,
375
- clear_cache=clear_cache,
376
- )
377
- return JSONResponse(
378
- content={"status": "paused"},
379
- status_code=HTTPStatus.OK.value,
380
- )
381
-
382
- except ValueError as err:
383
- return JSONResponse(
384
- content={"error": str(err)},
385
- status_code=HTTPStatus.BAD_REQUEST.value,
386
- )
387
- except Exception as err: # pragma: no cover - defensive
388
- logger.exception("Failed to pause generation")
389
- return JSONResponse(
390
- content={"error": f"Failed to pause generation: {err}"},
391
- status_code=HTTPStatus.INTERNAL_SERVER_ERROR.value,
392
- )
393
-
394
-
395
-@router.post("/resume")
396
-async def resume_generation(raw_request: Request) -> JSONResponse:
397
- """Resume generation after a pause."""
398
-
399
- engine = engine_client(raw_request)
400
-
401
- try:
402
- await engine.resume_generation()
403
- return JSONResponse(
404
- content={"status": "resumed"},
405
- status_code=HTTPStatus.OK.value,
406
- )
407
- except Exception as err: # pragma: no cover - defensive
408
- logger.exception("Failed to resume generation")
409
- return JSONResponse(
410
- content={"error": f"Failed to resume generation: {err}"},
411
- status_code=HTTPStatus.INTERNAL_SERVER_ERROR.value,
412
- )
413
-
414
-
415
-@router.get("/is_paused")
416
-async def is_paused(raw_request: Request) -> JSONResponse:
417
- """Return the current pause status."""
418
-
419
- engine = engine_client(raw_request)
420
-
421
- try:
422
- paused = await engine.is_paused()
423
- except Exception as err: # pragma: no cover - defensive
424
- logger.exception("Failed to fetch pause status")
425
- return JSONResponse(
426
- content={"error": f"Failed to fetch pause status: {err}"},
427
- status_code=HTTPStatus.INTERNAL_SERVER_ERROR.value,
428
- )
429
-
430
- return JSONResponse(content={"is_paused": paused})
431
-
432
-
433
-@router.post(
434
- "/tokenize",
435
- dependencies=[Depends(validate_json_request)],
436
- responses={
437
- HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
438
- HTTPStatus.NOT_FOUND.value: {"model": ErrorResponse},
439
- HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
440
- HTTPStatus.NOT_IMPLEMENTED.value: {"model": ErrorResponse},
441
- },
442
-)
443
-@with_cancellation
444
-async def tokenize(request: TokenizeRequest, raw_request: Request):
445
- handler = tokenization(raw_request)
446
-
447
- try:
448
- generator = await handler.create_tokenize(request, raw_request)
449
- except NotImplementedError as e:
450
- raise HTTPException(
451
- status_code=HTTPStatus.NOT_IMPLEMENTED.value, detail=str(e)
452
- ) from e
453
- except Exception as e:
454
- raise HTTPException(
455
- status_code=HTTPStatus.INTERNAL_SERVER_ERROR.value, detail=str(e)
456
- ) from e
457
-
458
- if isinstance(generator, ErrorResponse):
459
- return JSONResponse(
460
- content=generator.model_dump(), status_code=generator.error.code
461
- )
462
- elif isinstance(generator, TokenizeResponse):
463
- return JSONResponse(content=generator.model_dump())
464
-
465
- assert_never(generator)
466
-
467
-
468
-@router.post(
469
- "/detokenize",
470
- dependencies=[Depends(validate_json_request)],
471
- responses={
472
- HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
473
- HTTPStatus.NOT_FOUND.value: {"model": ErrorResponse},
474
- HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
475
- },
476
-)
477
-@with_cancellation
478
-async def detokenize(request: DetokenizeRequest, raw_request: Request):
479
- handler = tokenization(raw_request)
480
-
481
- try:
482
- generator = await handler.create_detokenize(request, raw_request)
483
- except OverflowError as e:
484
- raise RequestValidationError(errors=[str(e)]) from e
485
- except Exception as e:
486
- raise HTTPException(
487
- status_code=HTTPStatus.INTERNAL_SERVER_ERROR.value, detail=str(e)
488
- ) from e
489
-
490
- if isinstance(generator, ErrorResponse):
491
- return JSONResponse(
492
- content=generator.model_dump(), status_code=generator.error.code
493
- )
494
- elif isinstance(generator, DetokenizeResponse):
495
- return JSONResponse(content=generator.model_dump())
496
-
497
- assert_never(generator)
498
-
499
-
500
-def maybe_register_tokenizer_info_endpoint(args):
501
- """Conditionally register the tokenizer info endpoint if enabled."""
502
- if getattr(args, "enable_tokenizer_info_endpoint", False):
503
-
504
- @router.get("/tokenizer_info")
505
- async def get_tokenizer_info(raw_request: Request):
506
- """Get comprehensive tokenizer information."""
507
- result = await tokenization(raw_request).get_tokenizer_info()
508
- return JSONResponse(
509
- content=result.model_dump(),
510
- status_code=result.error.code
511
- if isinstance(result, ErrorResponse)
512
- else 200,
513
- )
514
-
515
-
301
@router.get("/v1/models")
302
async def show_available_models(raw_request: Request):
303
handler = models(raw_request)
@@ -898,33 +683,6 @@ if envs.VLLM_SERVER_DEV_MODE:
683
await engine_client(raw_request).reset_mm_cache()
684
return Response(status_code=200)
685
901
- @router.post("/sleep")
902
- async def sleep(raw_request: Request):
903
- # get POST params
904
- level = raw_request.query_params.get("level", "1")
905
- await engine_client(raw_request).sleep(int(level))
906
- # FIXME: in v0 with frontend multiprocessing, the sleep command
907
- # is sent but does not finish yet when we return a response.
908
- return Response(status_code=200)
909
-
910
- @router.post("/wake_up")
911
- async def wake_up(raw_request: Request):
912
- tags = raw_request.query_params.getlist("tags")
913
- if tags == []:
914
- # set to None to wake up all tags if no tags are provided
915
- tags = None
916
- logger.info("wake up the engine with tags: %s", tags)
917
- await engine_client(raw_request).wake_up(tags)
918
- # FIXME: in v0 with frontend multiprocessing, the wake-up command
919
- # is sent but does not finish yet when we return a response.
920
- return Response(status_code=200)
921
-
922
- @router.get("/is_sleeping")
923
- async def is_sleeping(raw_request: Request):
924
- logger.info("check whether the engine is sleeping")
925
- is_sleeping = await engine_client(raw_request).is_sleeping()
926
- return JSONResponse(content={"is_sleeping": is_sleeping})
927
-
686
@router.post("/collective_rpc")
687
async def collective_rpc(raw_request: Request):
688
try:
@@ -952,138 +710,13 @@ if envs.VLLM_SERVER_DEV_MODE:
710
return Response(status_code=200)
711
response: list[Any] = []
712
for result in results:
955
- if result is None or isinstance(result, (dict, list)):
713
+ if result is None or isinstance(result, dict | list):
714
response.append(result)
715
else:
716
response.append(str(result))
717
return JSONResponse(content={"results": response})
718
719
962
-@router.post(
963
- "/scale_elastic_ep",
964
- dependencies=[Depends(validate_json_request)],
965
- responses={
966
- HTTPStatus.OK.value: {"model": dict},
967
- HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
968
- HTTPStatus.REQUEST_TIMEOUT.value: {"model": ErrorResponse},
969
- HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
970
- },
971
-)
972
-async def scale_elastic_ep(raw_request: Request):
973
- try:
974
- body = await raw_request.json()
975
- except json.JSONDecodeError as e:
976
- raise HTTPException(status_code=400, detail="Invalid JSON format") from e # noqa: B904
977
-
978
- new_data_parallel_size = body.get("new_data_parallel_size")
979
- drain_timeout = body.get("drain_timeout", 120) # Default 2 minutes
980
-
981
- if new_data_parallel_size is None:
982
- raise HTTPException(
983
- status_code=400, detail="new_data_parallel_size is required"
984
- )
985
-
986
- if not isinstance(new_data_parallel_size, int) or new_data_parallel_size <= 0:
987
- raise HTTPException(
988
- status_code=400, detail="new_data_parallel_size must be a positive integer"
989
- )
990
-
991
- if not isinstance(drain_timeout, int) or drain_timeout <= 0:
992
- raise HTTPException(
993
- status_code=400, detail="drain_timeout must be a positive integer"
994
- )
995
-
996
- # Set scaling flag to prevent new requests
997
- global _scaling_elastic_ep
998
- _scaling_elastic_ep = True
999
- client = engine_client(raw_request)
1000
- try:
1001
- await client.scale_elastic_ep(new_data_parallel_size, drain_timeout)
1002
- return JSONResponse(
1003
- {
1004
- "message": f"Scaled to {new_data_parallel_size} data parallel engines",
1005
- }
1006
- )
1007
- except TimeoutError as e:
1008
- raise HTTPException(
1009
- status_code=408,
1010
- detail="Scale failed due to request drain timeout "
1011
- f"after {drain_timeout} seconds",
1012
- ) from e
1013
- except Exception as e:
1014
- logger.error("Scale failed: %s", e)
1015
- raise HTTPException(status_code=500, detail="Scale failed") from e
1016
- finally:
1017
- _scaling_elastic_ep = False
1018
-
1019
-
1020
-@router.post("/is_scaling_elastic_ep")
1021
-async def is_scaling_elastic_ep(raw_request: Request):
1022
- return JSONResponse({"is_scaling_elastic_ep": _scaling_elastic_ep})
1023
-
1024
-
1025
-@router.post(
1026
- "/inference/v1/generate",
1027
- dependencies=[Depends(validate_json_request)],
1028
- responses={
1029
- HTTPStatus.OK.value: {"content": {"text/event-stream": {}}},
1030
- HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
1031
- HTTPStatus.NOT_FOUND.value: {"model": ErrorResponse},
1032
- HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
1033
- },
1034
-)
1035
-@with_cancellation
1036
-@load_aware_call
1037
-async def generate(request: GenerateRequest, raw_request: Request):
1038
- handler = generate_tokens(raw_request)
1039
- if handler is None:
1040
- return base(raw_request).create_error_response(
1041
- message="The model does not support generate tokens API"
1042
- )
1043
- try:
1044
- generator = await handler.serve_tokens(request, raw_request)
1045
- except Exception as e:
1046
- raise HTTPException(
1047
- status_code=HTTPStatus.INTERNAL_SERVER_ERROR.value, detail=str(e)
1048
- ) from e
1049
- if isinstance(generator, ErrorResponse):
1050
- return JSONResponse(
1051
- content=generator.model_dump(), status_code=generator.error.code
1052
- )
1053
-
1054
- elif isinstance(generator, GenerateResponse):
1055
- return JSONResponse(content=generator.model_dump())
1056
-
1057
- return StreamingResponse(content=generator, media_type="text/event-stream")
1058
-
1059
-
1060
-if envs.VLLM_TORCH_PROFILER_DIR:
1061
- logger.warning_once(
1062
- "Torch Profiler is enabled in the API server. This should ONLY be "
1063
- "used for local development!"
1064
- )
1065
-elif envs.VLLM_TORCH_CUDA_PROFILE:
1066
- logger.warning_once(
1067
- "CUDA Profiler is enabled in the API server. This should ONLY be "
1068
- "used for local development!"
1069
- )
1070
-if envs.VLLM_TORCH_PROFILER_DIR or envs.VLLM_TORCH_CUDA_PROFILE:
1071
-
1072
- @router.post("/start_profile")
1073
- async def start_profile(raw_request: Request):
1074
- logger.info("Starting profiler...")
1075
- await engine_client(raw_request).start_profile()
1076
- logger.info("Profiler started.")
1077
- return Response(status_code=200)
1078
-
1079
- @router.post("/stop_profile")
1080
- async def stop_profile(raw_request: Request):
1081
- logger.info("Stopping profiler...")
1082
- await engine_client(raw_request).stop_profile()
1083
- logger.info("Profiler stopped.")
1084
- return Response(status_code=200)
1085
-
1086
-
720
def load_log_config(log_config_file: str | None) -> dict | None:
721
if not log_config_file:
722
return None
@@ -1176,41 +809,6 @@ class XRequestIdMiddleware:
809
return self.app(scope, receive, send_with_request_id)
810
811
1179
-# Global variable to track scaling state
1180
-_scaling_elastic_ep = False
1181
-
1182
-
1183
-class ScalingMiddleware:
1184
- """
1185
- Middleware that checks if the model is currently scaling and
1186
- returns a 503 Service Unavailable response if it is.
1187
-
1188
- This middleware applies to all HTTP requests and prevents
1189
- processing when the model is in a scaling state.
1190
- """
1191
-
1192
- def __init__(self, app: ASGIApp) -> None:
1193
- self.app = app
1194
-
1195
- def __call__(self, scope: Scope, receive: Receive, send: Send) -> Awaitable[None]:
1196
- if scope["type"] != "http":
1197
- return self.app(scope, receive, send)
1198
-
1199
- # Check global scaling state
1200
- global _scaling_elastic_ep
1201
- if _scaling_elastic_ep:
1202
- # Return 503 Service Unavailable response
1203
- response = JSONResponse(
1204
- content={
1205
- "error": "The model is currently scaling. Please try again later."
1206
- },
1207
- status_code=503,
1208
- )
1209
- return response(scope, receive, send)
1210
-
1211
- return self.app(scope, receive, send)
1212
-
1213
-
812
def _extract_content_from_chunk(chunk_data: dict) -> str:
813
"""Extract content from a streaming response chunk."""
814
try:
@@ -1353,15 +951,10 @@ def build_app(args: Namespace) -> FastAPI:
951
)
952
else:
953
app = FastAPI(lifespan=lifespan)
954
+ app.state.args = args
955
+ from vllm.entrypoints.serve import register_vllm_serve_api_routers
956
1357
- if envs.VLLM_ALLOW_RUNTIME_LORA_UPDATING:
1358
- logger.warning(
1359
- "LoRA dynamic loading & unloading is enabled in the API server. "
1360
- "This should ONLY be used for local development!"
1361
- )
1362
- from vllm.entrypoints.dynamic_lora import register_dynamic_lora_routes
1363
-
1364
- register_dynamic_lora_routes(router)
957
+ register_vllm_serve_api_routers(app)
958
959
from vllm.entrypoints.sagemaker.routes import register_sagemaker_routes
960
@@ -1370,8 +963,6 @@ def build_app(args: Namespace) -> FastAPI:
963
964
app.root_path = args.root_path
965
1373
- mount_metrics(app)
1374
-
966
from vllm.entrypoints.pooling import register_pooling_api_routers
967
968
register_pooling_api_routers(app)
@@ -1462,31 +1053,6 @@ def build_app(args: Namespace) -> FastAPI:
1053
)
1054
1055
app = sagemaker_standards.bootstrap(app)
1465
- # Optional endpoints
1466
- if args.tokens_only:
1467
-
1468
- @app.post("/abort_requests")
1469
- async def abort_requests(raw_request: Request):
1470
- """
1471
- Abort one or more requests. To be used in a
1472
- Disaggregated Everything setup.
1473
- """
1474
- try:
1475
- body = await raw_request.json()
1476
- except json.JSONDecodeError as e:
1477
- raise HTTPException(
1478
- status_code=HTTPStatus.BAD_REQUEST.value,
1479
- detail=f"JSON decode error: {e}",
1480
- ) from e
1481
- request_ids = body.get("request_ids")
1482
- if request_ids is None:
1483
- raise HTTPException(
1484
- status_code=HTTPStatus.BAD_REQUEST.value,
1485
- detail="Missing 'request_ids' in request body",
1486
- )
1487
- # Abort requests in background
1488
- asyncio.create_task(engine_client(raw_request).abort(request_ids))
1489
- return Response(status_code=200)
1056
1057
return app
1058
@@ -1515,7 +1081,7 @@ async def init_app_state(
1081
state.engine_client = engine_client
1082
state.log_stats = not args.disable_log_stats
1083
state.vllm_config = vllm_config
1518
-
1084
+ state.args = args
1085
supported_tasks = await engine_client.get_supported_tasks()
1086
logger.info("Supported tasks: %s", supported_tasks)
1087
@@ -1839,7 +1405,6 @@ async def run_server_worker(
1405
args,
1406
client_config=client_config,
1407
) as engine_client:
1842
- maybe_register_tokenizer_info_endpoint(args)
1408
app = build_app(args)
1409
1410
await init_app_state(engine_client, app.state, args)
vllm/entrypoints/openai/serving_engine.py
+1
-2
@@ -74,8 +74,6 @@ from vllm.entrypoints.openai.protocol import (
74
ErrorResponse,
75
FunctionCall,
76
FunctionDefinition,
77
- GenerateRequest,
78
- GenerateResponse,
77
ResponsesRequest,
78
TokenizeChatRequest,
79
TokenizeCompletionRequest,
@@ -87,6 +85,7 @@ from vllm.entrypoints.openai.protocol import (
85
from vllm.entrypoints.openai.serving_models import OpenAIServingModels
86
from vllm.entrypoints.openai.tool_parsers import ToolParser, ToolParserManager
87
from vllm.entrypoints.renderer import BaseRenderer, CompletionRenderer, RenderConfig
88
+from vllm.entrypoints.serve.disagg.protocol import GenerateRequest, GenerateResponse
89
from vllm.entrypoints.utils import _validate_truncation_size
90
from vllm.inputs.data import PromptType
91
from vllm.inputs.data import TokensPrompt as EngineTokensPrompt
vllm/entrypoints/sagemaker/routes.py
+1
-1
@@ -16,7 +16,6 @@ from vllm.entrypoints.openai.api_server import (
16
completion,
17
create_chat_completion,
18
create_completion,
19
- health,
19
validate_json_request,
20
)
21
from vllm.entrypoints.openai.protocol import (
@@ -38,6 +37,7 @@ from vllm.entrypoints.pooling.score.api_router import (
37
score,
38
)
39
from vllm.entrypoints.pooling.score.protocol import RerankRequest, ScoreRequest
40
+from vllm.entrypoints.serve.instrumentator.health import health
41
42
# TODO: RequestType = TypeForm[BaseModel] when recognized by type checkers
43
# (requires typing_extensions >= 4.13)
vllm/entrypoints/serve/__init__.py
new
+60
@@ -0,0 +1,60 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+
5
+from fastapi import FastAPI
6
+
7
+
8
+def register_vllm_serve_api_routers(app: FastAPI):
9
+ from vllm.entrypoints.serve.lora.api_router import (
10
+ attach_router as attach_lora_router,
11
+ )
12
+
13
+ attach_lora_router(app)
14
+ from vllm.entrypoints.serve.elastic_ep.api_router import (
15
+ attach_router as attach_elastic_ep_router,
16
+ )
17
+
18
+ attach_elastic_ep_router(app)
19
+
20
+ from vllm.entrypoints.serve.profile.api_router import (
21
+ attach_router as attach_profile_router,
22
+ )
23
+
24
+ attach_profile_router(app)
25
+
26
+ from vllm.entrypoints.serve.sleep.api_router import (
27
+ attach_router as attach_sleep_router,
28
+ )
29
+
30
+ attach_sleep_router(app)
31
+
32
+ from vllm.entrypoints.serve.tokenize.api_router import (
33
+ attach_router as attach_tokenize_router,
34
+ )
35
+
36
+ attach_tokenize_router(app)
37
+
38
+ from vllm.entrypoints.serve.disagg.api_router import (
39
+ attach_router as attach_disagg_router,
40
+ )
41
+
42
+ attach_disagg_router(app)
43
+
44
+ from vllm.entrypoints.serve.rlhf.api_router import (
45
+ attach_router as attach_rlhf_router,
46
+ )
47
+
48
+ attach_rlhf_router(app)
49
+
50
+ from vllm.entrypoints.serve.instrumentator.metrics import (
51
+ attach_router as attach_metrics_router,
52
+ )
53
+
54
+ attach_metrics_router(app)
55
+
56
+ from vllm.entrypoints.serve.instrumentator.health import (
57
+ attach_router as attach_health_router,
58
+ )
59
+
60
+ attach_health_router(app)
vllm/entrypoints/serve/disagg/__init__.py
vllm/entrypoints/serve/disagg/api_router.py
new
+110
@@ -0,0 +1,110 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+
5
+import asyncio
6
+import json
7
+from http import HTTPStatus
8
+
9
+from fastapi import APIRouter, Depends, FastAPI, HTTPException, Request, Response
10
+from fastapi.responses import JSONResponse, StreamingResponse
11
+
12
+from vllm.engine.protocol import EngineClient
13
+from vllm.entrypoints.openai.api_server import validate_json_request
14
+from vllm.entrypoints.openai.protocol import (
15
+ ErrorResponse,
16
+)
17
+from vllm.entrypoints.serve.disagg.protocol import (
18
+ GenerateRequest,
19
+ GenerateResponse,
20
+)
21
+from vllm.entrypoints.serve.disagg.serving import (
22
+ ServingTokens,
23
+)
24
+from vllm.entrypoints.serve.tokenize.serving import OpenAIServingTokenization
25
+from vllm.entrypoints.utils import (
26
+ load_aware_call,
27
+ with_cancellation,
28
+)
29
+from vllm.logger import init_logger
30
+
31
+logger = init_logger(__name__)
32
+
33
+
34
+def tokenization(request: Request) -> OpenAIServingTokenization:
35
+ return request.app.state.openai_serving_tokenization
36
+
37
+
38
+def generate_tokens(request: Request) -> ServingTokens | None:
39
+ return request.app.state.serving_tokens
40
+
41
+
42
+def engine_client(request: Request) -> EngineClient:
43
+ return request.app.state.engine_client
44
+
45
+
46
+router = APIRouter()
47
+
48
+
49
+@router.post(
50
+ "/inference/v1/generate",
51
+ dependencies=[Depends(validate_json_request)],
52
+ responses={
53
+ HTTPStatus.OK.value: {"content": {"text/event-stream": {}}},
54
+ HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
55
+ HTTPStatus.NOT_FOUND.value: {"model": ErrorResponse},
56
+ HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
57
+ },
58
+)
59
+@with_cancellation
60
+@load_aware_call
61
+async def generate(request: GenerateRequest, raw_request: Request):
62
+ handler = generate_tokens(raw_request)
63
+ if handler is None:
64
+ return tokenization(raw_request).create_error_response(
65
+ message="The model does not support generate tokens API"
66
+ )
67
+ try:
68
+ generator = await handler.serve_tokens(request, raw_request)
69
+ except Exception as e:
70
+ raise HTTPException(
71
+ status_code=HTTPStatus.INTERNAL_SERVER_ERROR.value, detail=str(e)
72
+ ) from e
73
+ if isinstance(generator, ErrorResponse):
74
+ return JSONResponse(
75
+ content=generator.model_dump(), status_code=generator.error.code
76
+ )
77
+
78
+ elif isinstance(generator, GenerateResponse):
79
+ return JSONResponse(content=generator.model_dump())
80
+
81
+ return StreamingResponse(content=generator, media_type="text/event-stream")
82
+
83
+
84
+def attach_router(app: FastAPI):
85
+ if getattr(app.state.args, "tokens_only", False):
86
+
87
+ @router.post("/abort_requests")
88
+ async def abort_requests(raw_request: Request):
89
+ """
90
+ Abort one or more requests. To be used in a
91
+ Disaggregated Everything setup.
92
+ """
93
+ try:
94
+ body = await raw_request.json()
95
+ except json.JSONDecodeError as e:
96
+ raise HTTPException(
97
+ status_code=HTTPStatus.BAD_REQUEST.value,
98
+ detail=f"JSON decode error: {e}",
99
+ ) from e
100
+ request_ids = body.get("request_ids")
101
+ if request_ids is None:
102
+ raise HTTPException(
103
+ status_code=HTTPStatus.BAD_REQUEST.value,
104
+ detail="Missing 'request_ids' in request body",
105
+ )
106
+ # Abort requests in background
107
+ asyncio.create_task(engine_client(raw_request).abort(request_ids))
108
+ return Response(status_code=200)
109
+
110
+ app.include_router(router)
vllm/entrypoints/serve/disagg/protocol.py
new
+90
@@ -0,0 +1,90 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+from typing import Any
4
+
5
+from pydantic import BaseModel, Field
6
+
7
+from vllm.entrypoints.openai.protocol import (
8
+ ChatCompletionLogProbs,
9
+ Logprob,
10
+ SamplingParams,
11
+ StreamOptions,
12
+)
13
+from vllm.utils import random_uuid
14
+
15
+
16
+####### Tokens IN <> Tokens OUT #######
17
+class GenerateRequest(BaseModel):
18
+ request_id: str = Field(
19
+ default_factory=lambda: f"{random_uuid()}",
20
+ description=(
21
+ "The request_id related to this request. If the caller does "
22
+ "not set it, a random_uuid will be generated. This id is used "
23
+ "through out the inference process and return in response."
24
+ ),
25
+ )
26
+ token_ids: list[int]
27
+ """The token ids to generate text from."""
28
+
29
+ # features: MultiModalFeatureSpec
30
+ # TODO (NickLucche): implement once Renderer work is completed
31
+ features: str | None = None
32
+ """The processed MM inputs for the model."""
33
+
34
+ sampling_params: SamplingParams
35
+ """The sampling parameters for the model."""
36
+
37
+ model: str | None = None
38
+
39
+ stream: bool | None = False
40
+ stream_options: StreamOptions | None = None
41
+ cache_salt: str | None = Field(
42
+ default=None,
43
+ description=(
44
+ "If specified, the prefix cache will be salted with the provided "
45
+ "string to prevent an attacker to guess prompts in multi-user "
46
+ "environments. The salt should be random, protected from "
47
+ "access by 3rd parties, and long enough to be "
48
+ "unpredictable (e.g., 43 characters base64-encoded, corresponding "
49
+ "to 256 bit)."
50
+ ),
51
+ )
52
+ priority: int = Field(
53
+ default=0,
54
+ description=(
55
+ "The priority of the request (lower means earlier handling; "
56
+ "default: 0). Any priority other than 0 will raise an error "
57
+ "if the served model does not use priority scheduling."
58
+ ),
59
+ )
60
+ kv_transfer_params: dict[str, Any] | None = Field(
61
+ default=None,
62
+ description="KVTransfer parameters used for disaggregated serving.",
63
+ )
64
+
65
+
66
+class GenerateResponseChoice(BaseModel):
67
+ index: int
68
+ logprobs: ChatCompletionLogProbs | None = None
69
+ # per OpenAI spec this is the default
70
+ finish_reason: str | None = "stop"
71
+ token_ids: list[int] | None = None
72
+
73
+
74
+class GenerateResponse(BaseModel):
75
+ request_id: str = Field(
76
+ default_factory=lambda: f"{random_uuid()}",
77
+ description=(
78
+ "The request_id related to this request. If the caller does "
79
+ "not set it, a random_uuid will be generated. This id is used "
80
+ "through out the inference process and return in response."
81
+ ),
82
+ )
83
+ choices: list[GenerateResponseChoice]
84
+
85
+ prompt_logprobs: list[dict[int, Logprob] | None] | None = None
86
+
87
+ kv_transfer_params: dict[str, Any] | None = Field(
88
+ default=None,
89
+ description="KVTransfer parameters used for disaggregated serving.",
90
+ )
vllm/entrypoints/serve/disagg/serving.py
renamed
+7
-3
@@ -1,5 +1,7 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+
5
import asyncio
6
import time
7
from collections.abc import AsyncGenerator
@@ -14,15 +16,17 @@ from vllm.entrypoints.openai.protocol import (
16
ChatCompletionLogProbs,
17
ChatCompletionLogProbsContent,
18
ErrorResponse,
17
- GenerateRequest,
18
- GenerateResponse,
19
- GenerateResponseChoice,
19
PromptTokenUsageInfo,
20
RequestResponseMetadata,
21
UsageInfo,
22
)
23
from vllm.entrypoints.openai.serving_engine import OpenAIServing, clamp_prompt_logprobs
24
from vllm.entrypoints.openai.serving_models import OpenAIServingModels
25
+from vllm.entrypoints.serve.disagg.protocol import (
26
+ GenerateRequest,
27
+ GenerateResponse,
28
+ GenerateResponseChoice,
29
+)
30
from vllm.inputs.data import TokensPrompt as EngineTokensPrompt
31
from vllm.logger import init_logger
32
from vllm.logprobs import Logprob
vllm/entrypoints/serve/elastic_ep/__init__.py
vllm/entrypoints/serve/elastic_ep/api_router.py
new
+96
@@ -0,0 +1,96 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+
5
+import json
6
+from http import HTTPStatus
7
+
8
+from fastapi import APIRouter, Depends, FastAPI, HTTPException, Request
9
+from fastapi.responses import JSONResponse
10
+
11
+from vllm.engine.protocol import EngineClient
12
+from vllm.entrypoints.openai.api_server import validate_json_request
13
+from vllm.entrypoints.openai.protocol import (
14
+ ErrorResponse,
15
+)
16
+from vllm.entrypoints.serve.elastic_ep.middleware import (
17
+ get_scaling_elastic_ep,
18
+ set_scaling_elastic_ep,
19
+)
20
+from vllm.logger import init_logger
21
+
22
+logger = init_logger(__name__)
23
+
24
+
25
+def engine_client(request: Request) -> EngineClient:
26
+ return request.app.state.engine_client
27
+
28
+
29
+router = APIRouter()
30
+
31
+
32
+@router.post(
33
+ "/scale_elastic_ep",
34
+ dependencies=[Depends(validate_json_request)],
35
+ responses={
36
+ HTTPStatus.OK.value: {"model": dict},
37
+ HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
38
+ HTTPStatus.REQUEST_TIMEOUT.value: {"model": ErrorResponse},
39
+ HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
40
+ },
41
+)
42
+async def scale_elastic_ep(raw_request: Request):
43
+ try:
44
+ body = await raw_request.json()
45
+ except json.JSONDecodeError as e:
46
+ raise HTTPException(status_code=400, detail="Invalid JSON format") from e # noqa: B904
47
+
48
+ new_data_parallel_size = body.get("new_data_parallel_size")
49
+ drain_timeout = body.get("drain_timeout", 120) # Default 2 minutes
50
+
51
+ if new_data_parallel_size is None:
52
+ raise HTTPException(
53
+ status_code=400, detail="new_data_parallel_size is required"
54
+ )
55
+
56
+ if not isinstance(new_data_parallel_size, int) or new_data_parallel_size <= 0:
57
+ raise HTTPException(
58
+ status_code=400,
59
+ detail="new_data_parallel_size must be a positive integer",
60
+ )
61
+
62
+ if not isinstance(drain_timeout, int) or drain_timeout <= 0:
63
+ raise HTTPException(
64
+ status_code=400, detail="drain_timeout must be a positive integer"
65
+ )
66
+
67
+ # Set scaling flag to prevent new requests
68
+ set_scaling_elastic_ep(True)
69
+ client = engine_client(raw_request)
70
+ try:
71
+ await client.scale_elastic_ep(new_data_parallel_size, drain_timeout)
72
+ return JSONResponse(
73
+ {
74
+ "message": f"Scaled to {new_data_parallel_size} data parallel engines",
75
+ }
76
+ )
77
+ except TimeoutError as e:
78
+ raise HTTPException(
79
+ status_code=408,
80
+ detail="Scale failed due to request drain timeout "
81
+ f"after {drain_timeout} seconds",
82
+ ) from e
83
+ except Exception as e:
84
+ logger.error("Scale failed: %s", e)
85
+ raise HTTPException(status_code=500, detail="Scale failed") from e
86
+ finally:
87
+ set_scaling_elastic_ep(False)
88
+
89
+
90
+@router.post("/is_scaling_elastic_ep")
91
+async def is_scaling_elastic_ep(raw_request: Request):
92
+ return JSONResponse({"is_scaling_elastic_ep": get_scaling_elastic_ep()})
93
+
94
+
95
+def attach_router(app: FastAPI):
96
+ app.include_router(router)
vllm/entrypoints/serve/elastic_ep/middleware.py
new
+49
@@ -0,0 +1,49 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+from collections.abc import Awaitable
5
+
6
+from fastapi.responses import JSONResponse
7
+from starlette.types import ASGIApp, Receive, Scope, Send
8
+
9
+# Global variable to track scaling state
10
+_scaling_elastic_ep = False
11
+
12
+
13
+def get_scaling_elastic_ep():
14
+ return _scaling_elastic_ep
15
+
16
+
17
+def set_scaling_elastic_ep(value):
18
+ global _scaling_elastic_ep
19
+ _scaling_elastic_ep = value
20
+
21
+
22
+class ScalingMiddleware:
23
+ """
24
+ Middleware that checks if the model is currently scaling and
25
+ returns a 503 Service Unavailable response if it is.
26
+
27
+ This middleware applies to all HTTP requests and prevents
28
+ processing when the model is in a scaling state.
29
+ """
30
+
31
+ def __init__(self, app: ASGIApp) -> None:
32
+ self.app = app
33
+
34
+ def __call__(self, scope: Scope, receive: Receive, send: Send) -> Awaitable[None]:
35
+ if scope["type"] != "http":
36
+ return self.app(scope, receive, send)
37
+
38
+ # Check global scaling state
39
+ if get_scaling_elastic_ep():
40
+ # Return 503 Service Unavailable response
41
+ response = JSONResponse(
42
+ content={
43
+ "error": "The model is currently scaling. Please try again later."
44
+ },
45
+ status_code=503,
46
+ )
47
+ return response(scope, receive, send)
48
+
49
+ return self.app(scope, receive, send)
vllm/entrypoints/serve/instrumentator/__init__.py
vllm/entrypoints/serve/instrumentator/health.py
new
+33
@@ -0,0 +1,33 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+
5
+from fastapi import APIRouter, Request
6
+from fastapi.responses import Response
7
+
8
+from vllm.engine.protocol import EngineClient
9
+from vllm.logger import init_logger
10
+from vllm.v1.engine.exceptions import EngineDeadError
11
+
12
+logger = init_logger(__name__)
13
+
14
+
15
+router = APIRouter()
16
+
17
+
18
+def engine_client(request: Request) -> EngineClient:
19
+ return request.app.state.engine_client
20
+
21
+
22
+@router.get("/health", response_class=Response)
23
+async def health(raw_request: Request) -> Response:
24
+ """Health check."""
25
+ try:
26
+ await engine_client(raw_request).check_health()
27
+ return Response(status_code=200)
28
+ except EngineDeadError:
29
+ return Response(status_code=503)
30
+
31
+
32
+def attach_router(app):
33
+ app.include_router(router)
vllm/entrypoints/serve/instrumentator/metrics.py
new
+46
@@ -0,0 +1,46 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+
5
+import re
6
+
7
+import prometheus_client
8
+from fastapi import FastAPI, Response
9
+from prometheus_client import make_asgi_app
10
+from prometheus_fastapi_instrumentator import Instrumentator
11
+from starlette.routing import Mount
12
+
13
+from vllm.v1.metrics.prometheus import get_prometheus_registry
14
+
15
+
16
+class PrometheusResponse(Response):
17
+ media_type = prometheus_client.CONTENT_TYPE_LATEST
18
+
19
+
20
+def attach_router(app: FastAPI):
21
+ """Mount prometheus metrics to a FastAPI app."""
22
+
23
+ registry = get_prometheus_registry()
24
+
25
+ # `response_class=PrometheusResponse` is needed to return an HTTP response
26
+ # with header "Content-Type: text/plain; version=0.0.4; charset=utf-8"
27
+ # instead of the default "application/json" which is incorrect.
28
+ # See https://github.com/trallnag/prometheus-fastapi-instrumentator/issues/163#issue-1296092364
29
+ Instrumentator(
30
+ excluded_handlers=[
31
+ "/metrics",
32
+ "/health",
33
+ "/load",
34
+ "/ping",
35
+ "/version",
36
+ "/server_info",
37
+ ],
38
+ registry=registry,
39
+ ).add().instrument(app).expose(app, response_class=PrometheusResponse)
40
+
41
+ # Add prometheus asgi middleware to route /metrics requests
42
+ metrics_route = Mount("/metrics", make_asgi_app(registry=registry))
43
+
44
+ # Workaround for 307 Redirect for /metrics
45
+ metrics_route.path_regex = re.compile("^/metrics(?P<path>.*)$")
46
+ app.routes.append(metrics_route)
vllm/entrypoints/serve/lora/__init__.py
vllm/entrypoints/serve/lora/api_router.py
renamed
+16
-3
@@ -1,9 +1,12 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+
5
import model_hosting_container_standards.sagemaker as sagemaker_standards
4
-from fastapi import APIRouter, Depends, Request
6
+from fastapi import APIRouter, Depends, FastAPI, Request
7
from fastapi.responses import JSONResponse, Response
8
9
+from vllm import envs
10
from vllm.entrypoints.openai.api_server import models, validate_json_request
11
from vllm.entrypoints.openai.protocol import (
12
ErrorResponse,
@@ -14,9 +17,18 @@ from vllm.entrypoints.openai.serving_models import OpenAIServingModels
17
from vllm.logger import init_logger
18
19
logger = init_logger(__name__)
20
+router = APIRouter()
21
22
19
-def register_dynamic_lora_routes(router: APIRouter):
23
+def attach_router(app: FastAPI):
24
+ if not envs.VLLM_ALLOW_RUNTIME_LORA_UPDATING:
25
+ """If LoRA dynamic loading & unloading is not enabled, do nothing."""
26
+ return
27
+ logger.warning(
28
+ "LoRA dynamic loading & unloading is enabled in the API server. "
29
+ "This should ONLY be used for local development!"
30
+ )
31
+
32
@sagemaker_standards.register_load_adapter_handler(
33
request_shape={
34
"lora_name": "body.name",
@@ -54,4 +66,5 @@ def register_dynamic_lora_routes(router: APIRouter):
66
67
return Response(status_code=200, content=response)
68
57
- return router
69
+ # register the router
70
+ app.include_router(router)
vllm/entrypoints/serve/profile/__init__.py
vllm/entrypoints/serve/profile/api_router.py
new
+49
@@ -0,0 +1,49 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+
5
+from fastapi import APIRouter, FastAPI, Request
6
+from fastapi.responses import Response
7
+
8
+import vllm.envs as envs
9
+from vllm.engine.protocol import EngineClient
10
+from vllm.logger import init_logger
11
+
12
+logger = init_logger(__name__)
13
+
14
+router = APIRouter()
15
+
16
+
17
+def engine_client(request: Request) -> EngineClient:
18
+ return request.app.state.engine_client
19
+
20
+
21
+@router.post("/start_profile")
22
+async def start_profile(raw_request: Request):
23
+ logger.info("Starting profiler...")
24
+ await engine_client(raw_request).start_profile()
25
+ logger.info("Profiler started.")
26
+ return Response(status_code=200)
27
+
28
+
29
+@router.post("/stop_profile")
30
+async def stop_profile(raw_request: Request):
31
+ logger.info("Stopping profiler...")
32
+ await engine_client(raw_request).stop_profile()
33
+ logger.info("Profiler stopped.")
34
+ return Response(status_code=200)
35
+
36
+
37
+def attach_router(app: FastAPI):
38
+ if envs.VLLM_TORCH_PROFILER_DIR:
39
+ logger.warning_once(
40
+ "Torch Profiler is enabled in the API server. This should ONLY be "
41
+ "used for local development!"
42
+ )
43
+ elif envs.VLLM_TORCH_CUDA_PROFILE:
44
+ logger.warning_once(
45
+ "CUDA Profiler is enabled in the API server. This should ONLY be "
46
+ "used for local development!"
47
+ )
48
+ if envs.VLLM_TORCH_PROFILER_DIR or envs.VLLM_TORCH_CUDA_PROFILE:
49
+ app.include_router(router)
vllm/entrypoints/serve/rlhf/__init__.py
vllm/entrypoints/serve/rlhf/api_router.py
new
+102
@@ -0,0 +1,102 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+
5
+from http import HTTPStatus
6
+
7
+from fastapi import APIRouter, FastAPI, Query, Request
8
+from fastapi.responses import JSONResponse
9
+
10
+from vllm.engine.protocol import EngineClient
11
+from vllm.logger import init_logger
12
+
13
+logger = init_logger(__name__)
14
+
15
+
16
+def engine_client(request: Request) -> EngineClient:
17
+ return request.app.state.engine_client
18
+
19
+
20
+router = APIRouter()
21
+
22
+
23
+@router.post("/pause")
24
+async def pause_generation(
25
+ raw_request: Request,
26
+ wait_for_inflight_requests: bool = Query(False),
27
+ clear_cache: bool = Query(True),
28
+) -> JSONResponse:
29
+ """Pause generation requests to allow weight updates.
30
+
31
+ Args:
32
+ wait_for_inflight_requests: When ``True`` waits for in-flight
33
+ requests to finish before pausing. When ``False`` (default),
34
+ aborts any in-flight requests immediately.
35
+ clear_cache: Whether to clear KV/prefix caches after draining.
36
+ """
37
+
38
+ engine = engine_client(raw_request)
39
+
40
+ try:
41
+ await engine.pause_generation(
42
+ wait_for_inflight_requests=wait_for_inflight_requests,
43
+ clear_cache=clear_cache,
44
+ )
45
+ return JSONResponse(
46
+ content={"status": "paused"},
47
+ status_code=HTTPStatus.OK.value,
48
+ )
49
+
50
+ except ValueError as err:
51
+ return JSONResponse(
52
+ content={"error": str(err)},
53
+ status_code=HTTPStatus.BAD_REQUEST.value,
54
+ )
55
+ except Exception as err: # pragma: no cover - defensive
56
+ logger.exception("Failed to pause generation")
57
+ return JSONResponse(
58
+ content={"error": f"Failed to pause generation: {err}"},
59
+ status_code=HTTPStatus.INTERNAL_SERVER_ERROR.value,
60
+ )
61
+
62
+
63
+@router.post("/resume")
64
+async def resume_generation(raw_request: Request) -> JSONResponse:
65
+ """Resume generation after a pause."""
66
+
67
+ engine = engine_client(raw_request)
68
+
69
+ try:
70
+ await engine.resume_generation()
71
+ return JSONResponse(
72
+ content={"status": "resumed"},
73
+ status_code=HTTPStatus.OK.value,
74
+ )
75
+ except Exception as err: # pragma: no cover - defensive
76
+ logger.exception("Failed to resume generation")
77
+ return JSONResponse(
78
+ content={"error": f"Failed to resume generation: {err}"},
79
+ status_code=HTTPStatus.INTERNAL_SERVER_ERROR.value,
80
+ )
81
+
82
+
83
+@router.get("/is_paused")
84
+async def is_paused(raw_request: Request) -> JSONResponse:
85
+ """Return the current pause status."""
86
+
87
+ engine = engine_client(raw_request)
88
+
89
+ try:
90
+ paused = await engine.is_paused()
91
+ except Exception as err: # pragma: no cover - defensive
92
+ logger.exception("Failed to fetch pause status")
93
+ return JSONResponse(
94
+ content={"error": f"Failed to fetch pause status: {err}"},
95
+ status_code=HTTPStatus.INTERNAL_SERVER_ERROR.value,
96
+ )
97
+
98
+ return JSONResponse(content={"is_paused": paused})
99
+
100
+
101
+def attach_router(app: FastAPI):
102
+ app.include_router(router)
vllm/entrypoints/serve/sleep/__init__.py
vllm/entrypoints/serve/sleep/api_router.py
new
+60
@@ -0,0 +1,60 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+
5
+from fastapi import APIRouter, FastAPI, Request
6
+from fastapi.responses import JSONResponse, Response
7
+
8
+import vllm.envs as envs
9
+from vllm.engine.protocol import EngineClient
10
+from vllm.logger import init_logger
11
+
12
+logger = init_logger(__name__)
13
+
14
+
15
+def engine_client(request: Request) -> EngineClient:
16
+ return request.app.state.engine_client
17
+
18
+
19
+router = APIRouter()
20
+
21
+
22
+@router.post("/sleep")
23
+async def sleep(raw_request: Request):
24
+ # get POST params
25
+ level = raw_request.query_params.get("level", "1")
26
+ await engine_client(raw_request).sleep(int(level))
27
+ # FIXME: in v0 with frontend multiprocessing, the sleep command
28
+ # is sent but does not finish yet when we return a response.
29
+ return Response(status_code=200)
30
+
31
+
32
+@router.post("/wake_up")
33
+async def wake_up(raw_request: Request):
34
+ tags = raw_request.query_params.getlist("tags")
35
+ if tags == []:
36
+ # set to None to wake up all tags if no tags are provided
37
+ tags = None
38
+ logger.info("wake up the engine with tags: %s", tags)
39
+ await engine_client(raw_request).wake_up(tags)
40
+ # FIXME: in v0 with frontend multiprocessing, the wake-up command
41
+ # is sent but does not finish yet when we return a response.
42
+ return Response(status_code=200)
43
+
44
+
45
+@router.get("/is_sleeping")
46
+async def is_sleeping(raw_request: Request):
47
+ logger.info("check whether the engine is sleeping")
48
+ is_sleeping = await engine_client(raw_request).is_sleeping()
49
+ return JSONResponse(content={"is_sleeping": is_sleeping})
50
+
51
+
52
+def attach_router(app: FastAPI):
53
+ if not envs.VLLM_SERVER_DEV_MODE:
54
+ return
55
+ logger.warning(
56
+ "SECURITY WARNING: Development endpoints are enabled! "
57
+ "This should NOT be used in production!"
58
+ )
59
+
60
+ app.include_router(router)
vllm/entrypoints/serve/tokenize/__init__.py
vllm/entrypoints/serve/tokenize/api_router.py
new
+118
@@ -0,0 +1,118 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+
5
+from http import HTTPStatus
6
+
7
+from fastapi import APIRouter, Depends, FastAPI, HTTPException, Request
8
+from fastapi.exceptions import RequestValidationError
9
+from fastapi.responses import JSONResponse
10
+from typing_extensions import assert_never
11
+
12
+from vllm.entrypoints.openai.api_server import validate_json_request
13
+from vllm.entrypoints.openai.protocol import (
14
+ DetokenizeRequest,
15
+ DetokenizeResponse,
16
+ ErrorResponse,
17
+ TokenizeRequest,
18
+ TokenizeResponse,
19
+)
20
+from vllm.entrypoints.serve.tokenize.serving import OpenAIServingTokenization
21
+from vllm.entrypoints.utils import (
22
+ with_cancellation,
23
+)
24
+from vllm.logger import init_logger
25
+
26
+logger = init_logger(__name__)
27
+
28
+
29
+def tokenization(request: Request) -> OpenAIServingTokenization:
30
+ return request.app.state.openai_serving_tokenization
31
+
32
+
33
+router = APIRouter()
34
+
35
+
36
+@router.post(
37
+ "/tokenize",
38
+ dependencies=[Depends(validate_json_request)],
39
+ responses={
40
+ HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
41
+ HTTPStatus.NOT_FOUND.value: {"model": ErrorResponse},
42
+ HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
43
+ HTTPStatus.NOT_IMPLEMENTED.value: {"model": ErrorResponse},
44
+ },
45
+)
46
+@with_cancellation
47
+async def tokenize(request: TokenizeRequest, raw_request: Request):
48
+ handler = tokenization(raw_request)
49
+
50
+ try:
51
+ generator = await handler.create_tokenize(request, raw_request)
52
+ except NotImplementedError as e:
53
+ raise HTTPException(
54
+ status_code=HTTPStatus.NOT_IMPLEMENTED.value, detail=str(e)
55
+ ) from e
56
+ except Exception as e:
57
+ raise HTTPException(
58
+ status_code=HTTPStatus.INTERNAL_SERVER_ERROR.value, detail=str(e)
59
+ ) from e
60
+
61
+ if isinstance(generator, ErrorResponse):
62
+ return JSONResponse(
63
+ content=generator.model_dump(), status_code=generator.error.code
64
+ )
65
+ elif isinstance(generator, TokenizeResponse):
66
+ return JSONResponse(content=generator.model_dump())
67
+
68
+ assert_never(generator)
69
+
70
+
71
+@router.post(
72
+ "/detokenize",
73
+ dependencies=[Depends(validate_json_request)],
74
+ responses={
75
+ HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
76
+ HTTPStatus.NOT_FOUND.value: {"model": ErrorResponse},
77
+ HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
78
+ },
79
+)
80
+@with_cancellation
81
+async def detokenize(request: DetokenizeRequest, raw_request: Request):
82
+ handler = tokenization(raw_request)
83
+
84
+ try:
85
+ generator = await handler.create_detokenize(request, raw_request)
86
+ except OverflowError as e:
87
+ raise RequestValidationError(errors=[str(e)]) from e
88
+ except Exception as e:
89
+ raise HTTPException(
90
+ status_code=HTTPStatus.INTERNAL_SERVER_ERROR.value, detail=str(e)
91
+ ) from e
92
+
93
+ if isinstance(generator, ErrorResponse):
94
+ return JSONResponse(
95
+ content=generator.model_dump(), status_code=generator.error.code
96
+ )
97
+ elif isinstance(generator, DetokenizeResponse):
98
+ return JSONResponse(content=generator.model_dump())
99
+
100
+ assert_never(generator)
101
+
102
+
103
+def attach_router(app: FastAPI):
104
+ if getattr(app.state.args, "enable_tokenizer_info_endpoint", False):
105
+ """Conditionally register the tokenizer info endpoint if enabled."""
106
+
107
+ @router.get("/tokenizer_info")
108
+ async def get_tokenizer_info(raw_request: Request):
109
+ """Get comprehensive tokenizer information."""
110
+ result = await tokenization(raw_request).get_tokenizer_info()
111
+ return JSONResponse(
112
+ content=result.model_dump(),
113
+ status_code=result.error.code
114
+ if isinstance(result, ErrorResponse)
115
+ else 200,
116
+ )
117
+
118
+ app.include_router(router)