@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / f25e0d1125
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 15, 09:45 UTC

[Bugfix]: make most of `test_openai_schema.py` pass (#17664)

David Xia committed May 14, 2025 at 20:04 UTC f25e0d1125f873201ae880b50df46a9e3d29f3ba
3 files changed +240 -35
vllm/entrypoints/openai/api_server.py
+238 -33
@@ -17,8 +17,10 @@ from collections.abc import AsyncIterator
17 from contextlib import asynccontextmanager
18 from functools import partial
19 from http import HTTPStatus
20 +from json import JSONDecodeError
21 from typing import Annotated, Optional, Union
22
23 +import prometheus_client
24 import uvloop
25 from fastapi import APIRouter, Depends, FastAPI, Form, HTTPException, Request
26 from fastapi.exceptions import RequestValidationError
@@ -305,15 +307,18 @@ async def validate_json_request(raw_request: Request):
307 content_type = raw_request.headers.get("content-type", "").lower()
308 media_type = content_type.split(";", maxsplit=1)[0]
309 if media_type != "application/json":
308 - raise HTTPException(
309 - status_code=HTTPStatus.UNSUPPORTED_MEDIA_TYPE,
310 - detail="Unsupported Media Type: Only 'application/json' is allowed"
311 - )
310 + raise RequestValidationError(errors=[
311 + "Unsupported Media Type: Only 'application/json' is allowed"
312 + ])
313
314
315 router = APIRouter()
316
317
318 +class PrometheusResponse(Response):
319 + media_type = prometheus_client.CONTENT_TYPE_LATEST
320 +
321 +
322 def mount_metrics(app: FastAPI):
323 # Lazy import for prometheus multiprocessing.
324 # We need to set PROMETHEUS_MULTIPROC_DIR environment variable
@@ -332,6 +337,10 @@ def mount_metrics(app: FastAPI):
337 registry = CollectorRegistry()
338 multiprocess.MultiProcessCollector(registry)
339
340 + # `response_class=PrometheusResponse` is needed to return an HTTP response
341 + # with header "Content-Type: text/plain; version=0.0.4; charset=utf-8"
342 + # instead of the default "application/json" which is incorrect.
343 + # See https://github.com/trallnag/prometheus-fastapi-instrumentator/issues/163#issue-1296092364
344 Instrumentator(
345 excluded_handlers=[
346 "/metrics",
@@ -342,7 +351,7 @@ def mount_metrics(app: FastAPI):
351 "/server_info",
352 ],
353 registry=registry,
345 - ).add().instrument(app).expose(app)
354 + ).add().instrument(app).expose(app, response_class=PrometheusResponse)
355
356 # Add prometheus asgi middleware to route /metrics requests
357 metrics_route = Mount("/metrics", make_asgi_app(registry=registry))
@@ -401,11 +410,11 @@ def engine_client(request: Request) -> EngineClient:
410 return request.app.state.engine_client
411
412
404 -@router.get("/health")
405 -async def health(raw_request: Request) -> JSONResponse:
413 +@router.get("/health", response_class=Response)
414 +async def health(raw_request: Request) -> Response:
415 """Health check."""
416 await engine_client(raw_request).check_health()
408 - return JSONResponse(content={}, status_code=200)
417 + return Response(status_code=200)
418
419
420 @router.get("/load")
@@ -427,18 +436,42 @@ async def get_server_load_metrics(request: Request):
436 content={'server_load': request.app.state.server_load_metrics})
437
438
430 -@router.api_route("/ping", methods=["GET", "POST"])
431 -async def ping(raw_request: Request) -> JSONResponse:
439 +@router.get("/ping", response_class=Response)
440 +@router.post("/ping", response_class=Response)
441 +async def ping(raw_request: Request) -> Response:
442 """Ping check. Endpoint required for SageMaker"""
443 return await health(raw_request)
444
445
436 -@router.post("/tokenize", dependencies=[Depends(validate_json_request)])
446 +@router.post("/tokenize",
447 + dependencies=[Depends(validate_json_request)],
448 + responses={
449 + HTTPStatus.BAD_REQUEST.value: {
450 + "model": ErrorResponse
451 + },
452 + HTTPStatus.NOT_FOUND.value: {
453 + "model": ErrorResponse
454 + },
455 + HTTPStatus.INTERNAL_SERVER_ERROR.value: {
456 + "model": ErrorResponse
457 + },
458 + HTTPStatus.NOT_IMPLEMENTED.value: {
459 + "model": ErrorResponse
460 + },
461 + })
462 @with_cancellation
463 async def tokenize(request: TokenizeRequest, raw_request: Request):
464 handler = tokenization(raw_request)
465
441 - generator = await handler.create_tokenize(request, raw_request)
466 + try:
467 + generator = await handler.create_tokenize(request, raw_request)
468 + except NotImplementedError as e:
469 + raise HTTPException(status_code=HTTPStatus.NOT_IMPLEMENTED.value,
470 + detail=str(e)) from e
471 + except Exception as e:
472 + raise HTTPException(status_code=HTTPStatus.INTERNAL_SERVER_ERROR.value,
473 + detail=str(e)) from e
474 +
475 if isinstance(generator, ErrorResponse):
476 return JSONResponse(content=generator.model_dump(),
477 status_code=generator.code)
@@ -448,12 +481,31 @@ async def tokenize(request: TokenizeRequest, raw_request: Request):
481 assert_never(generator)
482
483
451 -@router.post("/detokenize", dependencies=[Depends(validate_json_request)])
484 +@router.post("/detokenize",
485 + dependencies=[Depends(validate_json_request)],
486 + responses={
487 + HTTPStatus.BAD_REQUEST.value: {
488 + "model": ErrorResponse
489 + },
490 + HTTPStatus.NOT_FOUND.value: {
491 + "model": ErrorResponse
492 + },
493 + HTTPStatus.INTERNAL_SERVER_ERROR.value: {
494 + "model": ErrorResponse
495 + },
496 + })
497 @with_cancellation
498 async def detokenize(request: DetokenizeRequest, raw_request: Request):
499 handler = tokenization(raw_request)
500
456 - generator = await handler.create_detokenize(request, raw_request)
501 + try:
502 + generator = await handler.create_detokenize(request, raw_request)
503 + except OverflowError as e:
504 + raise RequestValidationError(errors=[str(e)]) from e
505 + except Exception as e:
506 + raise HTTPException(status_code=HTTPStatus.INTERNAL_SERVER_ERROR.value,
507 + detail=str(e)) from e
508 +
509 if isinstance(generator, ErrorResponse):
510 return JSONResponse(content=generator.model_dump(),
511 status_code=generator.code)
@@ -478,7 +530,23 @@ async def show_version():
530
531
532 @router.post("/v1/chat/completions",
481 - dependencies=[Depends(validate_json_request)])
533 + dependencies=[Depends(validate_json_request)],
534 + responses={
535 + HTTPStatus.OK.value: {
536 + "content": {
537 + "text/event-stream": {}
538 + }
539 + },
540 + HTTPStatus.BAD_REQUEST.value: {
541 + "model": ErrorResponse
542 + },
543 + HTTPStatus.NOT_FOUND.value: {
544 + "model": ErrorResponse
545 + },
546 + HTTPStatus.INTERNAL_SERVER_ERROR.value: {
547 + "model": ErrorResponse
548 + }
549 + })
550 @with_cancellation
551 @load_aware_call
552 async def create_chat_completion(request: ChatCompletionRequest,
@@ -500,7 +568,24 @@ async def create_chat_completion(request: ChatCompletionRequest,
568 return StreamingResponse(content=generator, media_type="text/event-stream")
569
570
503 -@router.post("/v1/completions", dependencies=[Depends(validate_json_request)])
571 +@router.post("/v1/completions",
572 + dependencies=[Depends(validate_json_request)],
573 + responses={
574 + HTTPStatus.OK.value: {
575 + "content": {
576 + "text/event-stream": {}
577 + }
578 + },
579 + HTTPStatus.BAD_REQUEST.value: {
580 + "model": ErrorResponse
581 + },
582 + HTTPStatus.NOT_FOUND.value: {
583 + "model": ErrorResponse
584 + },
585 + HTTPStatus.INTERNAL_SERVER_ERROR.value: {
586 + "model": ErrorResponse
587 + },
588 + })
589 @with_cancellation
590 @load_aware_call
591 async def create_completion(request: CompletionRequest, raw_request: Request):
@@ -509,7 +594,15 @@ async def create_completion(request: CompletionRequest, raw_request: Request):
594 return base(raw_request).create_error_response(
595 message="The model does not support Completions API")
596
512 - generator = await handler.create_completion(request, raw_request)
597 + try:
598 + generator = await handler.create_completion(request, raw_request)
599 + except OverflowError as e:
600 + raise HTTPException(status_code=HTTPStatus.BAD_REQUEST.value,
601 + detail=str(e)) from e
602 + except Exception as e:
603 + raise HTTPException(status_code=HTTPStatus.INTERNAL_SERVER_ERROR.value,
604 + detail=str(e)) from e
605 +
606 if isinstance(generator, ErrorResponse):
607 return JSONResponse(content=generator.model_dump(),
608 status_code=generator.code)
@@ -519,7 +612,16 @@ async def create_completion(request: CompletionRequest, raw_request: Request):
612 return StreamingResponse(content=generator, media_type="text/event-stream")
613
614
522 -@router.post("/v1/embeddings", dependencies=[Depends(validate_json_request)])
615 +@router.post("/v1/embeddings",
616 + dependencies=[Depends(validate_json_request)],
617 + responses={
618 + HTTPStatus.BAD_REQUEST.value: {
619 + "model": ErrorResponse
620 + },
621 + HTTPStatus.INTERNAL_SERVER_ERROR.value: {
622 + "model": ErrorResponse
623 + },
624 + })
625 @with_cancellation
626 @load_aware_call
627 async def create_embedding(request: EmbeddingRequest, raw_request: Request):
@@ -566,7 +668,16 @@ async def create_embedding(request: EmbeddingRequest, raw_request: Request):
668 assert_never(generator)
669
670
569 -@router.post("/pooling", dependencies=[Depends(validate_json_request)])
671 +@router.post("/pooling",
672 + dependencies=[Depends(validate_json_request)],
673 + responses={
674 + HTTPStatus.BAD_REQUEST.value: {
675 + "model": ErrorResponse
676 + },
677 + HTTPStatus.INTERNAL_SERVER_ERROR.value: {
678 + "model": ErrorResponse
679 + },
680 + })
681 @with_cancellation
682 @load_aware_call
683 async def create_pooling(request: PoolingRequest, raw_request: Request):
@@ -606,7 +717,16 @@ async def create_classify(request: ClassificationRequest,
717 assert_never(generator)
718
719
609 -@router.post("/score", dependencies=[Depends(validate_json_request)])
720 +@router.post("/score",
721 + dependencies=[Depends(validate_json_request)],
722 + responses={
723 + HTTPStatus.BAD_REQUEST.value: {
724 + "model": ErrorResponse
725 + },
726 + HTTPStatus.INTERNAL_SERVER_ERROR.value: {
727 + "model": ErrorResponse
728 + },
729 + })
730 @with_cancellation
731 @load_aware_call
732 async def create_score(request: ScoreRequest, raw_request: Request):
@@ -625,7 +745,16 @@ async def create_score(request: ScoreRequest, raw_request: Request):
745 assert_never(generator)
746
747
628 -@router.post("/v1/score", dependencies=[Depends(validate_json_request)])
748 +@router.post("/v1/score",
749 + dependencies=[Depends(validate_json_request)],
750 + responses={
751 + HTTPStatus.BAD_REQUEST.value: {
752 + "model": ErrorResponse
753 + },
754 + HTTPStatus.INTERNAL_SERVER_ERROR.value: {
755 + "model": ErrorResponse
756 + },
757 + })
758 @with_cancellation
759 @load_aware_call
760 async def create_score_v1(request: ScoreRequest, raw_request: Request):
@@ -636,12 +765,28 @@ async def create_score_v1(request: ScoreRequest, raw_request: Request):
765 return await create_score(request, raw_request)
766
767
639 -@router.post("/v1/audio/transcriptions")
768 +@router.post("/v1/audio/transcriptions",
769 + responses={
770 + HTTPStatus.OK.value: {
771 + "content": {
772 + "text/event-stream": {}
773 + }
774 + },
775 + HTTPStatus.BAD_REQUEST.value: {
776 + "model": ErrorResponse
777 + },
778 + HTTPStatus.UNPROCESSABLE_ENTITY.value: {
779 + "model": ErrorResponse
780 + },
781 + HTTPStatus.INTERNAL_SERVER_ERROR.value: {
782 + "model": ErrorResponse
783 + },
784 + })
785 @with_cancellation
786 @load_aware_call
642 -async def create_transcriptions(request: Annotated[TranscriptionRequest,
643 - Form()],
644 - raw_request: Request):
787 +async def create_transcriptions(raw_request: Request,
788 + request: Annotated[TranscriptionRequest,
789 + Form()]):
790 handler = transcription(raw_request)
791 if handler is None:
792 return base(raw_request).create_error_response(
@@ -661,7 +806,16 @@ async def create_transcriptions(request: Annotated[TranscriptionRequest,
806 return StreamingResponse(content=generator, media_type="text/event-stream")
807
808
664 -@router.post("/rerank", dependencies=[Depends(validate_json_request)])
809 +@router.post("/rerank",
810 + dependencies=[Depends(validate_json_request)],
811 + responses={
812 + HTTPStatus.BAD_REQUEST.value: {
813 + "model": ErrorResponse
814 + },
815 + HTTPStatus.INTERNAL_SERVER_ERROR.value: {
816 + "model": ErrorResponse
817 + },
818 + })
819 @with_cancellation
820 @load_aware_call
821 async def do_rerank(request: RerankRequest, raw_request: Request):
@@ -679,7 +833,16 @@ async def do_rerank(request: RerankRequest, raw_request: Request):
833 assert_never(generator)
834
835
682 -@router.post("/v1/rerank", dependencies=[Depends(validate_json_request)])
836 +@router.post("/v1/rerank",
837 + dependencies=[Depends(validate_json_request)],
838 + responses={
839 + HTTPStatus.BAD_REQUEST.value: {
840 + "model": ErrorResponse
841 + },
842 + HTTPStatus.INTERNAL_SERVER_ERROR.value: {
843 + "model": ErrorResponse
844 + },
845 + })
846 @with_cancellation
847 async def do_rerank_v1(request: RerankRequest, raw_request: Request):
848 logger.warning_once(
@@ -690,7 +853,16 @@ async def do_rerank_v1(request: RerankRequest, raw_request: Request):
853 return await do_rerank(request, raw_request)
854
855
693 -@router.post("/v2/rerank", dependencies=[Depends(validate_json_request)])
856 +@router.post("/v2/rerank",
857 + dependencies=[Depends(validate_json_request)],
858 + responses={
859 + HTTPStatus.BAD_REQUEST.value: {
860 + "model": ErrorResponse
861 + },
862 + HTTPStatus.INTERNAL_SERVER_ERROR.value: {
863 + "model": ErrorResponse
864 + },
865 + })
866 @with_cancellation
867 async def do_rerank_v2(request: RerankRequest, raw_request: Request):
868 return await do_rerank(request, raw_request)
@@ -770,12 +942,29 @@ if envs.VLLM_SERVER_DEV_MODE:
942 return JSONResponse(content={"is_sleeping": is_sleeping})
943
944
773 -@router.post("/invocations", dependencies=[Depends(validate_json_request)])
945 +@router.post("/invocations",
946 + dependencies=[Depends(validate_json_request)],
947 + responses={
948 + HTTPStatus.BAD_REQUEST.value: {
949 + "model": ErrorResponse
950 + },
951 + HTTPStatus.UNSUPPORTED_MEDIA_TYPE.value: {
952 + "model": ErrorResponse
953 + },
954 + HTTPStatus.INTERNAL_SERVER_ERROR.value: {
955 + "model": ErrorResponse
956 + },
957 + })
958 async def invocations(raw_request: Request):
959 """
960 For SageMaker, routes requests to other handlers based on model `task`.
961 """
778 - body = await raw_request.json()
962 + try:
963 + body = await raw_request.json()
964 + except JSONDecodeError as e:
965 + raise HTTPException(status_code=HTTPStatus.BAD_REQUEST.value,
966 + detail=f"JSON decode error: {e}") from e
967 +
968 task = raw_request.app.state.task
969
970 if task not in TASK_HANDLERS:
@@ -866,10 +1055,26 @@ def build_app(args: Namespace) -> FastAPI:
1055 allow_headers=args.allowed_headers,
1056 )
1057
1058 + @app.exception_handler(HTTPException)
1059 + async def http_exception_handler(_: Request, exc: HTTPException):
1060 + err = ErrorResponse(message=exc.detail,
1061 + type=HTTPStatus(exc.status_code).phrase,
1062 + code=exc.status_code)
1063 + return JSONResponse(err.model_dump(), status_code=exc.status_code)
1064 +
1065 @app.exception_handler(RequestValidationError)
870 - async def validation_exception_handler(_, exc):
871 - err = ErrorResponse(message=str(exc),
872 - type="BadRequestError",
1066 + async def validation_exception_handler(_: Request,
1067 + exc: RequestValidationError):
1068 + exc_str = str(exc)
1069 + errors_str = str(exc.errors())
1070 +
1071 + if exc.errors() and errors_str and errors_str != exc_str:
1072 + message = f"{exc_str} {errors_str}"
1073 + else:
1074 + message = exc_str
1075 +
1076 + err = ErrorResponse(message=message,
1077 + type=HTTPStatus.BAD_REQUEST.phrase,
1078 code=HTTPStatus.BAD_REQUEST)
1079 return JSONResponse(err.model_dump(),
1080 status_code=HTTPStatus.BAD_REQUEST)
vllm/entrypoints/openai/serving_chat.py
+1 -1
@@ -197,7 +197,7 @@ class OpenAIServingChat(OpenAIServing):
197 except (ValueError, TypeError, RuntimeError,
198 jinja2.TemplateError) as e:
199 logger.exception("Error in preprocessing prompt inputs")
200 - return self.create_error_response(str(e))
200 + return self.create_error_response(f"{e} {e.__cause__}")
201
202 request_id = "chatcmpl-" \
203 f"{self._base_request_id(raw_request, request.request_id)}"
vllm/entrypoints/openai/serving_tokenization.py
+1 -1
@@ -91,7 +91,7 @@ class OpenAIServingTokenization(OpenAIServing):
91 )
92 except (ValueError, TypeError, jinja2.TemplateError) as e:
93 logger.exception("Error in preprocessing prompt inputs")
94 - return self.create_error_response(str(e))
94 + return self.create_error_response(f"{e} {e.__cause__}")
95
96 input_ids: list[int] = []
97 for i, engine_prompt in enumerate(engine_prompts):