@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 37dfa60037
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 15, 23:30 UTC

[Bugfix] Missing Content Type returns 500 Internal Server Error (#13193)

Vaibhav Jain committed Feb 13, 2025 at 20:22 UTC 37dfa6003782d872a375cc41429ca16fe241f83f
2 files changed +43 -15
tests/entrypoints/openai/test_basic.py
+16
@@ -156,3 +156,19 @@ async def test_request_cancellation(server: RemoteOpenAIServer):
156 max_tokens=10)
157
158 assert len(response.choices) == 1
159 +
160 +
161 +@pytest.mark.asyncio
162 +async def test_request_wrong_content_type(server: RemoteOpenAIServer):
163 +
164 + chat_input = [{"role": "user", "content": "Write a long story"}]
165 + client = server.get_async_client()
166 +
167 + with pytest.raises(openai.APIStatusError):
168 + await client.chat.completions.create(
169 + messages=chat_input,
170 + model=MODEL_NAME,
171 + max_tokens=10000,
172 + extra_headers={
173 + "Content-Type": "application/x-www-form-urlencoded"
174 + })
vllm/entrypoints/openai/api_server.py
+27 -15
@@ -19,7 +19,7 @@ from http import HTTPStatus
19 from typing import AsyncIterator, Dict, Optional, Set, Tuple, Union
20
21 import uvloop
22 -from fastapi import APIRouter, FastAPI, HTTPException, Request
22 +from fastapi import APIRouter, Depends, FastAPI, HTTPException, Request
23 from fastapi.exceptions import RequestValidationError
24 from fastapi.middleware.cors import CORSMiddleware
25 from fastapi.responses import JSONResponse, Response, StreamingResponse
@@ -252,6 +252,15 @@ async def build_async_engine_client_from_engine_args(
252 multiprocess.mark_process_dead(engine_process.pid)
253
254
255 +async def validate_json_request(raw_request: Request):
256 + content_type = raw_request.headers.get("content-type", "").lower()
257 + if content_type != "application/json":
258 + raise HTTPException(
259 + status_code=HTTPStatus.UNSUPPORTED_MEDIA_TYPE,
260 + detail="Unsupported Media Type: Only 'application/json' is allowed"
261 + )
262 +
263 +
264 router = APIRouter()
265
266
@@ -335,7 +344,7 @@ async def ping(raw_request: Request) -> Response:
344 return await health(raw_request)
345
346
338 -@router.post("/tokenize")
347 +@router.post("/tokenize", dependencies=[Depends(validate_json_request)])
348 @with_cancellation
349 async def tokenize(request: TokenizeRequest, raw_request: Request):
350 handler = tokenization(raw_request)
@@ -350,7 +359,7 @@ async def tokenize(request: TokenizeRequest, raw_request: Request):
359 assert_never(generator)
360
361
353 -@router.post("/detokenize")
362 +@router.post("/detokenize", dependencies=[Depends(validate_json_request)])
363 @with_cancellation
364 async def detokenize(request: DetokenizeRequest, raw_request: Request):
365 handler = tokenization(raw_request)
@@ -379,7 +388,8 @@ async def show_version():
388 return JSONResponse(content=ver)
389
390
382 -@router.post("/v1/chat/completions")
391 +@router.post("/v1/chat/completions",
392 + dependencies=[Depends(validate_json_request)])
393 @with_cancellation
394 async def create_chat_completion(request: ChatCompletionRequest,
395 raw_request: Request):
@@ -400,7 +410,7 @@ async def create_chat_completion(request: ChatCompletionRequest,
410 return StreamingResponse(content=generator, media_type="text/event-stream")
411
412
403 -@router.post("/v1/completions")
413 +@router.post("/v1/completions", dependencies=[Depends(validate_json_request)])
414 @with_cancellation
415 async def create_completion(request: CompletionRequest, raw_request: Request):
416 handler = completion(raw_request)
@@ -418,7 +428,7 @@ async def create_completion(request: CompletionRequest, raw_request: Request):
428 return StreamingResponse(content=generator, media_type="text/event-stream")
429
430
421 -@router.post("/v1/embeddings")
431 +@router.post("/v1/embeddings", dependencies=[Depends(validate_json_request)])
432 @with_cancellation
433 async def create_embedding(request: EmbeddingRequest, raw_request: Request):
434 handler = embedding(raw_request)
@@ -464,7 +474,7 @@ async def create_embedding(request: EmbeddingRequest, raw_request: Request):
474 assert_never(generator)
475
476
467 -@router.post("/pooling")
477 +@router.post("/pooling", dependencies=[Depends(validate_json_request)])
478 @with_cancellation
479 async def create_pooling(request: PoolingRequest, raw_request: Request):
480 handler = pooling(raw_request)
@@ -482,7 +492,7 @@ async def create_pooling(request: PoolingRequest, raw_request: Request):
492 assert_never(generator)
493
494
485 -@router.post("/score")
495 +@router.post("/score", dependencies=[Depends(validate_json_request)])
496 @with_cancellation
497 async def create_score(request: ScoreRequest, raw_request: Request):
498 handler = score(raw_request)
@@ -500,7 +510,7 @@ async def create_score(request: ScoreRequest, raw_request: Request):
510 assert_never(generator)
511
512
503 -@router.post("/v1/score")
513 +@router.post("/v1/score", dependencies=[Depends(validate_json_request)])
514 @with_cancellation
515 async def create_score_v1(request: ScoreRequest, raw_request: Request):
516 logger.warning(
@@ -510,7 +520,7 @@ async def create_score_v1(request: ScoreRequest, raw_request: Request):
520 return await create_score(request, raw_request)
521
522
513 -@router.post("/rerank")
523 +@router.post("/rerank", dependencies=[Depends(validate_json_request)])
524 @with_cancellation
525 async def do_rerank(request: RerankRequest, raw_request: Request):
526 handler = rerank(raw_request)
@@ -527,7 +537,7 @@ async def do_rerank(request: RerankRequest, raw_request: Request):
537 assert_never(generator)
538
539
530 -@router.post("/v1/rerank")
540 +@router.post("/v1/rerank", dependencies=[Depends(validate_json_request)])
541 @with_cancellation
542 async def do_rerank_v1(request: RerankRequest, raw_request: Request):
543 logger.warning_once(
@@ -538,7 +548,7 @@ async def do_rerank_v1(request: RerankRequest, raw_request: Request):
548 return await do_rerank(request, raw_request)
549
550
541 -@router.post("/v2/rerank")
551 +@router.post("/v2/rerank", dependencies=[Depends(validate_json_request)])
552 @with_cancellation
553 async def do_rerank_v2(request: RerankRequest, raw_request: Request):
554 return await do_rerank(request, raw_request)
@@ -582,7 +592,7 @@ if envs.VLLM_SERVER_DEV_MODE:
592 return Response(status_code=200)
593
594
585 -@router.post("/invocations")
595 +@router.post("/invocations", dependencies=[Depends(validate_json_request)])
596 async def invocations(raw_request: Request):
597 """
598 For SageMaker, routes requests to other handlers based on model `task`.
@@ -632,7 +642,8 @@ if envs.VLLM_ALLOW_RUNTIME_LORA_UPDATING:
642 "Lora dynamic loading & unloading is enabled in the API server. "
643 "This should ONLY be used for local development!")
644
635 - @router.post("/v1/load_lora_adapter")
645 + @router.post("/v1/load_lora_adapter",
646 + dependencies=[Depends(validate_json_request)])
647 async def load_lora_adapter(request: LoadLoraAdapterRequest,
648 raw_request: Request):
649 handler = models(raw_request)
@@ -643,7 +654,8 @@ if envs.VLLM_ALLOW_RUNTIME_LORA_UPDATING:
654
655 return Response(status_code=200, content=response)
656
646 - @router.post("/v1/unload_lora_adapter")
657 + @router.post("/v1/unload_lora_adapter",
658 + dependencies=[Depends(validate_json_request)])
659 async def unload_lora_adapter(request: UnloadLoraAdapterRequest,
660 raw_request: Request):
661 handler = models(raw_request)