A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 23:30 UTC
[Bugfix] Missing Content Type returns 500 Internal Server Error (#13193)
Vaibhav Jain committed
Feb 13, 2025 at 20:22 UTC
37dfa6003782d872a375cc41429ca16fe241f83f
2 files changed
+43
-15
tests/entrypoints/openai/test_basic.py
+16
@@ -156,3 +156,19 @@ async def test_request_cancellation(server: RemoteOpenAIServer):
156
max_tokens=10)
157
158
assert len(response.choices) == 1
159
+
160
+
161
+@pytest.mark.asyncio
162
+async def test_request_wrong_content_type(server: RemoteOpenAIServer):
163
+
164
+ chat_input = [{"role": "user", "content": "Write a long story"}]
165
+ client = server.get_async_client()
166
+
167
+ with pytest.raises(openai.APIStatusError):
168
+ await client.chat.completions.create(
169
+ messages=chat_input,
170
+ model=MODEL_NAME,
171
+ max_tokens=10000,
172
+ extra_headers={
173
+ "Content-Type": "application/x-www-form-urlencoded"
174
+ })
vllm/entrypoints/openai/api_server.py
+27
-15
@@ -19,7 +19,7 @@ from http import HTTPStatus
19
from typing import AsyncIterator, Dict, Optional, Set, Tuple, Union
20
21
import uvloop
22
-from fastapi import APIRouter, FastAPI, HTTPException, Request
22
+from fastapi import APIRouter, Depends, FastAPI, HTTPException, Request
23
from fastapi.exceptions import RequestValidationError
24
from fastapi.middleware.cors import CORSMiddleware
25
from fastapi.responses import JSONResponse, Response, StreamingResponse
@@ -252,6 +252,15 @@ async def build_async_engine_client_from_engine_args(
252
multiprocess.mark_process_dead(engine_process.pid)
253
254
255
+async def validate_json_request(raw_request: Request):
256
+ content_type = raw_request.headers.get("content-type", "").lower()
257
+ if content_type != "application/json":
258
+ raise HTTPException(
259
+ status_code=HTTPStatus.UNSUPPORTED_MEDIA_TYPE,
260
+ detail="Unsupported Media Type: Only 'application/json' is allowed"
261
+ )
262
+
263
+
264
router = APIRouter()
265
266
@@ -335,7 +344,7 @@ async def ping(raw_request: Request) -> Response:
344
return await health(raw_request)
345
346
338
-@router.post("/tokenize")
347
+@router.post("/tokenize", dependencies=[Depends(validate_json_request)])
348
@with_cancellation
349
async def tokenize(request: TokenizeRequest, raw_request: Request):
350
handler = tokenization(raw_request)
@@ -350,7 +359,7 @@ async def tokenize(request: TokenizeRequest, raw_request: Request):
359
assert_never(generator)
360
361
353
-@router.post("/detokenize")
362
+@router.post("/detokenize", dependencies=[Depends(validate_json_request)])
363
@with_cancellation
364
async def detokenize(request: DetokenizeRequest, raw_request: Request):
365
handler = tokenization(raw_request)
@@ -379,7 +388,8 @@ async def show_version():
388
return JSONResponse(content=ver)
389
390
382
-@router.post("/v1/chat/completions")
391
+@router.post("/v1/chat/completions",
392
+ dependencies=[Depends(validate_json_request)])
393
@with_cancellation
394
async def create_chat_completion(request: ChatCompletionRequest,
395
raw_request: Request):
@@ -400,7 +410,7 @@ async def create_chat_completion(request: ChatCompletionRequest,
410
return StreamingResponse(content=generator, media_type="text/event-stream")
411
412
403
-@router.post("/v1/completions")
413
+@router.post("/v1/completions", dependencies=[Depends(validate_json_request)])
414
@with_cancellation
415
async def create_completion(request: CompletionRequest, raw_request: Request):
416
handler = completion(raw_request)
@@ -418,7 +428,7 @@ async def create_completion(request: CompletionRequest, raw_request: Request):
428
return StreamingResponse(content=generator, media_type="text/event-stream")
429
430
421
-@router.post("/v1/embeddings")
431
+@router.post("/v1/embeddings", dependencies=[Depends(validate_json_request)])
432
@with_cancellation
433
async def create_embedding(request: EmbeddingRequest, raw_request: Request):
434
handler = embedding(raw_request)
@@ -464,7 +474,7 @@ async def create_embedding(request: EmbeddingRequest, raw_request: Request):
474
assert_never(generator)
475
476
467
-@router.post("/pooling")
477
+@router.post("/pooling", dependencies=[Depends(validate_json_request)])
478
@with_cancellation
479
async def create_pooling(request: PoolingRequest, raw_request: Request):
480
handler = pooling(raw_request)
@@ -482,7 +492,7 @@ async def create_pooling(request: PoolingRequest, raw_request: Request):
492
assert_never(generator)
493
494
485
-@router.post("/score")
495
+@router.post("/score", dependencies=[Depends(validate_json_request)])
496
@with_cancellation
497
async def create_score(request: ScoreRequest, raw_request: Request):
498
handler = score(raw_request)
@@ -500,7 +510,7 @@ async def create_score(request: ScoreRequest, raw_request: Request):
510
assert_never(generator)
511
512
503
-@router.post("/v1/score")
513
+@router.post("/v1/score", dependencies=[Depends(validate_json_request)])
514
@with_cancellation
515
async def create_score_v1(request: ScoreRequest, raw_request: Request):
516
logger.warning(
@@ -510,7 +520,7 @@ async def create_score_v1(request: ScoreRequest, raw_request: Request):
520
return await create_score(request, raw_request)
521
522
513
-@router.post("/rerank")
523
+@router.post("/rerank", dependencies=[Depends(validate_json_request)])
524
@with_cancellation
525
async def do_rerank(request: RerankRequest, raw_request: Request):
526
handler = rerank(raw_request)
@@ -527,7 +537,7 @@ async def do_rerank(request: RerankRequest, raw_request: Request):
537
assert_never(generator)
538
539
530
-@router.post("/v1/rerank")
540
+@router.post("/v1/rerank", dependencies=[Depends(validate_json_request)])
541
@with_cancellation
542
async def do_rerank_v1(request: RerankRequest, raw_request: Request):
543
logger.warning_once(
@@ -538,7 +548,7 @@ async def do_rerank_v1(request: RerankRequest, raw_request: Request):
548
return await do_rerank(request, raw_request)
549
550
541
-@router.post("/v2/rerank")
551
+@router.post("/v2/rerank", dependencies=[Depends(validate_json_request)])
552
@with_cancellation
553
async def do_rerank_v2(request: RerankRequest, raw_request: Request):
554
return await do_rerank(request, raw_request)
@@ -582,7 +592,7 @@ if envs.VLLM_SERVER_DEV_MODE:
592
return Response(status_code=200)
593
594
585
-@router.post("/invocations")
595
+@router.post("/invocations", dependencies=[Depends(validate_json_request)])
596
async def invocations(raw_request: Request):
597
"""
598
For SageMaker, routes requests to other handlers based on model `task`.
@@ -632,7 +642,8 @@ if envs.VLLM_ALLOW_RUNTIME_LORA_UPDATING:
642
"Lora dynamic loading & unloading is enabled in the API server. "
643
"This should ONLY be used for local development!")
644
635
- @router.post("/v1/load_lora_adapter")
645
+ @router.post("/v1/load_lora_adapter",
646
+ dependencies=[Depends(validate_json_request)])
647
async def load_lora_adapter(request: LoadLoraAdapterRequest,
648
raw_request: Request):
649
handler = models(raw_request)
@@ -643,7 +654,8 @@ if envs.VLLM_ALLOW_RUNTIME_LORA_UPDATING:
654
655
return Response(status_code=200, content=response)
656
646
- @router.post("/v1/unload_lora_adapter")
657
+ @router.post("/v1/unload_lora_adapter",
658
+ dependencies=[Depends(validate_json_request)])
659
async def unload_lora_adapter(request: UnloadLoraAdapterRequest,
660
raw_request: Request):
661
handler = models(raw_request)