A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 8, 22:45 UTC
use ORJSONResponse when available to improve the efficiency of request process (#33548)
Signed-off-by: augusto.yjh <augusto.yjh@antgroup.com>
Augusto Yao committed
Feb 4, 2026 at 18:04 UTC
4c8d1bf361c0ad4066f2c90f636b5fabe80a94eb
1 file changed
+18
-1
vllm/entrypoints/pooling/embed/api_router.py
+18
-1
@@ -1,5 +1,7 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+import importlib.util
4
+from functools import lru_cache
5
from http import HTTPStatus
6
7
from fastapi import APIRouter, Depends, Request
@@ -15,9 +17,24 @@ from vllm.entrypoints.pooling.embed.protocol import (
17
)
18
from vllm.entrypoints.pooling.embed.serving import OpenAIServingEmbedding
19
from vllm.entrypoints.utils import load_aware_call, with_cancellation
20
+from vllm.logger import init_logger
21
22
router = APIRouter()
23
24
+logger = init_logger(__name__)
25
+
26
+
27
+@lru_cache(maxsize=1)
28
+def _get_json_response_cls():
29
+ if importlib.util.find_spec("orjson") is not None:
30
+ from fastapi.responses import ORJSONResponse
31
+
32
+ return ORJSONResponse
33
+ logger.warning_once(
34
+ "To make v1/embeddings API fast, please install orjson by `pip install orjson`"
35
+ )
36
+ return JSONResponse
37
+
38
39
def embedding(request: Request) -> OpenAIServingEmbedding | None:
40
return request.app.state.openai_serving_embedding
@@ -54,7 +71,7 @@ async def create_embedding(
71
content=generator.model_dump(), status_code=generator.error.code
72
)
73
elif isinstance(generator, EmbeddingResponse):
57
- return JSONResponse(content=generator.model_dump())
74
+ return _get_json_response_cls()(content=generator.model_dump())
75
elif isinstance(generator, EmbeddingBytesResponse):
76
return StreamingResponse(
77
content=generator.content,