A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 23:45 UTC
[Frontend] Add request_id to the Request object so they can be controlled better via external load balancers (#21009)
Signed-off-by: Kourosh Hakhamaneshi <kourosh@anyscale.com>
kourosh hakhamaneshi committed
Jul 25, 2025 at 06:49 UTC
9fe98d42508f9a67d5b1c491a4a306966ded3976
3 files changed
+27
-3
vllm/entrypoints/openai/protocol.py
+21
@@ -1007,6 +1007,13 @@ class CompletionRequest(OpenAIBaseModel):
1007
"default: 0). Any priority other than 0 will raise an error "
1008
"if the served model does not use priority scheduling."),
1009
)
1010
+ request_id: str = Field(
1011
+ default_factory=lambda: f"{random_uuid()}",
1012
+ description=(
1013
+ "The request_id related to this request. If the caller does "
1014
+ "not set it, a random_uuid will be generated. This id is used "
1015
+ "through out the inference process and return in response."),
1016
+ )
1017
logits_processors: Optional[LogitsProcessors] = Field(
1018
default=None,
1019
description=(
@@ -1251,6 +1258,13 @@ class EmbeddingCompletionRequest(OpenAIBaseModel):
1258
"default: 0). Any priority other than 0 will raise an error "
1259
"if the served model does not use priority scheduling."),
1260
)
1261
+ request_id: str = Field(
1262
+ default_factory=lambda: f"{random_uuid()}",
1263
+ description=(
1264
+ "The request_id related to this request. If the caller does "
1265
+ "not set it, a random_uuid will be generated. This id is used "
1266
+ "through out the inference process and return in response."),
1267
+ )
1268
1269
# --8<-- [end:embedding-extra-params]
1270
@@ -1302,6 +1316,13 @@ class EmbeddingChatRequest(OpenAIBaseModel):
1316
"default: 0). Any priority other than 0 will raise an error "
1317
"if the served model does not use priority scheduling."),
1318
)
1319
+ request_id: str = Field(
1320
+ default_factory=lambda: f"{random_uuid()}",
1321
+ description=(
1322
+ "The request_id related to this request. If the caller does "
1323
+ "not set it, a random_uuid will be generated. This id is used "
1324
+ "through out the inference process and return in response."),
1325
+ )
1326
# --8<-- [end:chat-embedding-extra-params]
1327
1328
@model_validator(mode="before")
vllm/entrypoints/openai/serving_completion.py
+3
-1
@@ -113,7 +113,9 @@ class OpenAIServingCompletion(OpenAIServing):
113
return self.create_error_response(
114
"Echo is unsupported with prompt embeds.")
115
116
- request_id = f"cmpl-{self._base_request_id(raw_request)}"
116
+ request_id = (
117
+ f"cmpl-"
118
+ f"{self._base_request_id(raw_request, request.request_id)}")
119
created_time = int(time.time())
120
121
request_metadata = RequestResponseMetadata(request_id=request_id)
vllm/entrypoints/openai/serving_embedding.py
+3
-2
@@ -163,8 +163,9 @@ class OpenAIServingEmbedding(EmbeddingMixin):
163
for the API specification. This API mimics the OpenAI Embedding API.
164
"""
165
model_name = self._get_model_name(request.model)
166
- request_id = (f"{self.request_id_prefix}-"
167
- f"{self._base_request_id(raw_request)}")
166
+ request_id = (
167
+ f"{self.request_id_prefix}-"
168
+ f"{self._base_request_id(raw_request, request.request_id)}")
169
170
ctx = EmbeddingServeContext(
171
request=request,