A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 09:30 UTC
[Frontend] Support custom request_id from request (#9550)
Co-authored-by: Yuhong Guo <yuhong.gyh@antgroup.com>
Yuhong Guo committed
Oct 23, 2024 at 02:07 UTC
434984e665fe4134ec749de5f1c412b7a1e647a1
2 files changed
+8
-2
vllm/entrypoints/openai/protocol.py
+6
@@ -284,6 +284,12 @@ class ChatCompletionRequest(OpenAIBaseModel):
284
"The priority of the request (lower means earlier handling; "
285
"default: 0). Any priority other than 0 will raise an error "
286
"if the served model does not use priority scheduling."))
287
+ request_id: str = Field(
288
+ default_factory=lambda: f"{random_uuid()}",
289
+ description=(
290
+ "The request_id related to this request. If the caller does "
291
+ "not set it, a random_uuid will be generated. This id is used "
292
+ "through out the inference process and return in response."))
293
294
# doc: end-chat-completion-extra-params
295
vllm/entrypoints/openai/serving_chat.py
+2
-2
@@ -38,7 +38,7 @@ from vllm.sequence import Logprob
38
from vllm.tracing import (contains_trace_headers, extract_trace_headers,
39
log_tracing_disabled_warning)
40
from vllm.transformers_utils.tokenizer import AnyTokenizer, MistralTokenizer
41
-from vllm.utils import iterate_with_cancellation, random_uuid
41
+from vllm.utils import iterate_with_cancellation
42
43
logger = init_logger(__name__)
44
@@ -176,7 +176,7 @@ class OpenAIServingChat(OpenAIServing):
176
"\"auto\" tool choice requires "
177
"--enable-auto-tool-choice and --tool-call-parser to be set")
178
179
- request_id = f"chat-{random_uuid()}"
179
+ request_id = f"chat-{request.request_id}"
180
181
request_metadata = RequestResponseMetadata(request_id=request_id)
182
if raw_request: