@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 434984e665
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 09:30 UTC

[Frontend] Support custom request_id from request (#9550)

Co-authored-by: Yuhong Guo <yuhong.gyh@antgroup.com>

Yuhong Guo committed Oct 23, 2024 at 02:07 UTC 434984e665fe4134ec749de5f1c412b7a1e647a1
2 files changed +8 -2
vllm/entrypoints/openai/protocol.py
+6
@@ -284,6 +284,12 @@ class ChatCompletionRequest(OpenAIBaseModel):
284 "The priority of the request (lower means earlier handling; "
285 "default: 0). Any priority other than 0 will raise an error "
286 "if the served model does not use priority scheduling."))
287 + request_id: str = Field(
288 + default_factory=lambda: f"{random_uuid()}",
289 + description=(
290 + "The request_id related to this request. If the caller does "
291 + "not set it, a random_uuid will be generated. This id is used "
292 + "through out the inference process and return in response."))
293
294 # doc: end-chat-completion-extra-params
295
vllm/entrypoints/openai/serving_chat.py
+2 -2
@@ -38,7 +38,7 @@ from vllm.sequence import Logprob
38 from vllm.tracing import (contains_trace_headers, extract_trace_headers,
39 log_tracing_disabled_warning)
40 from vllm.transformers_utils.tokenizer import AnyTokenizer, MistralTokenizer
41 -from vllm.utils import iterate_with_cancellation, random_uuid
41 +from vllm.utils import iterate_with_cancellation
42
43 logger = init_logger(__name__)
44
@@ -176,7 +176,7 @@ class OpenAIServingChat(OpenAIServing):
176 "\"auto\" tool choice requires "
177 "--enable-auto-tool-choice and --tool-call-parser to be set")
178
179 - request_id = f"chat-{random_uuid()}"
179 + request_id = f"chat-{request.request_id}"
180
181 request_metadata = RequestResponseMetadata(request_id=request_id)
182 if raw_request: