A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 04:30 UTC
[Bugfix] Backport request id validation to v0 (#11036)
Signed-off-by: Joe Runde <Joseph.Runde@ibm.com>
Joe Runde committed
Dec 10, 2024 at 09:38 UTC
9b9cef3145381721fa950c89718fe71849ac2a55
2 files changed
+5
-1
vllm/engine/multiprocessing/client.py
+4
@@ -576,6 +576,10 @@ class MQLLMEngineClient(EngineClient):
576
if self._errored_with is not None:
577
raise ENGINE_DEAD_ERROR(self._errored_with)
578
579
+ # Ensure the request id is unique among running requests
580
+ if request_id in self.output_queues:
581
+ raise ValueError(f"Request {request_id} already exists")
582
+
583
# Constructing guided decoding logits processors is expensive, so we do
584
# it here to avoid contending with cpu resources and the GIL on the
585
# backend process.
vllm/v1/engine/async_llm.py
+1
-1
@@ -152,7 +152,7 @@ class AsyncLLM(EngineClient):
152
"""Add new request to the AsyncLLM."""
153
154
if self.detokenizer.is_request_active(request_id):
155
- raise KeyError(f"Request {request_id} already exists.")
155
+ raise ValueError(f"Request {request_id} already exists.")
156
157
# 1) Create a new AsyncStream for the request.
158
stream = self._add_request_to_streams(request_id)