@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 9b9cef3145
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 04:30 UTC

[Bugfix] Backport request id validation to v0 (#11036)

Signed-off-by: Joe Runde <Joseph.Runde@ibm.com>

Joe Runde committed Dec 10, 2024 at 09:38 UTC 9b9cef3145381721fa950c89718fe71849ac2a55
2 files changed +5 -1
vllm/engine/multiprocessing/client.py
+4
@@ -576,6 +576,10 @@ class MQLLMEngineClient(EngineClient):
576 if self._errored_with is not None:
577 raise ENGINE_DEAD_ERROR(self._errored_with)
578
579 + # Ensure the request id is unique among running requests
580 + if request_id in self.output_queues:
581 + raise ValueError(f"Request {request_id} already exists")
582 +
583 # Constructing guided decoding logits processors is expensive, so we do
584 # it here to avoid contending with cpu resources and the GIL on the
585 # backend process.
vllm/v1/engine/async_llm.py
+1 -1
@@ -152,7 +152,7 @@ class AsyncLLM(EngineClient):
152 """Add new request to the AsyncLLM."""
153
154 if self.detokenizer.is_request_active(request_id):
155 - raise KeyError(f"Request {request_id} already exists.")
155 + raise ValueError(f"Request {request_id} already exists.")
156
157 # 1) Create a new AsyncStream for the request.
158 stream = self._add_request_to_streams(request_id)