A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 13, 22:00 UTC
Fix(llm): Abort orphaned requests when llm.chat() batch fails Fixes #26081 (#27420)
Signed-off-by: vensenmu <vensenmu@gmail.com>
Vensen committed
Nov 3, 2025 at 00:24 UTC
0ce743f4e1879ffa250e471f6894633ef125418e
2 files changed
+75
-14
tests/entrypoints/llm/test_chat.py
+53
@@ -6,6 +6,7 @@ import pytest
6
7
from vllm import LLM
8
from vllm.distributed import cleanup_dist_env_and_memory
9
+from vllm.sampling_params import SamplingParams
10
11
from ..openai.test_vision import TEST_IMAGE_ASSETS
12
@@ -23,6 +24,29 @@ def text_llm():
24
cleanup_dist_env_and_memory()
25
26
27
+@pytest.fixture(scope="function")
28
+def llm_for_failure_test():
29
+ """
30
+ Fixture for testing issue #26081.
31
+ Uses a small max_model_len to easily trigger length errors.
32
+ """
33
+ # pytest caches the fixture so we use weakref.proxy to
34
+ # enable garbage collection
35
+ llm = LLM(
36
+ model="meta-llama/Llama-3.2-1B-Instruct",
37
+ enforce_eager=True,
38
+ seed=0,
39
+ max_model_len=128,
40
+ disable_log_stats=True,
41
+ )
42
+
43
+ yield weakref.proxy(llm)
44
+
45
+ del llm
46
+
47
+ cleanup_dist_env_and_memory()
48
+
49
+
50
def test_chat(text_llm):
51
prompt1 = "Explain the concept of entropy."
52
messages = [
@@ -157,3 +181,32 @@ def test_chat_extra_kwargs(thinking_llm, enable_thinking):
181
else:
182
# The chat template includes dummy thinking process
183
assert think_id in prompt_token_ids
184
+
185
+
186
+def test_chat_batch_failure_cleanup(llm_for_failure_test):
187
+ """
188
+ Tests that if a batch call to llm.chat() fails mid-way
189
+ (e.g., due to one invalid prompt), the requests that
190
+ were already enqueued are properly aborted and do not
191
+ pollute the queue for subsequent calls.
192
+ (Fixes Issue #26081)
193
+ """
194
+ llm = llm_for_failure_test
195
+ valid_msg = [{"role": "user", "content": "Hello"}]
196
+ long_text = "This is a very long text to test the error " * 50
197
+ invalid_msg = [{"role": "user", "content": long_text}]
198
+ batch_1 = [
199
+ valid_msg,
200
+ valid_msg,
201
+ invalid_msg,
202
+ ]
203
+ batch_2 = [
204
+ valid_msg,
205
+ valid_msg,
206
+ ]
207
+ sampling_params = SamplingParams(temperature=0, max_tokens=10)
208
+ with pytest.raises(ValueError, match="longer than the maximum model length"):
209
+ llm.chat(batch_1, sampling_params=sampling_params)
210
+ outputs_2 = llm.chat(batch_2, sampling_params=sampling_params)
211
+ assert len(outputs_2) == len(batch_2)
212
+ assert llm.llm_engine.get_num_unfinished_requests() == 0
vllm/entrypoints/llm.py
+22
-14
@@ -1588,20 +1588,27 @@ class LLM:
1588
tqdm_func = use_tqdm if callable(use_tqdm) else tqdm
1589
it = tqdm_func(it, desc="Adding requests")
1590
1591
- for i, prompt in enumerate(it):
1592
- if isinstance(prompt, dict):
1593
- self._validate_mm_data_and_uuids(
1594
- prompt.get("multi_modal_data"), prompt.get("multi_modal_uuids")
1595
- )
1591
+ added_request_ids: list[str] = []
1592
1597
- self._add_request(
1598
- prompt,
1599
- params[i] if isinstance(params, Sequence) else params,
1600
- lora_request=lora_request[i]
1601
- if isinstance(lora_request, Sequence)
1602
- else lora_request,
1603
- priority=priority[i] if priority else 0,
1604
- )
1593
+ try:
1594
+ for i, prompt in enumerate(it):
1595
+ if isinstance(prompt, dict):
1596
+ self._validate_mm_data_and_uuids(
1597
+ prompt.get("multi_modal_data"), prompt.get("multi_modal_uuids")
1598
+ )
1599
+ request_id = self._add_request(
1600
+ prompt,
1601
+ params[i] if isinstance(params, Sequence) else params,
1602
+ lora_request=lora_request[i]
1603
+ if isinstance(lora_request, Sequence)
1604
+ else lora_request,
1605
+ priority=priority[i] if priority else 0,
1606
+ )
1607
+ added_request_ids.append(request_id)
1608
+ except Exception as e:
1609
+ if added_request_ids:
1610
+ self.llm_engine.abort_request(added_request_ids)
1611
+ raise e
1612
1613
def _validate_mm_data_and_uuids(
1614
self,
@@ -1684,7 +1691,7 @@ class LLM:
1691
params: SamplingParams | PoolingParams,
1692
lora_request: LoRARequest | None = None,
1693
priority: int = 0,
1687
- ) -> None:
1694
+ ) -> str:
1695
prompt_text, _, _ = get_prompt_components(prompt)
1696
request_id = str(next(self.request_counter))
1697
@@ -1705,6 +1712,7 @@ class LLM:
1712
priority=priority,
1713
prompt_text=prompt_text,
1714
)
1715
+ return request_id
1716
1717
def _run_engine(
1718
self, *, use_tqdm: bool | Callable[..., tqdm] = True