A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 11, 05:00 UTC
[BugFix] Handle errors when preprocessing added requests (#30895)
Signed-off-by: Nick Hill <nhill@redhat.com>
Nick Hill committed
Dec 18, 2025 at 17:29 UTC
45c0526ac9517ae52a91e3e07bd023ce9f1a7c69
3 files changed
+93
-3
.buildkite/test-pipeline.yaml
+4
-1
@@ -319,7 +319,10 @@ steps:
319
# TODO: accuracy does not match, whether setting
320
# VLLM_USE_FLASHINFER_SAMPLER or not on H100.
321
- pytest -v -s v1/e2e
322
- - pytest -v -s v1/engine
322
+ # Run this test standalone for now;
323
+ # need to untangle use (implicit) use of spawn/fork across the tests.
324
+ - pytest -v -s v1/engine/test_preprocess_error_handling.py
325
+ - pytest -v -s v1/engine --ignore v1/engine/test_preprocess_error_handling.py
326
327
- label: V1 Test entrypoints # 35min
328
timeout_in_minutes: 50
tests/v1/engine/test_preprocess_error_handling.py
new
+56
@@ -0,0 +1,56 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+import pytest
5
+import torch.cuda
6
+
7
+from vllm import LLM, SamplingParams
8
+from vllm.v1.engine import EngineCoreRequest
9
+from vllm.v1.engine.core import EngineCore
10
+
11
+MODEL_NAME = "hmellor/tiny-random-LlamaForCausalLM"
12
+
13
+
14
+def test_preprocess_error_handling(monkeypatch: pytest.MonkeyPatch):
15
+ """Test that preprocessing errors are handled gracefully."""
16
+
17
+ assert not torch.cuda.is_initialized(), (
18
+ "fork needs to be used for the engine "
19
+ "core process and this isn't possible if cuda is already initialized"
20
+ )
21
+
22
+ # Store original method to call for non-failing requests
23
+ original_preprocess = EngineCore.preprocess_add_request
24
+
25
+ # Monkeypatch to make preprocess_add_request raise an exception
26
+ # only for requests with "FAIL" in the first token
27
+ def conditional_failing_preprocess(self, request: EngineCoreRequest):
28
+ # Fail if the first token id is 333
29
+ if request.prompt_token_ids and request.prompt_token_ids[0] == 333:
30
+ raise ValueError("Simulated preprocessing error!")
31
+ return original_preprocess(self, request)
32
+
33
+ monkeypatch.setattr(
34
+ EngineCore, "preprocess_add_request", conditional_failing_preprocess
35
+ )
36
+
37
+ llm = LLM(model=MODEL_NAME)
38
+
39
+ # Create a failing request by crafting a request with an invalid token
40
+ # We need to use a direct approach since LLM.generate tokenizes for us
41
+ from vllm.inputs import TokensPrompt
42
+
43
+ # This should raise an exception due to the preprocessing failure
44
+ # Special token id to trigger the failure
45
+ failing_prompt = TokensPrompt(prompt_token_ids=[333])
46
+ outputs = llm.generate(failing_prompt, SamplingParams(max_tokens=10)) # type: ignore
47
+ assert len(outputs) == 1
48
+ assert len(outputs[0].outputs[0].token_ids) == 0
49
+ assert outputs[0].finished
50
+ assert outputs[0].outputs[0].finish_reason == "error"
51
+
52
+ # Verify the engine is still functional with a normal request
53
+ outputs = llm.generate("Hello, my name is", SamplingParams(max_tokens=10))
54
+ assert len(outputs) == 1
55
+ assert len(outputs[0].outputs[0].token_ids) > 0
56
+ assert outputs[0].outputs[0].finish_reason in ("stop", "length")
vllm/v1/engine/core.py
+33
-2
@@ -43,9 +43,11 @@ from vllm.v1.core.kv_cache_utils import (
43
from vllm.v1.core.sched.interface import SchedulerInterface
44
from vllm.v1.core.sched.output import SchedulerOutput
45
from vllm.v1.engine import (
46
+ EngineCoreOutput,
47
EngineCoreOutputs,
48
EngineCoreRequest,
49
EngineCoreRequestType,
50
+ FinishReason,
51
ReconfigureDistributedRequest,
52
ReconfigureRankType,
53
UtilityOutput,
@@ -1055,9 +1057,14 @@ class EngineCoreProc(EngineCore):
1057
request_type = EngineCoreRequestType(bytes(type_frame.buffer))
1058
1059
# Deserialize the request data.
1060
+ request: Any
1061
if request_type == EngineCoreRequestType.ADD:
1059
- request = add_request_decoder.decode(data_frames)
1060
- request = self.preprocess_add_request(request)
1062
+ req: EngineCoreRequest = add_request_decoder.decode(data_frames)
1063
+ try:
1064
+ request = self.preprocess_add_request(req)
1065
+ except Exception:
1066
+ self._handle_request_preproc_error(req)
1067
+ continue
1068
else:
1069
request = generic_decoder.decode(data_frames)
1070
@@ -1141,6 +1148,30 @@ class EngineCoreProc(EngineCore):
1148
# Limit the number of buffers to reuse.
1149
reuse_buffers.append(buffer)
1150
1151
+ def _handle_request_preproc_error(self, request: EngineCoreRequest) -> None:
1152
+ """Log and return a request-scoped error response for exceptions raised
1153
+ from the add request preprocessing in the input socket processing thread.
1154
+ """
1155
+ logger.exception(
1156
+ "Unexpected error pre-processing request %s", request.request_id
1157
+ )
1158
+ self.output_queue.put_nowait(
1159
+ (
1160
+ request.client_index,
1161
+ EngineCoreOutputs(
1162
+ engine_index=self.engine_index,
1163
+ finished_requests={request.request_id},
1164
+ outputs=[
1165
+ EngineCoreOutput(
1166
+ request_id=request.request_id,
1167
+ new_token_ids=[],
1168
+ finish_reason=FinishReason.ERROR,
1169
+ )
1170
+ ],
1171
+ ),
1172
+ )
1173
+ )
1174
+
1175
1176
class DPEngineCoreProc(EngineCoreProc):
1177
"""ZMQ-wrapper for running EngineCore in background process