A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 5, 11:00 UTC
[BugFix] --max-model-len=-1 causes over-limit requests to hang and starve the entire service (#39102)
Signed-off-by: triangle14 <y1019026570@gmail.com> Signed-off-by: mgoin <mgoin64@gmail.com> Co-authored-by: mgoin <mgoin64@gmail.com>
triangleXIV committed
Apr 9, 2026 at 05:03 UTC
7c94ae16c6c265a095235ee90c87226e931ca409
5 files changed
+82
-4
tests/v1/e2e/general/test_context_length.py
+40
@@ -15,6 +15,7 @@ import pytest
15
16
from tests.conftest import VllmRunner
17
from tests.utils import create_new_process_for_each_test
18
+from vllm.exceptions import VLLMValidationError
19
20
21
@create_new_process_for_each_test()
@@ -61,3 +62,42 @@ def test_decoder_max_context_length_validation(
62
with pytest.raises(ValueError) as excinfo:
63
vllm_model.generate_greedy(prompt_ids, max_tokens)
64
assert expected_msg in str(excinfo.value)
65
+
66
+
67
+@create_new_process_for_each_test()
68
+@pytest.mark.parametrize("model", ["JackFram/llama-160m"])
69
+def test_auto_fit_max_model_len_rejects_oversized_input(
70
+ model: str,
71
+ vllm_runner: type[VllmRunner],
72
+) -> None:
73
+ """When max_model_len='auto' and KV cache memory is very limited,
74
+ the engine auto-fits max_model_len to a small value. The frontend
75
+ must see this reduced value and reject prompts that exceed it,
76
+ rather than accepting them and hanging."""
77
+
78
+ # Use a tiny KV cache budget to force auto-fit to a very small
79
+ # max_model_len (e.g. ~16 tokens).
80
+ kv_cache_bytes = 1_000_000 # 1 MB
81
+
82
+ with vllm_runner(
83
+ model_name=model,
84
+ max_model_len=-1,
85
+ max_num_seqs=1,
86
+ enforce_eager=True,
87
+ kv_cache_memory_bytes=kv_cache_bytes,
88
+ load_format="dummy",
89
+ ) as vllm_model:
90
+ auto_fitted_len = (
91
+ vllm_model.llm.llm_engine.vllm_config.model_config.max_model_len
92
+ )
93
+ # Sanity check: auto-fit should have reduced it well below the
94
+ # model's native context length.
95
+ assert auto_fitted_len < 2048, (
96
+ f"Expected auto-fit to reduce max_model_len significantly, "
97
+ f"but got {auto_fitted_len}"
98
+ )
99
+
100
+ # A prompt longer than the auto-fitted length must be rejected.
101
+ oversized_prompt = [[43] * (auto_fitted_len + 10)]
102
+ with pytest.raises(VLLMValidationError, match="Please reduce the length"):
103
+ vllm_model.generate_greedy(oversized_prompt, max_tokens=4)
tests/v1/engine/test_engine_core_client.py
+1
-1
@@ -114,7 +114,7 @@ def test_mp_client_uses_env_timeout(monkeypatch: pytest.MonkeyPatch):
114
return 1
115
116
def recv_multipart(self):
117
- return (b"\x00\x00", b"ready")
117
+ return (b"\x00\x00", b"")
118
119
class DummySocket:
120
def send_multipart(self, _msg, *, copy: bool = False, track: bool = False):
vllm/v1/engine/__init__.py
+14
@@ -63,6 +63,20 @@ class FinishReason(enum.IntEnum):
63
return FINISH_REASON_STRINGS[self.value]
64
65
66
+class EngineCoreReadyResponse(
67
+ msgspec.Struct,
68
+ array_like=True, # type: ignore[call-arg]
69
+ omit_defaults=True, # type: ignore[call-arg]
70
+):
71
+ """Sent from EngineCore to the frontend during the ready handshake.
72
+
73
+ Contains post-initialization config that may differ from the original
74
+ values (e.g. max_model_len after KV cache auto-fitting).
75
+ """
76
+
77
+ max_model_len: int | None = None
78
+
79
+
80
class EngineCoreRequest(
81
msgspec.Struct,
82
array_like=True, # type: ignore[call-arg]
vllm/v1/engine/core.py
+6
-1
@@ -52,6 +52,7 @@ from vllm.v1.engine import (
52
EEPNotificationType,
53
EngineCoreOutput,
54
EngineCoreOutputs,
55
+ EngineCoreReadyResponse,
56
EngineCoreRequest,
57
EngineCoreRequestType,
58
FinishReason,
@@ -1385,11 +1386,15 @@ class EngineCoreProc(EngineCore):
1386
1387
# Register sockets with poller.
1388
poller = zmq.Poller()
1389
+ ready_response = EngineCoreReadyResponse(
1390
+ max_model_len=self.vllm_config.model_config.max_model_len,
1391
+ )
1392
+ ready_payload = msgspec.msgpack.encode(ready_response)
1393
for input_socket in input_sockets:
1394
# Send initial message to each input socket - this is required
1395
# before the front-end ROUTER socket can send input messages
1396
# back to us.
1392
- input_socket.send(b"")
1397
+ input_socket.send(ready_payload)
1398
poller.register(input_socket, zmq.POLLIN)
1399
1400
if coord_socket is not None:
vllm/v1/engine/core_client.py
+21
-2
@@ -35,6 +35,7 @@ from vllm.v1.engine import (
35
EEP_NOTIFICATION_CALL_ID,
36
EEPNotificationType,
37
EngineCoreOutputs,
38
+ EngineCoreReadyResponse,
39
EngineCoreRequest,
40
EngineCoreRequestType,
41
PauseMode,
@@ -456,6 +457,22 @@ class ElasticScalingCache:
457
pending_notifications: dict[EEPNotificationType, set[int]]
458
459
460
+_ready_response_decoder = msgspec.msgpack.Decoder(EngineCoreReadyResponse)
461
+
462
+
463
+def _apply_ready_response(payload: bytes, vllm_config: VllmConfig) -> None:
464
+ """Decode an EngineCoreReadyResponse and sync any post-initialization
465
+ config changes (e.g. auto-fitted max_model_len) back to the frontend."""
466
+ if not payload:
467
+ return
468
+ response = _ready_response_decoder.decode(payload)
469
+ if response.max_model_len is not None:
470
+ vllm_config.model_config.max_model_len = min(
471
+ vllm_config.model_config.max_model_len,
472
+ response.max_model_len,
473
+ )
474
+
475
+
476
class MPClient(EngineCoreClient):
477
"""
478
MPClient: base client for multi-proc EngineCore.
@@ -589,8 +606,9 @@ class MPClient(EngineCoreClient):
606
f"timeout, set the environment variable: "
607
f"VLLM_ENGINE_READY_TIMEOUT_S=<seconds>"
608
)
592
- identity, _ = sync_input_socket.recv_multipart()
609
+ identity, payload = sync_input_socket.recv_multipart()
610
identities.remove(identity)
611
+ _apply_ready_response(payload, vllm_config)
612
613
self.core_engine: EngineIdentity = self.core_engines[0]
614
self.utility_results: dict[int, AnyFuture] = {}
@@ -1582,8 +1600,9 @@ class DPLBAsyncMPClient(DPAsyncMPClient):
1600
f"timeout, set the environment variable: "
1601
f"VLLM_ENGINE_READY_TIMEOUT_S=<seconds>"
1602
)
1585
- identity, _ = sync_input_socket.recv_multipart()
1603
+ identity, payload = sync_input_socket.recv_multipart()
1604
new_engine_identities.discard(identity)
1605
+ _apply_ready_response(payload, self.vllm_config)
1606
1607
# NOTE(yongji): Before we schedule any requests on the new workers,
1608
# we should wait for them to switch to the new setup.