A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 09:30 UTC
[Frontend] Don't log duplicate error stacktrace for every request in the batch (#9023)
Signed-off-by: Wallas Santos <wallashss@ibm.com>
Wallas Henrique committed
Oct 21, 2024 at 18:49 UTC
711f3a7806de8729e8e9cedf04e056c374d8e626
2 files changed
+53
-10
tests/mq_llm_engine/test_error_handling.py
+41
-10
@@ -59,15 +59,7 @@ async def test_evil_forward(tmp_socket):
59
await asyncio.sleep(2.0)
60
await client.check_health()
61
62
- # Throws an error in first forward pass.
63
- with pytest.raises(RAISED_ERROR):
64
- async for _ in client.generate(prompt="Hello my name is",
65
- sampling_params=SamplingParams(),
66
- request_id=uuid.uuid4()):
67
- pass
68
- assert client.errored
69
-
70
- # Engine is errored, should get ENGINE_DEAD_ERROR.
62
+ # Throws an error that should get ENGINE_DEAD_ERROR.
63
with pytest.raises(MQEngineDeadError):
64
async for _ in client.generate(prompt="Hello my name is",
65
sampling_params=SamplingParams(),
@@ -149,7 +141,7 @@ async def test_failed_abort(tmp_socket):
141
client = await engine.make_client()
142
assert client.is_running
143
152
- # Firsh check health should work.
144
+ # First check health should work.
145
await client.check_health()
146
147
# Trigger an abort on the client side.
@@ -174,6 +166,45 @@ async def test_failed_abort(tmp_socket):
166
client.close()
167
168
169
+@pytest.mark.asyncio
170
+async def test_batch_error(tmp_socket):
171
+ with RemoteMQLLMEngine(engine_args=ENGINE_ARGS,
172
+ ipc_path=tmp_socket,
173
+ run_fn=run_with_evil_abort) as engine:
174
+
175
+ client = await engine.make_client()
176
+ assert client.is_running
177
+
178
+ # First check health should work.
179
+ await client.check_health()
180
+
181
+ # Batch of requests
182
+ async def do_generate(client):
183
+ # min_tokens=2048 to keep busy the engine busy
184
+ # to get enough time to get process a request
185
+ # that will crash the engine
186
+ params = SamplingParams(min_tokens=2048, max_tokens=2048)
187
+ async for _ in client.generate(prompt="Hello my name is",
188
+ sampling_params=params,
189
+ request_id=uuid.uuid4()):
190
+ pass
191
+
192
+ tasks = [asyncio.create_task(do_generate(client)) for _ in range(10)]
193
+
194
+ # This request will force a processing batch to raise
195
+ # an exception and next the engine get errored
196
+ await client.abort(request_id="foo")
197
+
198
+ # The batch of those request failed, then they
199
+ # should get the same exception as a MQEngineDeadError.
200
+ errors = await asyncio.gather(*tasks, return_exceptions=True)
201
+ for e in errors:
202
+ assert isinstance(e, MQEngineDeadError)
203
+ assert "KeyError" in repr(e)
204
+
205
+ client.close()
206
+
207
+
208
@pytest.mark.asyncio
209
async def test_bad_request(tmp_socket):
210
with RemoteMQLLMEngine(engine_args=ENGINE_ARGS,
vllm/engine/multiprocessing/client.py
+12
@@ -204,8 +204,20 @@ class MQLLMEngineClient(EngineClient):
204
# (and record only the first one)
205
if is_engine_errored and not self._errored_with:
206
self._errored_with = exception
207
+ # If engine is errored, no matter the type of exception
208
+ # it will no longer be able to receive new requests,
209
+ # therefore we have to inform that the current
210
+ # processed requests failed as well. Send back a dead
211
+ # engine error give this feedback and also give a
212
+ # 'hint' to the server to shutdown next.
213
+ exception = self.dead_error
214
215
if request_id is None:
216
+ # If request_id is None, then the engine raised an
217
+ # exception for a batch, and we may not know the
218
+ # request that caused it, neither if it was actually
219
+ # caused by any of them (e.g. CUDA OOM). Therefore we
220
+ # broadcast the same exception for all requests.
221
for queue_i in tuple(self.output_queues.values()):
222
queue_i.put_nowait(exception)
223
else: