@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 711f3a7806
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 09:30 UTC

[Frontend] Don't log duplicate error stacktrace for every request in the batch (#9023)

Signed-off-by: Wallas Santos <wallashss@ibm.com>

Wallas Henrique committed Oct 21, 2024 at 18:49 UTC 711f3a7806de8729e8e9cedf04e056c374d8e626
2 files changed +53 -10
tests/mq_llm_engine/test_error_handling.py
+41 -10
@@ -59,15 +59,7 @@ async def test_evil_forward(tmp_socket):
59 await asyncio.sleep(2.0)
60 await client.check_health()
61
62 - # Throws an error in first forward pass.
63 - with pytest.raises(RAISED_ERROR):
64 - async for _ in client.generate(prompt="Hello my name is",
65 - sampling_params=SamplingParams(),
66 - request_id=uuid.uuid4()):
67 - pass
68 - assert client.errored
69 -
70 - # Engine is errored, should get ENGINE_DEAD_ERROR.
62 + # Throws an error that should get ENGINE_DEAD_ERROR.
63 with pytest.raises(MQEngineDeadError):
64 async for _ in client.generate(prompt="Hello my name is",
65 sampling_params=SamplingParams(),
@@ -149,7 +141,7 @@ async def test_failed_abort(tmp_socket):
141 client = await engine.make_client()
142 assert client.is_running
143
152 - # Firsh check health should work.
144 + # First check health should work.
145 await client.check_health()
146
147 # Trigger an abort on the client side.
@@ -174,6 +166,45 @@ async def test_failed_abort(tmp_socket):
166 client.close()
167
168
169 +@pytest.mark.asyncio
170 +async def test_batch_error(tmp_socket):
171 + with RemoteMQLLMEngine(engine_args=ENGINE_ARGS,
172 + ipc_path=tmp_socket,
173 + run_fn=run_with_evil_abort) as engine:
174 +
175 + client = await engine.make_client()
176 + assert client.is_running
177 +
178 + # First check health should work.
179 + await client.check_health()
180 +
181 + # Batch of requests
182 + async def do_generate(client):
183 + # min_tokens=2048 to keep busy the engine busy
184 + # to get enough time to get process a request
185 + # that will crash the engine
186 + params = SamplingParams(min_tokens=2048, max_tokens=2048)
187 + async for _ in client.generate(prompt="Hello my name is",
188 + sampling_params=params,
189 + request_id=uuid.uuid4()):
190 + pass
191 +
192 + tasks = [asyncio.create_task(do_generate(client)) for _ in range(10)]
193 +
194 + # This request will force a processing batch to raise
195 + # an exception and next the engine get errored
196 + await client.abort(request_id="foo")
197 +
198 + # The batch of those request failed, then they
199 + # should get the same exception as a MQEngineDeadError.
200 + errors = await asyncio.gather(*tasks, return_exceptions=True)
201 + for e in errors:
202 + assert isinstance(e, MQEngineDeadError)
203 + assert "KeyError" in repr(e)
204 +
205 + client.close()
206 +
207 +
208 @pytest.mark.asyncio
209 async def test_bad_request(tmp_socket):
210 with RemoteMQLLMEngine(engine_args=ENGINE_ARGS,
vllm/engine/multiprocessing/client.py
+12
@@ -204,8 +204,20 @@ class MQLLMEngineClient(EngineClient):
204 # (and record only the first one)
205 if is_engine_errored and not self._errored_with:
206 self._errored_with = exception
207 + # If engine is errored, no matter the type of exception
208 + # it will no longer be able to receive new requests,
209 + # therefore we have to inform that the current
210 + # processed requests failed as well. Send back a dead
211 + # engine error give this feedback and also give a
212 + # 'hint' to the server to shutdown next.
213 + exception = self.dead_error
214
215 if request_id is None:
216 + # If request_id is None, then the engine raised an
217 + # exception for a batch, and we may not know the
218 + # request that caused it, neither if it was actually
219 + # caused by any of them (e.g. CUDA OOM). Therefore we
220 + # broadcast the same exception for all requests.
221 for queue_i in tuple(self.output_queues.values()):
222 queue_i.put_nowait(exception)
223 else: