@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / b903e1ba7f
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 18:15 UTC

[Frontend] error suppression cleanup (#7786)

Signed-off-by: Joe Runde <Joseph.Runde@ibm.com>

Joe Runde committed Aug 22, 2024 at 15:50 UTC b903e1ba7fca15f0dc49ab49a9ec8107f625c048
3 files changed +18 -7
tests/entrypoints/openai/rpc/test_zmq_client.py
+4 -3
@@ -75,11 +75,12 @@ async def test_client_aborts_use_timeouts(monkeypatch, dummy_server,
75 m.setattr(dummy_server, "abort", lambda x: None)
76 m.setattr(client, "_data_timeout", 10)
77
78 - # Ensure the client doesn't hang
78 + # The client should suppress timeouts on `abort`s
79 + # and return normally, assuming the server will eventually
80 + # abort the request.
81 client_task = asyncio.get_running_loop().create_task(
82 client.abort("test request id"))
81 - with pytest.raises(TimeoutError, match="Server didn't reply within"):
82 - await asyncio.wait_for(client_task, timeout=0.05)
83 + await asyncio.wait_for(client_task, timeout=0.05)
84
85
86 @pytest.mark.asyncio
vllm/entrypoints/openai/api_server.py
+2 -3
@@ -6,7 +6,7 @@ import os
6 import re
7 import tempfile
8 from argparse import Namespace
9 -from contextlib import asynccontextmanager, suppress
9 +from contextlib import asynccontextmanager
10 from http import HTTPStatus
11 from typing import AsyncIterator, Optional, Set
12
@@ -83,8 +83,7 @@ async def lifespan(app: FastAPI):
83 async def _force_log():
84 while True:
85 await asyncio.sleep(10)
86 - with suppress(Exception):
87 - await async_engine_client.do_log_stats()
86 + await async_engine_client.do_log_stats()
87
88 if not engine_args.disable_log_stats:
89 task = asyncio.create_task(_force_log())
vllm/entrypoints/openai/rpc/client.py
+12 -1
@@ -335,7 +335,18 @@ class AsyncEngineRPCClient:
335
336 async def abort(self, request_id: str):
337 """Send an ABORT_REQUEST signal to the RPC Server"""
338 - with suppress(RPCClientClosedError):
338 +
339 + # Suppress timeouts as well.
340 + # In cases where the server is busy processing requests and a very
341 + # large volume of abort requests arrive, it is likely that the server
342 + # will not be able to ack all of them in time. We have seen this when
343 + # we abort 20k requests at once while another 2k are processing- many
344 + # of them time out, but we see the server successfully abort all of the
345 + # requests.
346 + # In this case we assume that the server has received or will receive
347 + # these abort requests, and ignore the timeout. This prevents a massive
348 + # wall of `TimeoutError` stack traces.
349 + with suppress(RPCClientClosedError, TimeoutError):
350 await self._send_one_way_rpc_request(
351 request=RPCAbortRequest(request_id),
352 error_message=f"RPCAbortRequest {request_id} failed")