A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 08:00 UTC
Make `AsyncLLMEngine` more robust & fix batched abort (#969)
Signed-off-by: Antoni Baum <antoni.baum@protonmail.com> Co-authored-by: Avnish Narayan <38871737+avnishn@users.noreply.github.com>
Antoni Baum committed
Sep 7, 2023 at 13:43 UTC
c07ece5ca490a90b2b19c33ab7da2d21e015d7bd
7 files changed
+345
-55
tests/async_engine/api_server_async_engine.py
new
+51
@@ -0,0 +1,51 @@
1
+"""vllm.entrypoints.api_server with some extra logging for testing."""
2
+import argparse
3
+from typing import Any, Dict
4
+
5
+import uvicorn
6
+from fastapi.responses import JSONResponse, Response
7
+
8
+import vllm.entrypoints.api_server
9
+from vllm.engine.arg_utils import AsyncEngineArgs
10
+from vllm.engine.async_llm_engine import AsyncLLMEngine
11
+
12
+app = vllm.entrypoints.api_server.app
13
+
14
+
15
+class AsyncLLMEngineWithStats(AsyncLLMEngine):
16
+
17
+ def __init__(self, *args, **kwargs):
18
+ super().__init__(*args, **kwargs)
19
+ self._num_aborts = 0
20
+
21
+ async def abort(self, request_id: str) -> None:
22
+ await super().abort(request_id)
23
+ self._num_aborts += 1
24
+
25
+ def testing_stats(self) -> Dict[str, Any]:
26
+ return {"num_aborted_requests": self._num_aborts}
27
+
28
+
29
+@app.get("/stats")
30
+def stats() -> Response:
31
+ """Get the statistics of the engine."""
32
+ return JSONResponse(engine.testing_stats())
33
+
34
+
35
+if __name__ == "__main__":
36
+ parser = argparse.ArgumentParser()
37
+ parser.add_argument("--host", type=str, default="localhost")
38
+ parser.add_argument("--port", type=int, default=8000)
39
+ parser = AsyncEngineArgs.add_cli_args(parser)
40
+ args = parser.parse_args()
41
+
42
+ engine_args = AsyncEngineArgs.from_cli_args(args)
43
+ engine = AsyncLLMEngineWithStats.from_engine_args(engine_args,
44
+ start_engine_loop=False)
45
+ vllm.entrypoints.api_server.engine = engine
46
+ uvicorn.run(
47
+ app,
48
+ host=args.host,
49
+ port=args.port,
50
+ log_level="debug",
51
+ timeout_keep_alive=vllm.entrypoints.api_server.TIMEOUT_KEEP_ALIVE)
tests/async_engine/test_api_server.py
new
+86
@@ -0,0 +1,86 @@
1
+import subprocess
2
+import sys
3
+import time
4
+from multiprocessing import Pool
5
+from pathlib import Path
6
+
7
+import pytest
8
+import requests
9
+
10
+
11
+def _query_server(prompt: str) -> dict:
12
+ response = requests.post("http://localhost:8000/generate",
13
+ json={
14
+ "prompt": prompt,
15
+ "max_tokens": 100,
16
+ "temperature": 0,
17
+ "ignore_eos": True
18
+ })
19
+ response.raise_for_status()
20
+ return response.json()
21
+
22
+
23
+@pytest.fixture
24
+def api_server():
25
+ script_path = Path(__file__).parent.joinpath(
26
+ "api_server_async_engine.py").absolute()
27
+ uvicorn_process = subprocess.Popen([
28
+ sys.executable, "-u",
29
+ str(script_path), "--model", "facebook/opt-125m"
30
+ ])
31
+ yield
32
+ uvicorn_process.terminate()
33
+
34
+
35
+def test_api_server(api_server):
36
+ """
37
+ Run the API server and test it.
38
+
39
+ We run both the server and requests in separate processes.
40
+
41
+ We test that the server can handle incoming requests, including
42
+ multiple requests at the same time, and that it can handle requests
43
+ being cancelled without crashing.
44
+ """
45
+ with Pool(32) as pool:
46
+ # Wait until the server is ready
47
+ prompts = ["Hello world"] * 1
48
+ result = None
49
+ while not result:
50
+ try:
51
+ for result in pool.map(_query_server, prompts):
52
+ break
53
+ except:
54
+ time.sleep(1)
55
+
56
+ # Actual tests start here
57
+ # Try with 1 prompt
58
+ for result in pool.map(_query_server, prompts):
59
+ assert result
60
+
61
+ num_aborted_requests = requests.get(
62
+ "http://localhost:8000/stats").json()["num_aborted_requests"]
63
+ assert num_aborted_requests == 0
64
+
65
+ # Try with 100 prompts
66
+ prompts = ["Hello world"] * 100
67
+ for result in pool.map(_query_server, prompts):
68
+ assert result
69
+
70
+ # Cancel requests
71
+ pool.map_async(_query_server, prompts)
72
+ time.sleep(0.01)
73
+ pool.terminate()
74
+ pool.join()
75
+
76
+ # check cancellation stats
77
+ num_aborted_requests = requests.get(
78
+ "http://localhost:8000/stats").json()["num_aborted_requests"]
79
+ assert num_aborted_requests > 0
80
+
81
+ # check that server still runs after cancellations
82
+ with Pool(32) as pool:
83
+ # Try with 100 prompts
84
+ prompts = ["Hello world"] * 100
85
+ for result in pool.map(_query_server, prompts):
86
+ assert result
tests/async_engine/test_request_tracker.py
new
+54
@@ -0,0 +1,54 @@
1
+import pytest
2
+
3
+from vllm.engine.async_llm_engine import RequestTracker
4
+from vllm.outputs import RequestOutput
5
+
6
+
7
+def test_request_tracker():
8
+ tracker = RequestTracker()
9
+ stream_1 = tracker.add_request("1")
10
+ new, finished = tracker.get_new_and_finished_requests()
11
+ assert len(new) == 1
12
+ assert new[0]["request_id"] == "1"
13
+ assert not finished
14
+ assert not stream_1.finished
15
+
16
+ stream_2 = tracker.add_request("2")
17
+ stream_3 = tracker.add_request("3")
18
+ new, finished = tracker.get_new_and_finished_requests()
19
+ assert len(new) == 2
20
+ assert new[0]["request_id"] == "2"
21
+ assert new[1]["request_id"] == "3"
22
+ assert not finished
23
+ assert not stream_2.finished
24
+ assert not stream_3.finished
25
+
26
+ # request_ids must be unique
27
+ with pytest.raises(KeyError):
28
+ tracker.add_request("1")
29
+
30
+ tracker.abort_request("1")
31
+ new, finished = tracker.get_new_and_finished_requests()
32
+ assert len(finished) == 1
33
+ assert "1" in finished
34
+ assert not new
35
+ assert stream_1.finished
36
+
37
+ stream_4 = tracker.add_request("4")
38
+ tracker.abort_request("4")
39
+ new, finished = tracker.get_new_and_finished_requests()
40
+ assert len(finished) == 1
41
+ assert "4" in finished
42
+ assert not new
43
+ assert stream_4.finished
44
+
45
+ stream_5 = tracker.add_request("5")
46
+ tracker.process_request_output(
47
+ RequestOutput("2", "output", [], [], finished=True))
48
+ new, finished = tracker.get_new_and_finished_requests()
49
+ assert len(finished) == 1
50
+ assert "2" in finished
51
+ assert len(new) == 1
52
+ assert new[0]["request_id"] == "5"
53
+ assert stream_2.finished
54
+ assert not stream_5.finished
vllm/core/scheduler.py
+4
-1
@@ -92,7 +92,10 @@ class Scheduler:
92
request_id = (request_id, )
93
request_ids = set(request_id)
94
for state_queue in [self.waiting, self.running, self.swapped]:
95
- for seq_group in state_queue:
95
+ # We need to reverse the list as we are removing elements
96
+ # from it as we iterate over it. If we don't do it,
97
+ # indices will get messed up and we will skip over elements.
98
+ for seq_group in reversed(state_queue):
99
if seq_group.request_id in request_ids:
100
# Remove the sequence group from the state queue.
101
state_queue.remove(seq_group)
vllm/engine/async_llm_engine.py
+148
-54
@@ -1,7 +1,7 @@
1
import asyncio
2
import time
3
from functools import partial
4
-from typing import Any, Dict, Iterable, List, Optional, Type, Union
4
+from typing import Any, Dict, Iterable, List, Optional, Set, Tuple, Type, Union
5
6
from vllm.config import ModelConfig
7
from vllm.engine.arg_utils import AsyncEngineArgs
@@ -14,6 +14,28 @@ from vllm.sampling_params import SamplingParams
14
logger = init_logger(__name__)
15
16
17
+class AsyncEngineDeadError(RuntimeError):
18
+ pass
19
+
20
+
21
+def _raise_exception_on_finish(task: asyncio.Task,
22
+ request_tracker: "RequestTracker") -> None:
23
+ msg = ("Task finished unexpectedly. This should never happen! "
24
+ "Please open an issue on Github.")
25
+ try:
26
+ try:
27
+ task.result()
28
+ except asyncio.CancelledError:
29
+ return
30
+ except Exception as exc:
31
+ raise AsyncEngineDeadError(
32
+ msg + " See stack trace above for the actual cause.") from exc
33
+ raise AsyncEngineDeadError(msg)
34
+ except Exception as exc:
35
+ request_tracker.propagate_exception(exc)
36
+ raise exc
37
+
38
+
39
class AsyncStream:
40
"""A stream of RequestOutputs for a request that can be
41
iterated over asynchronously."""
@@ -43,15 +65,90 @@ class AsyncStream:
65
result = await self._queue.get()
66
if result is StopIteration:
67
raise StopAsyncIteration
68
+ elif isinstance(result, Exception):
69
+ raise result
70
return result
71
72
49
-def _raise_exception_on_finish(task: asyncio.Task) -> None:
50
- try:
51
- task.result()
52
- except Exception as e:
53
- raise RuntimeError("Task finished unexpectedly.") from e
54
- raise RuntimeError("Task finished unexpectedly.")
73
+class RequestTracker:
74
+ """Synchronous abstraction for tracking requests."""
75
+
76
+ def __init__(self) -> None:
77
+ self._request_streams: Dict[str, AsyncStream] = {}
78
+ self._finished_requests: asyncio.Queue[str] = asyncio.Queue()
79
+ self._new_requests: asyncio.Queue[Tuple[AsyncStream,
80
+ dict]] = asyncio.Queue()
81
+
82
+ def __contains__(self, item):
83
+ return item in self._request_streams
84
+
85
+ def propagate_exception(self, exc: Exception) -> None:
86
+ """Propagate an exception to all request streams."""
87
+ for stream in self._request_streams.values():
88
+ stream.put(exc)
89
+
90
+ def process_request_output(self,
91
+ request_output: RequestOutput,
92
+ *,
93
+ verbose: bool = False) -> None:
94
+ """Process a request output from the engine."""
95
+ request_id = request_output.request_id
96
+
97
+ self._request_streams[request_id].put(request_output)
98
+ if request_output.finished:
99
+ if verbose:
100
+ logger.info(f"Finished request {request_id}.")
101
+ self.abort_request(request_id)
102
+
103
+ def add_request(self, request_id: str,
104
+ **engine_add_request_kwargs) -> AsyncStream:
105
+ """Add a request to be sent to the engine on the next background
106
+ loop iteration."""
107
+ if request_id in self._request_streams:
108
+ raise KeyError(f"Request {request_id} already exists.")
109
+
110
+ stream = AsyncStream(request_id)
111
+ self._new_requests.put_nowait((stream, {
112
+ "request_id": request_id,
113
+ **engine_add_request_kwargs
114
+ }))
115
+ return stream
116
+
117
+ def abort_request(self, request_id: str, *, verbose: bool = False) -> None:
118
+ """Abort a request during next background loop iteration."""
119
+ if verbose:
120
+ logger.info(f"Aborted request {request_id}.")
121
+
122
+ self._finished_requests.put_nowait(request_id)
123
+
124
+ if request_id not in self._request_streams or self._request_streams[
125
+ request_id].finished:
126
+ # The request has already finished or been aborted.
127
+ return
128
+
129
+ self._request_streams[request_id].finish()
130
+
131
+ def get_new_and_finished_requests(self) -> Tuple[List[dict], Set[str]]:
132
+ """Get the new requests and finished requests to be
133
+ sent to the engine."""
134
+ new_requests: List[dict] = []
135
+ finished_requests: Set[str] = set()
136
+
137
+ while not self._finished_requests.empty():
138
+ request_id = self._finished_requests.get_nowait()
139
+ finished_requests.add(request_id)
140
+ self._request_streams.pop(request_id, None)
141
+
142
+ while not self._new_requests.empty():
143
+ stream, new_request = self._new_requests.get_nowait()
144
+ if stream.request_id in finished_requests:
145
+ # The request has already been aborted.
146
+ stream.finish()
147
+ continue
148
+ self._request_streams[stream.request_id] = stream
149
+ new_requests.append(new_request)
150
+
151
+ return new_requests, finished_requests
152
153
154
class _AsyncLLMEngine(LLMEngine):
@@ -150,16 +247,15 @@ class AsyncLLMEngine:
247
self.log_requests = log_requests
248
self.engine = self._init_engine(*args, **kwargs)
249
153
- # Request id -> stream.
154
- self.request_streams: Dict[str, AsyncStream] = {}
155
- self.finished_requests: asyncio.Queue[str] = asyncio.Queue()
250
+ self.request_tracker: RequestTracker = RequestTracker()
251
self.background_loop = None
252
if start_engine_loop:
253
self.start_background_loop()
254
255
@property
256
def is_running(self) -> bool:
162
- return self.background_loop is not None
257
+ return (self.background_loop is not None
258
+ and not self.background_loop.done())
259
260
def start_background_loop(self) -> None:
261
"""Start the background loop."""
@@ -167,7 +263,9 @@ class AsyncLLMEngine:
263
raise RuntimeError("Background loop is already running.")
264
self.background_loop = asyncio.get_event_loop().create_task(
265
self.run_engine_loop())
170
- self.background_loop.add_done_callback(_raise_exception_on_finish)
266
+ self.background_loop.add_done_callback(
267
+ partial(_raise_exception_on_finish,
268
+ request_tracker=self.request_tracker))
269
270
def _init_engine(self, *args,
271
**kwargs) -> Union[_AsyncLLMEngine, "ray.ObjectRef"]:
@@ -181,6 +279,21 @@ class AsyncLLMEngine:
279
280
async def engine_step(self):
281
"""Kick the engine to process the waiting requests."""
282
+
283
+ new_requests, finished_requests = (
284
+ self.request_tracker.get_new_and_finished_requests())
285
+
286
+ for new_request in new_requests:
287
+ # Add the request into the vLLM engine's waiting queue.
288
+ # TODO: Maybe add add_request_batch to reduce Ray overhead
289
+ if self.engine_use_ray:
290
+ await self.engine.add_request.remote(**new_request)
291
+ else:
292
+ self.engine.add_request(**new_request)
293
+
294
+ if finished_requests:
295
+ await self._engine_abort(finished_requests)
296
+
297
if self.engine_use_ray:
298
request_outputs = await self.engine.step.remote()
299
else:
@@ -188,20 +301,8 @@ class AsyncLLMEngine:
301
302
# Put the outputs into the corresponding streams.
303
for request_output in request_outputs:
191
- request_id = request_output.request_id
192
- self.request_streams[request_id].put(request_output)
193
- if request_output.finished:
194
- if self.log_requests:
195
- logger.info(f"Finished request {request_id}.")
196
- self.request_streams[request_id].finish()
197
- self.finished_requests.put_nowait(request_id)
198
-
199
- finished_request = set()
200
- while not self.finished_requests.empty():
201
- finished_request.add(self.finished_requests.get_nowait())
202
- await self._engine_abort(finished_request)
203
- for request_id in finished_request:
204
- del self.request_streams[request_id]
304
+ self.request_tracker.process_request_output(
305
+ request_output, verbose=self.log_requests)
306
307
async def _engine_abort(self, request_ids: Iterable[str]):
308
if self.engine_use_ray:
@@ -228,25 +329,19 @@ class AsyncLLMEngine:
329
f"sampling params: {sampling_params}, "
330
f"prompt token ids: {prompt_token_ids}.")
331
231
- if request_id in self.request_streams:
232
- raise KeyError(f"Request {request_id} already exists.")
233
- stream = AsyncStream(request_id)
234
- self.request_streams[request_id] = stream
332
+ if not self.is_running:
333
+ raise AsyncEngineDeadError(
334
+ "Background loop is not running. If it was running, "
335
+ "inspect the output to find the stacktrace of the "
336
+ "error that caused the background loop to stop "
337
+ "(AsyncEngineDeadError).")
338
236
- # Add the request into the vLLM engine's waiting queue.
237
- if self.engine_use_ray:
238
- await self.engine.add_request.remote(
239
- request_id,
240
- prompt,
241
- sampling_params,
242
- prompt_token_ids=prompt_token_ids,
243
- arrival_time=arrival_time)
244
- else:
245
- self.engine.add_request(request_id,
246
- prompt,
247
- sampling_params,
248
- prompt_token_ids=prompt_token_ids,
249
- arrival_time=arrival_time)
339
+ stream = self.request_tracker.add_request(
340
+ request_id,
341
+ prompt=prompt,
342
+ sampling_params=sampling_params,
343
+ prompt_token_ids=prompt_token_ids,
344
+ arrival_time=arrival_time)
345
346
return stream
347
@@ -300,6 +395,13 @@ class AsyncLLMEngine:
395
Args:
396
request_id: The unique id of the request.
397
"""
398
+ if not self.is_running:
399
+ raise AsyncEngineDeadError(
400
+ "Background loop is not running. If it was running, "
401
+ "inspect the output to find the stacktrace of the "
402
+ "error that caused the background loop to stop "
403
+ "(AsyncEngineDeadError).")
404
+
405
return self._abort(request_id)
406
407
def _abort(self, request_id: str) -> None:
@@ -311,16 +413,8 @@ class AsyncLLMEngine:
413
Args:
414
request_id: The unique id of the request.
415
"""
314
- if request_id not in self.request_streams or self.request_streams[
315
- request_id].finished:
316
- # The request has already finished or been aborted.
317
- return
318
-
319
- if self.log_requests:
320
- logger.info(f"Aborted request {request_id}.")
321
-
322
- self.request_streams[request_id].finish()
323
- self.finished_requests.put_nowait(request_id)
416
+ self.request_tracker.abort_request(request_id,
417
+ verbose=self.log_requests)
418
419
async def get_model_config(self) -> ModelConfig:
420
"""Get the model configuration of the vLLM engine."""
vllm/entrypoints/api_server.py
+1
@@ -14,6 +14,7 @@ from vllm.utils import random_uuid
14
TIMEOUT_KEEP_ALIVE = 5 # seconds.
15
TIMEOUT_TO_PREVENT_DEADLOCK = 1 # seconds.
16
app = FastAPI()
17
+engine = None
18
19
20
@app.post("/generate")
vllm/entrypoints/openai/api_server.py
+1
@@ -44,6 +44,7 @@ TIMEOUT_KEEP_ALIVE = 5 # seconds
44
logger = init_logger(__name__)
45
served_model = None
46
app = fastapi.FastAPI()
47
+engine = None
48
49
50
def create_error_response(status_code: HTTPStatus,